From a40f6b9606f64449a213330a1dfad5b36ac9e61d Mon Sep 17 00:00:00 2001 From: kexi1 Date: Thu, 24 Oct 2024 15:13:22 +0800 Subject: [PATCH 1/2] judge the keyboard using adb --- Mobile-Agent-v2/MobileAgent/controller.py | 14 ++++++++++++++ Mobile-Agent-v2/run.py | 13 ++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/Mobile-Agent-v2/MobileAgent/controller.py b/Mobile-Agent-v2/MobileAgent/controller.py index dac33ce..2a60d23 100644 --- a/Mobile-Agent-v2/MobileAgent/controller.py +++ b/Mobile-Agent-v2/MobileAgent/controller.py @@ -57,4 +57,18 @@ def back(adb_path): def home(adb_path): command = adb_path + f" shell am start -a android.intent.action.MAIN -c android.intent.category.HOME" + subprocess.run(command, capture_output=True, text=True, shell=True) + +def get_all_input_method(adb_path): + command = adb_path + " shell ime list -a" + result = subprocess.run(command, capture_output=True, text=True, shell=True) + return result.stdout + +def get_current_input_method(adb_path): + command = adb_path + " shell settings get secure default_input_method" + result = subprocess.run(command, capture_output=True, text=True, shell=True) + return result.stdout.strip() + +def set_input_method(adb_path, package="com.android.adbkeyboard/.AdbIME"): + command = adb_path + f" shell settings put secure default_input_method " + package subprocess.run(command, capture_output=True, text=True, shell=True) \ No newline at end of file diff --git a/Mobile-Agent-v2/run.py b/Mobile-Agent-v2/run.py index cda145e..bb07db5 100644 --- a/Mobile-Agent-v2/run.py +++ b/Mobile-Agent-v2/run.py @@ -8,7 +8,7 @@ from MobileAgent.api import inference_chat from MobileAgent.text_localization import ocr from MobileAgent.icon_localization import det -from MobileAgent.controller import get_screenshot, tap, slide, type, back, home +from MobileAgent.controller import get_screenshot, tap, slide, type, back, home, get_all_input_method, get_current_input_method, set_input_method from MobileAgent.prompt import get_action_prompt, get_reflect_prompt, get_memory_prompt, get_process_prompt from MobileAgent.chat import init_action_chat, init_reflect_chat, init_memory_chat, add_response, add_response_two_image @@ -291,13 +291,12 @@ def get_perception_infos(adb_path, screenshot_file): os.mkdir(temp_file) keyboard = False - keyboard_height_limit = 0.9 * height - for perception_info in perception_infos: - if perception_info['coordinates'][1] < keyboard_height_limit: - continue - if 'ADB Keyboard' in perception_info['text']: + if not "adbkeyboard" in get_current_input_method(adb_path): + if "adbkeyboard" in get_all_input_method(adb_path): + set_input_method(adb_path) keyboard = True - break + else: + keyboard = True prompt_action = get_action_prompt(instruction, perception_infos, width, height, keyboard, summary_history, action_history, summary, action, add_info, error_flag, completed_requirements, memory) chat_action = init_action_chat() From 0939b76affd088bf0af1062fd0b3239969d00b3e Mon Sep 17 00:00:00 2001 From: kexi <103545608+kx-kexi@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:19:23 +0800 Subject: [PATCH 2/2] judge keyboard using adb --- Mobile-Agent-v2/run.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Mobile-Agent-v2/run.py b/Mobile-Agent-v2/run.py index bb07db5..33a5e02 100644 --- a/Mobile-Agent-v2/run.py +++ b/Mobile-Agent-v2/run.py @@ -280,7 +280,13 @@ def get_perception_infos(adb_path, screenshot_file): os.mkdir(screenshot) error_flag = False - +keyboard = False +if not "adbkeyboard" in get_current_input_method(adb_path): + if "adbkeyboard" in get_all_input_method(adb_path): + set_input_method(adb_path) + keyboard = True +else: + keyboard = True iter = 0 while True: iter += 1 @@ -289,14 +295,6 @@ def get_perception_infos(adb_path, screenshot_file): perception_infos, width, height = get_perception_infos(adb_path, screenshot_file) shutil.rmtree(temp_file) os.mkdir(temp_file) - - keyboard = False - if not "adbkeyboard" in get_current_input_method(adb_path): - if "adbkeyboard" in get_all_input_method(adb_path): - set_input_method(adb_path) - keyboard = True - else: - keyboard = True prompt_action = get_action_prompt(instruction, perception_infos, width, height, keyboard, summary_history, action_history, summary, action, add_info, error_flag, completed_requirements, memory) chat_action = init_action_chat()