diff --git a/.github/badges/coverage.svg b/.github/badges/coverage.svg index 9029937..b750dd9 100644 --- a/.github/badges/coverage.svg +++ b/.github/badges/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 76% - 76% + 77% + 77% diff --git a/docs/assets/tappa_setup_1.png b/docs/assets/tappa_setup_1.png new file mode 100644 index 0000000..d7e8d60 Binary files /dev/null and b/docs/assets/tappa_setup_1.png differ diff --git a/docs/assets/tappa_setup_2.png b/docs/assets/tappa_setup_2.png new file mode 100644 index 0000000..4e68081 Binary files /dev/null and b/docs/assets/tappa_setup_2.png differ diff --git a/docs/assets/tappa_setup_3.png b/docs/assets/tappa_setup_3.png new file mode 100644 index 0000000..a987fcd Binary files /dev/null and b/docs/assets/tappa_setup_3.png differ diff --git a/docs/assets/tappa_setup_4.png b/docs/assets/tappa_setup_4.png new file mode 100644 index 0000000..3c52264 Binary files /dev/null and b/docs/assets/tappa_setup_4.png differ diff --git a/docs/emu_setup.md b/docs/emu_setup.md index 8c6c70d..2189314 100644 --- a/docs/emu_setup.md +++ b/docs/emu_setup.md @@ -137,6 +137,40 @@ And disable the clipboard suggestions : ![](assets/swiftkey_setup_4.png){ width="300" } +### Preparing Tappa keyboard + +Tappa keyboard isn't installed on the emulator by default : you need to install it first. + +!!! note + If you want to run the tests in parallel on several emulators, you need to repeat these steps for each emulator. + +Start the emulator, then go to Google, and paste [this link](https://play.google.com/store/apps/details?id=com.tappa.keyboard&hl=en_US) to install Tappa keyboard. + +!!! tip + If the clipboard isn't shared with the emulator, open a terminal and run : + + ```bash + adb shell input text "https://play.google.com/store/apps/details?id=com.tappa.keyboard&hl=en_US" + ``` + +Install the keyboard on your emulator : + +![](assets/tappa_setup_1.png){ width="300" } + +Open the app, follow the instructions to activate the keyboard. + +--- + +By default, Tappa keyboard suggests contact names, and it may interfere with the results across runs. You should disable the option. First, access the text correction settings : + +![](assets/tappa_setup_2.png){ width="300" } + +![](assets/tappa_setup_3.png){ width="300" } + +And disable "Personalized suggestions" and "Suggest Contact names" : + +![](assets/tappa_setup_4.png){ width="300" } + ## Setting up iOS emulator ### Creating the emulator diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 4a473d9..7f623dc 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -10,6 +10,8 @@ >>>KeyboardKit Open-source|results/keyboardkit_oss.json >>>KeyboardKit Pro|results/keyboardkit_pro.json >>>Gboard|results/gboard.json +>>>Swiftkey|results/swiftkey.json +>>>Tappa keyboard|results/tappa.json !!! info The metrics used in this leaderboard are : diff --git a/docs/results/swiftkey.json b/docs/results/swiftkey.json new file mode 100644 index 0000000..c1d1f26 --- /dev/null +++ b/docs/results/swiftkey.json @@ -0,0 +1,362 @@ +{ + "next_word_prediction": { + "score": { + "accuracy": 0.12, + "top3_accuracy": 0.33, + "n": 2436 + }, + "per_domain": { + "narrative": { + "accuracy": 0.1, + "top3_accuracy": 0.32, + "n": 1556 + }, + "dialogue": { + "accuracy": 0.14, + "top3_accuracy": 0.35, + "n": 880 + } + }, + "performances": { + "mean_memory": "599 KB", + "min_memory": "347 KB", + "max_memory": "628 KB", + "mean_runtime": "4.52 s", + "fastest_runtime": "577 ms", + "slowest_runtime": "29.2 s" + } + }, + "auto_completion": { + "score": { + "accuracy": 0.45, + "top3_accuracy": 0.76, + "n": 2426 + }, + "per_domain": { + "narrative": { + "accuracy": 0.43, + "top3_accuracy": 0.74, + "n": 1557 + }, + "dialogue": { + "accuracy": 0.5, + "top3_accuracy": 0.79, + "n": 869 + } + }, + "per_completion_rate": { + "<25%": { + "accuracy": 0.27, + "top3_accuracy": 0.33, + "n": 73 + }, + "25%~50%": { + "accuracy": 0.51, + "top3_accuracy": 0.61, + "n": 435 + }, + "50%~75%": { + "accuracy": 0.51, + "top3_accuracy": 0.78, + "n": 1363 + }, + ">75%": { + "accuracy": 0.29, + "top3_accuracy": 0.88, + "n": 555 + } + }, + "per_other": { + "without_typo": { + "accuracy": 0.47, + "top3_accuracy": 0.79, + "n": 2247 + }, + "with_typo": { + "accuracy": 0.28, + "top3_accuracy": 0.38, + "n": 179 + } + }, + "performances": { + "mean_memory": "599 KB", + "min_memory": "330 KB", + "max_memory": "628 KB", + "mean_runtime": "22.7 s", + "fastest_runtime": "10.3 s", + "slowest_runtime": "74.6 s" + } + }, + "auto_correction": { + "score": { + "accuracy": 0.93, + "precision": 0.83, + "recall": 0.55, + "fscore": 0.68, + "top3_accuracy": 0.94, + "top3_precision": 0.88, + "top3_recall": 0.61, + "top3_fscore": 0.73, + "n_typo": 328, + "n": 2531 + }, + "per_domain": { + "narrative": { + "accuracy": 0.92, + "precision": 0.75, + "recall": 0.5, + "fscore": 0.61, + "top3_accuracy": 0.93, + "top3_precision": 0.83, + "top3_recall": 0.57, + "top3_fscore": 0.69, + "n_typo": 204, + "n": 1601 + }, + "dialogue": { + "accuracy": 0.95, + "precision": 0.95, + "recall": 0.64, + "fscore": 0.78, + "top3_accuracy": 0.95, + "top3_precision": 0.96, + "top3_recall": 0.66, + "top3_fscore": 0.8, + "n_typo": 124, + "n": 930 + } + }, + "per_typo_type": { + "DELETE_SPELLING_SYMBOL": { + "accuracy": 0.95, + "precision": 0.89, + "recall": 0.73, + "fscore": 0.81, + "top3_accuracy": 0.95, + "top3_precision": 0.89, + "top3_recall": 0.73, + "top3_fscore": 0.81, + "n_typo": 11, + "n": 85 + }, + "DELETE_SPACE": { + "accuracy": 0.99, + "precision": 0.9, + "recall": 1.0, + "fscore": 0.94, + "top3_accuracy": 1.0, + "top3_precision": 1.0, + "top3_recall": 1.0, + "top3_fscore": 1.0, + "n_typo": 9, + "n": 69 + }, + "DELETE_PUNCTUATION": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "DELETE_CHAR": { + "accuracy": 0.95, + "precision": 0.85, + "recall": 0.77, + "fscore": 0.81, + "top3_accuracy": 0.96, + "top3_precision": 0.9, + "top3_recall": 0.82, + "top3_fscore": 0.86, + "n_typo": 22, + "n": 170 + }, + "ADD_SPELLING_SYMBOL": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_SPACE": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_PUNCTUATION": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_CHAR": { + "accuracy": 0.96, + "precision": 0.86, + "recall": 0.81, + "fscore": 0.84, + "top3_accuracy": 0.97, + "top3_precision": 0.89, + "top3_recall": 0.84, + "top3_fscore": 0.86, + "n_typo": 37, + "n": 286 + }, + "SUBSTITUTE_CHAR": { + "accuracy": 0.89, + "precision": 0.71, + "recall": 0.28, + "fscore": 0.42, + "top3_accuracy": 0.91, + "top3_precision": 0.83, + "top3_recall": 0.38, + "top3_fscore": 0.54, + "n_typo": 53, + "n": 409 + }, + "SIMPLIFY_ACCENT": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "SIMPLIFY_CASE": { + "accuracy": 0.88, + "precision": 0.57, + "recall": 0.21, + "fscore": 0.32, + "top3_accuracy": 0.88, + "top3_precision": 0.67, + "top3_recall": 0.21, + "top3_fscore": 0.34, + "n_typo": 19, + "n": 147 + }, + "TRANSPOSE_CHAR": { + "accuracy": 0.95, + "precision": 0.86, + "recall": 0.76, + "fscore": 0.81, + "top3_accuracy": 0.97, + "top3_precision": 0.91, + "top3_recall": 0.81, + "top3_fscore": 0.86, + "n_typo": 75, + "n": 579 + }, + "COMMON_TYPO": { + "accuracy": 0.91, + "precision": 0.79, + "recall": 0.41, + "fscore": 0.56, + "top3_accuracy": 0.93, + "top3_precision": 0.87, + "top3_recall": 0.49, + "top3_fscore": 0.65, + "n_typo": 83, + "n": 640 + } + }, + "per_number_of_typos": { + "1": { + "accuracy": 0.93, + "precision": 0.82, + "recall": 0.56, + "fscore": 0.68, + "top3_accuracy": 0.94, + "top3_precision": 0.88, + "top3_recall": 0.62, + "top3_fscore": 0.74, + "n_typo": 309, + "n": 2385 + }, + "2": { + "accuracy": 0.92, + "precision": 0.88, + "recall": 0.44, + "fscore": 0.6, + "top3_accuracy": 0.92, + "top3_precision": 0.88, + "top3_recall": 0.44, + "top3_fscore": 0.6, + "n_typo": 16, + "n": 123 + }, + "3+": { + "accuracy": 0.87, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0.87, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 3, + "n": 23 + } + }, + "performances": { + "mean_memory": "600 KB", + "min_memory": "344 KB", + "max_memory": "628 KB", + "mean_runtime": "30.1 s", + "fastest_runtime": "19.7 s", + "slowest_runtime": "96.8 s" + } + }, + "swipe_resolution": { + "score": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 19 + }, + "per_domain": { + "narrative": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 13 + }, + "dialogue": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 6 + } + }, + "performances": { + "mean_memory": "593 KB", + "min_memory": "385 KB", + "max_memory": "628 KB", + "mean_runtime": "38.5 μs", + "fastest_runtime": "28.6 μs", + "slowest_runtime": "50.1 μs" + } + }, + "overall_score": 0.47350000000000003 +} diff --git a/docs/results/tappa.json b/docs/results/tappa.json new file mode 100644 index 0000000..c123cd5 --- /dev/null +++ b/docs/results/tappa.json @@ -0,0 +1,362 @@ +{ + "next_word_prediction": { + "score": { + "accuracy": 0.13, + "top3_accuracy": 0.23, + "n": 2436 + }, + "per_domain": { + "narrative": { + "accuracy": 0.13, + "top3_accuracy": 0.22, + "n": 1556 + }, + "dialogue": { + "accuracy": 0.13, + "top3_accuracy": 0.24, + "n": 880 + } + }, + "performances": { + "mean_memory": "589 KB", + "min_memory": "344 KB", + "max_memory": "617 KB", + "mean_runtime": "8.37 s", + "fastest_runtime": "741 ms", + "slowest_runtime": "60.4 s" + } + }, + "auto_completion": { + "score": { + "accuracy": 0.0033, + "top3_accuracy": 0.58, + "n": 2426 + }, + "per_domain": { + "narrative": { + "accuracy": 0.0032, + "top3_accuracy": 0.57, + "n": 1557 + }, + "dialogue": { + "accuracy": 0.0035, + "top3_accuracy": 0.58, + "n": 869 + } + }, + "per_completion_rate": { + "<25%": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 73 + }, + "25%~50%": { + "accuracy": 0.0046, + "top3_accuracy": 0.35, + "n": 435 + }, + "50%~75%": { + "accuracy": 0.0044, + "top3_accuracy": 0.63, + "n": 1363 + }, + ">75%": { + "accuracy": 0, + "top3_accuracy": 0.7, + "n": 555 + } + }, + "per_other": { + "without_typo": { + "accuracy": 0.0036, + "top3_accuracy": 0.61, + "n": 2247 + }, + "with_typo": { + "accuracy": 0, + "top3_accuracy": 0.18, + "n": 179 + } + }, + "performances": { + "mean_memory": "589 KB", + "min_memory": "322 KB", + "max_memory": "617 KB", + "mean_runtime": "22.5 s", + "fastest_runtime": "5.1 s", + "slowest_runtime": "279 s" + } + }, + "auto_correction": { + "score": { + "accuracy": 0.85, + "precision": 0.43, + "recall": 0.42, + "fscore": 0.42, + "top3_accuracy": 0.9, + "top3_precision": 0.59, + "top3_recall": 0.71, + "top3_fscore": 0.64, + "n_typo": 328, + "n": 2531 + }, + "per_domain": { + "narrative": { + "accuracy": 0.88, + "precision": 0.52, + "recall": 0.45, + "fscore": 0.49, + "top3_accuracy": 0.92, + "top3_precision": 0.66, + "top3_recall": 0.72, + "top3_fscore": 0.69, + "n_typo": 204, + "n": 1601 + }, + "dialogue": { + "accuracy": 0.81, + "precision": 0.31, + "recall": 0.38, + "fscore": 0.34, + "top3_accuracy": 0.86, + "top3_precision": 0.49, + "top3_recall": 0.7, + "top3_fscore": 0.57, + "n_typo": 124, + "n": 930 + } + }, + "per_typo_type": { + "DELETE_SPELLING_SYMBOL": { + "accuracy": 0.86, + "precision": 0.45, + "recall": 0.45, + "fscore": 0.45, + "top3_accuracy": 0.88, + "top3_precision": 0.54, + "top3_recall": 0.64, + "top3_fscore": 0.58, + "n_typo": 11, + "n": 85 + }, + "DELETE_SPACE": { + "accuracy": 0.86, + "precision": 0.44, + "recall": 0.44, + "fscore": 0.44, + "top3_accuracy": 0.94, + "top3_precision": 0.69, + "top3_recall": 1.0, + "top3_fscore": 0.8, + "n_typo": 9, + "n": 69 + }, + "DELETE_PUNCTUATION": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "DELETE_CHAR": { + "accuracy": 0.84, + "precision": 0.38, + "recall": 0.36, + "fscore": 0.37, + "top3_accuracy": 0.89, + "top3_precision": 0.58, + "top3_recall": 0.68, + "top3_fscore": 0.62, + "n_typo": 22, + "n": 170 + }, + "ADD_SPELLING_SYMBOL": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_SPACE": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_PUNCTUATION": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_CHAR": { + "accuracy": 0.85, + "precision": 0.44, + "recall": 0.46, + "fscore": 0.45, + "top3_accuracy": 0.91, + "top3_precision": 0.61, + "top3_recall": 0.81, + "top3_fscore": 0.69, + "n_typo": 37, + "n": 286 + }, + "SUBSTITUTE_CHAR": { + "accuracy": 0.87, + "precision": 0.5, + "recall": 0.57, + "fscore": 0.53, + "top3_accuracy": 0.92, + "top3_precision": 0.63, + "top3_recall": 0.87, + "top3_fscore": 0.72, + "n_typo": 53, + "n": 409 + }, + "SIMPLIFY_ACCENT": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "SIMPLIFY_CASE": { + "accuracy": 0.81, + "precision": 0.15, + "recall": 0.11, + "fscore": 0.13, + "top3_accuracy": 0.84, + "top3_precision": 0.38, + "top3_recall": 0.32, + "top3_fscore": 0.35, + "n_typo": 19, + "n": 147 + }, + "TRANSPOSE_CHAR": { + "accuracy": 0.87, + "precision": 0.48, + "recall": 0.53, + "fscore": 0.5, + "top3_accuracy": 0.91, + "top3_precision": 0.62, + "top3_recall": 0.81, + "top3_fscore": 0.69, + "n_typo": 75, + "n": 579 + }, + "COMMON_TYPO": { + "accuracy": 0.84, + "precision": 0.36, + "recall": 0.33, + "fscore": 0.35, + "top3_accuracy": 0.88, + "top3_precision": 0.54, + "top3_recall": 0.59, + "top3_fscore": 0.56, + "n_typo": 83, + "n": 640 + } + }, + "per_number_of_typos": { + "1": { + "accuracy": 0.85, + "precision": 0.43, + "recall": 0.43, + "fscore": 0.43, + "top3_accuracy": 0.9, + "top3_precision": 0.59, + "top3_recall": 0.72, + "top3_fscore": 0.64, + "n_typo": 309, + "n": 2385 + }, + "2": { + "accuracy": 0.85, + "precision": 0.4, + "recall": 0.38, + "fscore": 0.39, + "top3_accuracy": 0.89, + "top3_precision": 0.58, + "top3_recall": 0.69, + "top3_fscore": 0.62, + "n_typo": 16, + "n": 123 + }, + "3+": { + "accuracy": 0.78, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0.83, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 3, + "n": 23 + } + }, + "performances": { + "mean_memory": "589 KB", + "min_memory": "336 KB", + "max_memory": "617 KB", + "mean_runtime": "25.9 s", + "fastest_runtime": "10.9 s", + "slowest_runtime": "110 s" + } + }, + "swipe_resolution": { + "score": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 19 + }, + "per_domain": { + "narrative": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 13 + }, + "dialogue": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 6 + } + }, + "performances": { + "mean_memory": "583 KB", + "min_memory": "377 KB", + "max_memory": "617 KB", + "mean_runtime": "65.9 μs", + "fastest_runtime": "21.5 μs", + "slowest_runtime": "90.6 μs" + } + }, + "overall_score": 0.3185 +} diff --git a/kebbie/emulator.py b/kebbie/emulator.py index 97047cd..065016e 100644 --- a/kebbie/emulator.py +++ b/kebbie/emulator.py @@ -613,9 +613,7 @@ def _take_screenshot(self): """ screen_data = self.driver.get_screenshot_as_png() screen = np.asarray(Image.open(io.BytesIO(screen_data))) - return cv2.resize( - screen, (self.screen_size["width"], self.screen_size["height"]), interpolation=cv2.INTER_AREA - ) + return screen.copy() def get_predictions(self, lang: str = "en") -> List[str]: """Retrieve the predictions displayed by the keyboard. @@ -775,11 +773,7 @@ def __init__( # Reset out keyboard to the original layer self.tap(layout["numbers"]["letters"], layout["keyboard_frame"]) - # Fix the keys' offset compared to the keyboard frame - if self.android: - self.layout = self._apply_status_bar_offset(layout) - else: - self.layout = layout + self.layout = layout def get_suggestions(self) -> List[str]: """Method to retrieve the keyboard suggestions from the XML tree. @@ -890,52 +884,6 @@ def _get_label(self, element: WebElement, current_layout: str, is_suggestion: bo else: return content - def _get_status_bar_bounds(self) -> List[int]: - """For layout detection, this method retrieve the bounds of the status - bar from the XML tree. - - Returns: - Bounds of the status bar. - """ - sb = self.driver.find_element(By.ID, "com.android.systemui:id/status_bar") - return self._get_frame(sb) - - def _apply_status_bar_offset(self, layout: Dict) -> Dict: - """Method offsetting the given layout to match the screen. - - On Android, somehow the detected positions for the keys aren't matching - what we see on screen. This is because of the status bar, which shift - everything. So, detect the status bar, and shift back the keys to the - right position. - - Args: - layout (Dict): Layout to fix. - - Returns: - Fixed layout. - """ - sb_bounds = self._get_status_bar_bounds() - dy = sb_bounds[3] - screen_size = layout["keyboard_frame"][1] + layout["keyboard_frame"][3] - - # First of all, offset the keyboard frame - frame_dy1 = int(dy * (layout["keyboard_frame"][1] / screen_size)) - frame_dy2 = int(dy * ((layout["keyboard_frame"][1] + layout["keyboard_frame"][3]) / screen_size)) - layout["keyboard_frame"][1] -= frame_dy1 - layout["keyboard_frame"][3] -= frame_dy2 - frame_dy1 - - # Then do the same for each keys of each layouts - for layer in ["lowercase", "uppercase", "numbers"]: - for k in layout[layer]: - dy1 = int(dy * ((layout["keyboard_frame"][1] + layout[layer][k][1]) / screen_size)) - dy2 = int( - dy * ((layout["keyboard_frame"][1] + layout[layer][k][1] + layout[layer][k][3]) / screen_size) - ) - layout[layer][k][1] -= dy1 - frame_dy1 - layout[layer][k][3] -= dy2 - dy1 - - return layout - class GboardLayoutDetector(LayoutDetector): """Layout detector for the Gboard keyboard. See `LayoutDetector` for more @@ -1177,8 +1125,8 @@ def get_suggestions(self) -> List[str]: suggestions = [] # Get the raw content as text, weed out useless elements - section = self.driver.page_source.split(f"{KEYBOARD_PACKAGE[TAPPA]}:id/toolbar")[1].split( - "" + section = self.driver.page_source.split(f"{KEYBOARD_PACKAGE[TAPPA]}:id/suggestions_strip")[1].split( + "" )[0] for line in section.split("\n"):