From b1ffa213d58cc10290002eb761c0c034bd527d44 Mon Sep 17 00:00:00 2001 From: Astariul Date: Mon, 27 May 2024 08:57:26 +0900 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20Add=20Yandex=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/leaderboard.md | 1 + docs/results/yandex.json | 362 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 docs/results/yandex.json diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 7f623dc..fbc3fe0 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -12,6 +12,7 @@ >>>Gboard|results/gboard.json >>>Swiftkey|results/swiftkey.json >>>Tappa keyboard|results/tappa.json +>>>Yandex keyboard|results/yandex.json !!! info The metrics used in this leaderboard are : diff --git a/docs/results/yandex.json b/docs/results/yandex.json new file mode 100644 index 0000000..11b101f --- /dev/null +++ b/docs/results/yandex.json @@ -0,0 +1,362 @@ +{ + "next_word_prediction": { + "score": { + "accuracy": 0.023, + "top3_accuracy": 0.29, + "n": 2436 + }, + "per_domain": { + "narrative": { + "accuracy": 0.024, + "top3_accuracy": 0.29, + "n": 1556 + }, + "dialogue": { + "accuracy": 0.019, + "top3_accuracy": 0.3, + "n": 880 + } + }, + "performances": { + "mean_memory": "585 KB", + "min_memory": "332 KB", + "max_memory": "614 KB", + "mean_runtime": "7.93 s", + "fastest_runtime": "1.95 s", + "slowest_runtime": "34.3 s" + } + }, + "auto_completion": { + "score": { + "accuracy": 0.072, + "top3_accuracy": 0.81, + "n": 2426 + }, + "per_domain": { + "narrative": { + "accuracy": 0.076, + "top3_accuracy": 0.81, + "n": 1557 + }, + "dialogue": { + "accuracy": 0.066, + "top3_accuracy": 0.81, + "n": 869 + } + }, + "per_completion_rate": { + "<25%": { + "accuracy": 0.12, + "top3_accuracy": 0.42, + "n": 73 + }, + "25%~50%": { + "accuracy": 0.12, + "top3_accuracy": 0.69, + "n": 435 + }, + "50%~75%": { + "accuracy": 0.073, + "top3_accuracy": 0.85, + "n": 1363 + }, + ">75%": { + "accuracy": 0.025, + "top3_accuracy": 0.86, + "n": 555 + } + }, + "per_other": { + "without_typo": { + "accuracy": 0.075, + "top3_accuracy": 0.85, + "n": 2247 + }, + "with_typo": { + "accuracy": 0.034, + "top3_accuracy": 0.36, + "n": 179 + } + }, + "performances": { + "mean_memory": "585 KB", + "min_memory": "316 KB", + "max_memory": "614 KB", + "mean_runtime": "23.8 s", + "fastest_runtime": "1.88 s", + "slowest_runtime": "72.6 s" + } + }, + "auto_correction": { + "score": { + "accuracy": 0.88, + "precision": 0.54, + "recall": 0.61, + "fscore": 0.57, + "top3_accuracy": 0.9, + "top3_precision": 0.59, + "top3_recall": 0.66, + "top3_fscore": 0.62, + "n_typo": 328, + "n": 2531 + }, + "per_domain": { + "narrative": { + "accuracy": 0.93, + "precision": 0.77, + "recall": 0.62, + "fscore": 0.7, + "top3_accuracy": 0.94, + "top3_precision": 0.82, + "top3_recall": 0.65, + "top3_fscore": 0.73, + "n_typo": 204, + "n": 1601 + }, + "dialogue": { + "accuracy": 0.81, + "precision": 0.36, + "recall": 0.59, + "fscore": 0.44, + "top3_accuracy": 0.82, + "top3_precision": 0.4, + "top3_recall": 0.66, + "top3_fscore": 0.49, + "n_typo": 124, + "n": 930 + } + }, + "per_typo_type": { + "DELETE_SPELLING_SYMBOL": { + "accuracy": 0.84, + "precision": 0.33, + "recall": 0.27, + "fscore": 0.3, + "top3_accuracy": 0.87, + "top3_precision": 0.5, + "top3_recall": 0.45, + "top3_fscore": 0.48, + "n_typo": 11, + "n": 85 + }, + "DELETE_SPACE": { + "accuracy": 0.84, + "precision": 0.33, + "recall": 0.22, + "fscore": 0.27, + "top3_accuracy": 0.84, + "top3_precision": 0.33, + "top3_recall": 0.22, + "top3_fscore": 0.27, + "n_typo": 9, + "n": 69 + }, + "DELETE_PUNCTUATION": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "DELETE_CHAR": { + "accuracy": 0.89, + "precision": 0.58, + "recall": 0.68, + "fscore": 0.62, + "top3_accuracy": 0.9, + "top3_precision": 0.6, + "top3_recall": 0.68, + "top3_fscore": 0.63, + "n_typo": 22, + "n": 170 + }, + "ADD_SPELLING_SYMBOL": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_SPACE": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_PUNCTUATION": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "ADD_CHAR": { + "accuracy": 0.92, + "precision": 0.64, + "recall": 0.92, + "fscore": 0.74, + "top3_accuracy": 0.93, + "top3_precision": 0.66, + "top3_recall": 0.95, + "top3_fscore": 0.76, + "n_typo": 37, + "n": 286 + }, + "SUBSTITUTE_CHAR": { + "accuracy": 0.89, + "precision": 0.56, + "recall": 0.64, + "fscore": 0.59, + "top3_accuracy": 0.91, + "top3_precision": 0.62, + "top3_recall": 0.74, + "top3_fscore": 0.67, + "n_typo": 53, + "n": 409 + }, + "SIMPLIFY_ACCENT": { + "accuracy": 0, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 0, + "n": 0 + }, + "SIMPLIFY_CASE": { + "accuracy": 0.81, + "precision": 0.091, + "recall": 0.053, + "fscore": 0.069, + "top3_accuracy": 0.82, + "top3_precision": 0.1, + "top3_recall": 0.053, + "top3_fscore": 0.071, + "n_typo": 19, + "n": 147 + }, + "TRANSPOSE_CHAR": { + "accuracy": 0.89, + "precision": 0.58, + "recall": 0.69, + "fscore": 0.62, + "top3_accuracy": 0.9, + "top3_precision": 0.61, + "top3_recall": 0.72, + "top3_fscore": 0.65, + "n_typo": 75, + "n": 579 + }, + "COMMON_TYPO": { + "accuracy": 0.88, + "precision": 0.53, + "recall": 0.58, + "fscore": 0.55, + "top3_accuracy": 0.89, + "top3_precision": 0.58, + "top3_recall": 0.64, + "top3_fscore": 0.61, + "n_typo": 83, + "n": 640 + } + }, + "per_number_of_typos": { + "1": { + "accuracy": 0.88, + "precision": 0.55, + "recall": 0.61, + "fscore": 0.57, + "top3_accuracy": 0.9, + "top3_precision": 0.59, + "top3_recall": 0.66, + "top3_fscore": 0.62, + "n_typo": 309, + "n": 2385 + }, + "2": { + "accuracy": 0.89, + "precision": 0.58, + "recall": 0.69, + "fscore": 0.62, + "top3_accuracy": 0.9, + "top3_precision": 0.61, + "top3_recall": 0.69, + "top3_fscore": 0.64, + "n_typo": 16, + "n": 123 + }, + "3+": { + "accuracy": 0.83, + "precision": 0, + "recall": 0, + "fscore": 0, + "top3_accuracy": 0.83, + "top3_precision": 0, + "top3_recall": 0, + "top3_fscore": 0, + "n_typo": 3, + "n": 23 + } + }, + "performances": { + "mean_memory": "585 KB", + "min_memory": "330 KB", + "max_memory": "614 KB", + "mean_runtime": "31 s", + "fastest_runtime": "2.21 s", + "slowest_runtime": "93.6 s" + } + }, + "swipe_resolution": { + "score": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 19 + }, + "per_domain": { + "narrative": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 13 + }, + "dialogue": { + "accuracy": 0, + "top3_accuracy": 0, + "n": 6 + } + }, + "performances": { + "mean_memory": "579 KB", + "min_memory": "370 KB", + "max_memory": "614 KB", + "mean_runtime": "38.6 μs", + "fastest_runtime": "9.54 μs", + "slowest_runtime": "100 μs" + } + }, + "overall_score": 0.4335 +}