Skip to content

Commit

Permalink
🚀 Add Yandex results
Browse files Browse the repository at this point in the history
  • Loading branch information
@astariul
astariul committed May 26, 2024
1 parent 2e850c8 commit b1ffa21
Show file tree
Hide file tree
Showing 2 changed files with 363 additions and 0 deletions.
1 change: 1 addition & 0 deletions docs/leaderboard.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
>>>Gboard|results/gboard.json
>>>Swiftkey|results/swiftkey.json
>>>Tappa keyboard|results/tappa.json
>>>Yandex keyboard|results/yandex.json
!!! info
The metrics used in this leaderboard are :
Expand Down
362 changes: 362 additions & 0 deletions docs/results/yandex.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,362 @@
{
"next_word_prediction": {
"score": {
"accuracy": 0.023,
"top3_accuracy": 0.29,
"n": 2436
},
"per_domain": {
"narrative": {
"accuracy": 0.024,
"top3_accuracy": 0.29,
"n": 1556
},
"dialogue": {
"accuracy": 0.019,
"top3_accuracy": 0.3,
"n": 880
}
},
"performances": {
"mean_memory": "585 KB",
"min_memory": "332 KB",
"max_memory": "614 KB",
"mean_runtime": "7.93 s",
"fastest_runtime": "1.95 s",
"slowest_runtime": "34.3 s"
}
},
"auto_completion": {
"score": {
"accuracy": 0.072,
"top3_accuracy": 0.81,
"n": 2426
},
"per_domain": {
"narrative": {
"accuracy": 0.076,
"top3_accuracy": 0.81,
"n": 1557
},
"dialogue": {
"accuracy": 0.066,
"top3_accuracy": 0.81,
"n": 869
}
},
"per_completion_rate": {
"<25%": {
"accuracy": 0.12,
"top3_accuracy": 0.42,
"n": 73
},
"25%~50%": {
"accuracy": 0.12,
"top3_accuracy": 0.69,
"n": 435
},
"50%~75%": {
"accuracy": 0.073,
"top3_accuracy": 0.85,
"n": 1363
},
">75%": {
"accuracy": 0.025,
"top3_accuracy": 0.86,
"n": 555
}
},
"per_other": {
"without_typo": {
"accuracy": 0.075,
"top3_accuracy": 0.85,
"n": 2247
},
"with_typo": {
"accuracy": 0.034,
"top3_accuracy": 0.36,
"n": 179
}
},
"performances": {
"mean_memory": "585 KB",
"min_memory": "316 KB",
"max_memory": "614 KB",
"mean_runtime": "23.8 s",
"fastest_runtime": "1.88 s",
"slowest_runtime": "72.6 s"
}
},
"auto_correction": {
"score": {
"accuracy": 0.88,
"precision": 0.54,
"recall": 0.61,
"fscore": 0.57,
"top3_accuracy": 0.9,
"top3_precision": 0.59,
"top3_recall": 0.66,
"top3_fscore": 0.62,
"n_typo": 328,
"n": 2531
},
"per_domain": {
"narrative": {
"accuracy": 0.93,
"precision": 0.77,
"recall": 0.62,
"fscore": 0.7,
"top3_accuracy": 0.94,
"top3_precision": 0.82,
"top3_recall": 0.65,
"top3_fscore": 0.73,
"n_typo": 204,
"n": 1601
},
"dialogue": {
"accuracy": 0.81,
"precision": 0.36,
"recall": 0.59,
"fscore": 0.44,
"top3_accuracy": 0.82,
"top3_precision": 0.4,
"top3_recall": 0.66,
"top3_fscore": 0.49,
"n_typo": 124,
"n": 930
}
},
"per_typo_type": {
"DELETE_SPELLING_SYMBOL": {
"accuracy": 0.84,
"precision": 0.33,
"recall": 0.27,
"fscore": 0.3,
"top3_accuracy": 0.87,
"top3_precision": 0.5,
"top3_recall": 0.45,
"top3_fscore": 0.48,
"n_typo": 11,
"n": 85
},
"DELETE_SPACE": {
"accuracy": 0.84,
"precision": 0.33,
"recall": 0.22,
"fscore": 0.27,
"top3_accuracy": 0.84,
"top3_precision": 0.33,
"top3_recall": 0.22,
"top3_fscore": 0.27,
"n_typo": 9,
"n": 69
},
"DELETE_PUNCTUATION": {
"accuracy": 0,
"precision": 0,
"recall": 0,
"fscore": 0,
"top3_accuracy": 0,
"top3_precision": 0,
"top3_recall": 0,
"top3_fscore": 0,
"n_typo": 0,
"n": 0
},
"DELETE_CHAR": {
"accuracy": 0.89,
"precision": 0.58,
"recall": 0.68,
"fscore": 0.62,
"top3_accuracy": 0.9,
"top3_precision": 0.6,
"top3_recall": 0.68,
"top3_fscore": 0.63,
"n_typo": 22,
"n": 170
},
"ADD_SPELLING_SYMBOL": {
"accuracy": 0,
"precision": 0,
"recall": 0,
"fscore": 0,
"top3_accuracy": 0,
"top3_precision": 0,
"top3_recall": 0,
"top3_fscore": 0,
"n_typo": 0,
"n": 0
},
"ADD_SPACE": {
"accuracy": 0,
"precision": 0,
"recall": 0,
"fscore": 0,
"top3_accuracy": 0,
"top3_precision": 0,
"top3_recall": 0,
"top3_fscore": 0,
"n_typo": 0,
"n": 0
},
"ADD_PUNCTUATION": {
"accuracy": 0,
"precision": 0,
"recall": 0,
"fscore": 0,
"top3_accuracy": 0,
"top3_precision": 0,
"top3_recall": 0,
"top3_fscore": 0,
"n_typo": 0,
"n": 0
},
"ADD_CHAR": {
"accuracy": 0.92,
"precision": 0.64,
"recall": 0.92,
"fscore": 0.74,
"top3_accuracy": 0.93,
"top3_precision": 0.66,
"top3_recall": 0.95,
"top3_fscore": 0.76,
"n_typo": 37,
"n": 286
},
"SUBSTITUTE_CHAR": {
"accuracy": 0.89,
"precision": 0.56,
"recall": 0.64,
"fscore": 0.59,
"top3_accuracy": 0.91,
"top3_precision": 0.62,
"top3_recall": 0.74,
"top3_fscore": 0.67,
"n_typo": 53,
"n": 409
},
"SIMPLIFY_ACCENT": {
"accuracy": 0,
"precision": 0,
"recall": 0,
"fscore": 0,
"top3_accuracy": 0,
"top3_precision": 0,
"top3_recall": 0,
"top3_fscore": 0,
"n_typo": 0,
"n": 0
},
"SIMPLIFY_CASE": {
"accuracy": 0.81,
"precision": 0.091,
"recall": 0.053,
"fscore": 0.069,
"top3_accuracy": 0.82,
"top3_precision": 0.1,
"top3_recall": 0.053,
"top3_fscore": 0.071,
"n_typo": 19,
"n": 147
},
"TRANSPOSE_CHAR": {
"accuracy": 0.89,
"precision": 0.58,
"recall": 0.69,
"fscore": 0.62,
"top3_accuracy": 0.9,
"top3_precision": 0.61,
"top3_recall": 0.72,
"top3_fscore": 0.65,
"n_typo": 75,
"n": 579
},
"COMMON_TYPO": {
"accuracy": 0.88,
"precision": 0.53,
"recall": 0.58,
"fscore": 0.55,
"top3_accuracy": 0.89,
"top3_precision": 0.58,
"top3_recall": 0.64,
"top3_fscore": 0.61,
"n_typo": 83,
"n": 640
}
},
"per_number_of_typos": {
"1": {
"accuracy": 0.88,
"precision": 0.55,
"recall": 0.61,
"fscore": 0.57,
"top3_accuracy": 0.9,
"top3_precision": 0.59,
"top3_recall": 0.66,
"top3_fscore": 0.62,
"n_typo": 309,
"n": 2385
},
"2": {
"accuracy": 0.89,
"precision": 0.58,
"recall": 0.69,
"fscore": 0.62,
"top3_accuracy": 0.9,
"top3_precision": 0.61,
"top3_recall": 0.69,
"top3_fscore": 0.64,
"n_typo": 16,
"n": 123
},
"3+": {
"accuracy": 0.83,
"precision": 0,
"recall": 0,
"fscore": 0,
"top3_accuracy": 0.83,
"top3_precision": 0,
"top3_recall": 0,
"top3_fscore": 0,
"n_typo": 3,
"n": 23
}
},
"performances": {
"mean_memory": "585 KB",
"min_memory": "330 KB",
"max_memory": "614 KB",
"mean_runtime": "31 s",
"fastest_runtime": "2.21 s",
"slowest_runtime": "93.6 s"
}
},
"swipe_resolution": {
"score": {
"accuracy": 0,
"top3_accuracy": 0,
"n": 19
},
"per_domain": {
"narrative": {
"accuracy": 0,
"top3_accuracy": 0,
"n": 13
},
"dialogue": {
"accuracy": 0,
"top3_accuracy": 0,
"n": 6
}
},
"performances": {
"mean_memory": "579 KB",
"min_memory": "370 KB",
"max_memory": "614 KB",
"mean_runtime": "38.6 μs",
"fastest_runtime": "9.54 μs",
"slowest_runtime": "100 μs"
}
},
"overall_score": 0.4335
}

0 comments on commit b1ffa21

Please sign in to comment.