Tentative result of Llama 3 Swallow 70 Instruct.

swallow-llm · Jun 30, 2024 · eb85b98 · eb85b98
1 parent 4899ebe
commit eb85b98
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 38 deletions.
diff --git a/_data/models.yml b/_data/models.yml
@@ -23,7 +23,7 @@
   size: 13
   type: base
 - basename: ''
-  missing: HumanEval
+  missing: ''
   model: Fugaku-LLM/Fugaku-LLM-13B
   name: Fugaku-LLM 13B
   size: 13
@@ -125,7 +125,7 @@
   size: 8
   type: instruct
 - basename: Llama 3 Swallow 70B
-  missing: Japanese tasks, Japanese MT-bench tasks, English tasks
+  missing: ''
   model: tokyotech-llm/Llama-3-Swallow-8B-Instruct-v0.1
   name: Llama 3 Swallow 70B Instruct
   size: 70

diff --git a/assets/data.js b/assets/data.js
@@ -178,9 +178,9 @@ const dataSet = [
     "MGSM": 0.044,
     "WMT20 (en-ja)": 0.2242,
     "WMT20 (ja-en)": 0.1105,
-    "Ja Avg": 0.3234,
+    "Ja Avg": 0.3025,
     "JMMLU": 0.2782,
-    "JHumanEval": NaN,
+    "JHumanEval": 0.1146,
     "OpenBookQA": 0.262,
     "TriviaQA": 0.3456,
     "HellaSwag": 0.4449,
@@ -189,10 +189,10 @@ const dataSet = [
     "MMLU": 0.2646,
     "GSM8K": 0.0652,
     "BBH": 0.3236,
-    "HumanEval": NaN,
-    "En Avg": 0.3482,
+    "HumanEval": 0.1537,
+    "En Avg": 0.3266,
     "SortKey": "Fugaku-LLM 013",
-    "Missing": "HumanEval"
+    "Missing": ""
   },
   {
     "Type": "inst",
@@ -824,38 +824,38 @@ const dataSet = [
     "Name": "Llama 3 Swallow 70B Instruct",
     "Base name": "Llama 3 Swallow 70B",
     "Size (B)": 70,
-    "Coding": NaN,
-    "Extraction": NaN,
-    "Humanities": NaN,
-    "Math": NaN,
-    "Reasoning": NaN,
-    "Roleplay": NaN,
-    "Stem": NaN,
-    "Writing": NaN,
-    "Ja MT-Bench": NaN,
-    "JCom": NaN,
-    "JEMHopQA": NaN,
-    "NIILC": NaN,
-    "JSQuAD": NaN,
-    "XL-Sum": NaN,
-    "MGSM": NaN,
-    "WMT20 (en-ja)": NaN,
-    "WMT20 (ja-en)": NaN,
-    "Ja Avg": NaN,
-    "JMMLU": NaN,
-    "JHumanEval": NaN,
-    "OpenBookQA": NaN,
-    "TriviaQA": NaN,
-    "HellaSwag": NaN,
-    "SQuAD2": NaN,
-    "XWINO": NaN,
-    "MMLU": NaN,
-    "GSM8K": NaN,
-    "BBH": NaN,
-    "HumanEval": NaN,
-    "En Avg": NaN,
+    "Coding": 0.5269,
+    "Extraction": 0.725,
+    "Humanities": 0.569,
+    "Math": 0.4669,
+    "Reasoning": 0.6121,
+    "Roleplay": 0.6238,
+    "Stem": 0.5533,
+    "Writing": 0.5698,
+    "Ja MT-Bench": 0.5809,
+    "JCom": 0.9607,
+    "JEMHopQA": 0.6188,
+    "NIILC": 0.6026,
+    "JSQuAD": 0.9236,
+    "XL-Sum": 0.1389,
+    "MGSM": 0.656,
+    "WMT20 (en-ja)": 0.2724,
+    "WMT20 (ja-en)": 0.2532,
+    "Ja Avg": 0.5683,
+    "JMMLU": 0.6572,
+    "JHumanEval": 0.6,
+    "OpenBookQA": 0.452,
+    "TriviaQA": 0.8174,
+    "HellaSwag": 0.6758,
+    "SQuAD2": 0.405,
+    "XWINO": 0.923,
+    "MMLU": 0.7883,
+    "GSM8K": 0.8688,
+    "BBH": 0.8152,
+    "HumanEval": 0.689,
+    "En Avg": 0.715,
     "SortKey": "Llama 3 Swallow  Instruct070",
-    "Missing": "Japanese tasks, Japanese MT-bench tasks, English tasks"
+    "Missing": ""
   },
   {
     "Type": "base",