Fix mc bugs for loss

EvolvingLMMs-Lab · Nov 2, 2024 · 2702647 · 2702647
1 parent 5720bd1
commit 2702647
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py
@@ -1444,8 +1444,12 @@ def process_results(self, doc, results, full_docs=None):
                 # and this stores our "regular" conditional loglikelihoods
                 lls = lls[::2]
 
-            pred = np.argmax(lls)
-            pred_norm = np.argmax(lls / completion_len)
+            # Warning :
+            # Here may be different from original lm-eval
+            # since we return the actual loss in many model loglikelihood
+            # we just use the argmin here
+            pred = np.argmin(lls)
+            pred_norm = np.argmin(lls / completion_len)
 
             if self.multiple_input:
                 gold = self.doc_to_text(doc)