diff --git a/configs/datasets/collections/chat_medium.py b/configs/datasets/collections/chat_medium.py index 577e3b0b4..13855c854 100644 --- a/configs/datasets/collections/chat_medium.py +++ b/configs/datasets/collections/chat_medium.py @@ -52,7 +52,7 @@ from ..nq.nq_gen_c788f6 import nq_datasets from ..triviaqa.triviaqa_gen_2121ce import triviaqa_datasets from ..flores.flores_gen_806ede import flores_datasets - from ..crowspairs.crowspairs_gen_21f7cb import crowspairs_datasets + from ..crowspairs.crowspairs_gen_381af0 import crowspairs_datasets from ..civilcomments.civilcomments_clp_a3c5fd import civilcomments_datasets from ..jigsawmultilingual.jigsawmultilingual_clp_fe50d8 import jigsawmultilingual_datasets from ..realtoxicprompts.realtoxicprompts_gen_7605e4 import realtoxicprompts_datasets diff --git a/opencompass/tasks/openicl_eval.py b/opencompass/tasks/openicl_eval.py index 1fec1288c..978a4a6b2 100644 --- a/opencompass/tasks/openicl_eval.py +++ b/opencompass/tasks/openicl_eval.py @@ -58,13 +58,11 @@ def _score(self): test_set = build_dataset_from_cfg(self.dataset_cfg).test # Postprocess dataset if necessary if 'dataset_postprocessor' in self.eval_cfg: - TEXT_POSTPROCESSORS.get( + proc = TEXT_POSTPROCESSORS.get( self.eval_cfg['dataset_postprocessor']['type']) def postprocess(sample): s = sample[self.output_column] - proc = TEXT_POSTPROCESSORS.get( - self.eval_cfg['dataset_postprocessor']['type']) sample[self.output_column] = proc(s) return sample @@ -130,13 +128,13 @@ def postprocess(sample): # Postprocess predictions if necessary if 'pred_postprocessor' in self.eval_cfg: - proc = TEXT_POSTPROCESSORS.get( - self.eval_cfg['pred_postprocessor']['type']) + kwargs = self.eval_cfg['pred_postprocessor'] + proc = TEXT_POSTPROCESSORS.get(kwargs.pop('type')) if sc_size is not None: - pred_strs = [[proc(s) for s in preds] + pred_strs = [[proc(s, **kwargs) for s in preds] for preds in pred_strs] else: - pred_strs = [proc(s) for s in pred_strs] + pred_strs = [proc(s, **kwargs) for s in pred_strs] # Get majority voting predictions if use self-consistency if sc_size is not None: diff --git a/opencompass/utils/text_postprocessors.py b/opencompass/utils/text_postprocessors.py index 9e57f83c0..8c5042180 100644 --- a/opencompass/utils/text_postprocessors.py +++ b/opencompass/utils/text_postprocessors.py @@ -48,7 +48,7 @@ def first_capital_postprocess(text: str) -> str: return '' -def first_option_postprocess(text: str, options) -> str: +def first_option_postprocess(text: str, options: str) -> str: """Find first valid option for text.""" patterns = [