* fix for "undefined opencc" bug for chinese_convert_mapper

modelscope · Apr 18, 2024 · b244565 · b244565
1 parent c1a8aa8
commit b244565
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/data_juicer/ops/mapper/chinese_convert_mapper.py b/data_juicer/ops/mapper/chinese_convert_mapper.py
@@ -7,10 +7,13 @@
 with AvailabilityChecking(['opencc'], OP_NAME):
     import opencc  # noqa: F401
 
+OPENCC_CONVERTER = None
+
 
 def prepare_converter(mode):
     global OPENCC_CONVERTER
-    OPENCC_CONVERTER = opencc.OpenCC(mode + '.json')
+    if OPENCC_CONVERTER is None:
+        OPENCC_CONVERTER = opencc.OpenCC(mode + '.json')
 
 
 @OPERATORS.register_module(OP_NAME)
@@ -70,9 +73,11 @@ def __init__(self, mode: str = 's2t', *args, **kwargs):
         ]
         assert mode in mode_list, 'Please make sure mode is one of {}'.format(
             mode_list)
-        prepare_converter(mode)
+        self.mode = mode
+        prepare_converter(self.mode)
 
     def process(self, sample):
+        prepare_converter(self.mode)
 
         sample[self.text_key] = OPENCC_CONVERTER.convert(sample[self.text_key])
         return sample