* allow to change the config of opencc converter

modelscope · Apr 18, 2024 · ff78d28 · ff78d28
1 parent b244565
commit ff78d28
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/data_juicer/ops/mapper/chinese_convert_mapper.py b/data_juicer/ops/mapper/chinese_convert_mapper.py
@@ -11,9 +11,15 @@
 
 
 def prepare_converter(mode):
+    mode_path = mode + '.json'
     global OPENCC_CONVERTER
     if OPENCC_CONVERTER is None:
-        OPENCC_CONVERTER = opencc.OpenCC(mode + '.json')
+        # empty converter
+        OPENCC_CONVERTER = opencc.OpenCC(mode_path)
+    if not OPENCC_CONVERTER.config.endswith(mode_path):
+        # the config is actually a config path
+        # update and get a new converter with specified mode
+        OPENCC_CONVERTER = opencc.OpenCC(mode_path)
 
 
 @OPERATORS.register_module(OP_NAME)