feat: support raw pattern

modelscope · Dec 19, 2023 · b3cc336 · b3cc336
1 parent 421c528
commit b3cc336
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 12 deletions.
diff --git a/data_juicer/ops/mapper/clean_email_mapper.py b/data_juicer/ops/mapper/clean_email_mapper.py
@@ -21,10 +21,10 @@ def __init__(self, pattern: str = None, repl: str = '', *args, **kwargs):
             self.pattern = r'[A-Za-z0-9.\-+_]+@[a-z0-9.\-+_]+\.[a-z]+'
         else:
             self.pattern = pattern
-            if pattern is not None and len(pattern) > 2:
-                if (pattern.startswith("r'") and pattern.endswith("'")
-                        or pattern.startswith('r"') and pattern.endswith('"')):
-                    self.pattern = pattern[2:-1]
+            if (len(pattern) > 2
+                    and pattern.startswith("r'") and pattern.endswith("'")
+                    or pattern.startswith('r"') and pattern.endswith('"')):
+                self.pattern = pattern[2:-1]
 
         self.repl = repl
 

diff --git a/data_juicer/ops/mapper/clean_ip_mapper.py b/data_juicer/ops/mapper/clean_ip_mapper.py
@@ -26,10 +26,10 @@ def __init__(self, pattern: str = None, repl: str = '', *args, **kwargs):
             self.pattern += r'([\da-fA-F]{1,4}:){7}[\da-fA-F]{1,4}'  # ipv6
         else:
             self.pattern = pattern
-            if pattern is not None and len(pattern) > 2:
-                if (pattern.startswith("r'") and pattern.endswith("'")
-                        or pattern.startswith('r"') and pattern.endswith('"')):
-                    self.pattern = pattern[2:-1]
+            if (len(pattern) > 2
+                    and pattern.startswith("r'") and pattern.endswith("'")
+                    or pattern.startswith('r"') and pattern.endswith('"')):
+                self.pattern = pattern[2:-1]
         self.repl = repl
 
     def process(self, sample):

diff --git a/data_juicer/ops/mapper/clean_links_mapper.py b/data_juicer/ops/mapper/clean_links_mapper.py
@@ -32,10 +32,10 @@ def __init__(self, pattern: str = None, repl: str = '', *args, **kwargs):
             self.pattern += r')'
         else:
             self.pattern = pattern
-            if pattern is not None and len(pattern) > 2:
-                if (pattern.startswith("r'") and pattern.endswith("'")
-                        or pattern.startswith('r"') and pattern.endswith('"')):
-                    self.pattern = pattern[2:-1]
+            if (len(pattern) > 2
+                    and pattern.startswith("r'") and pattern.endswith("'")
+                    or pattern.startswith('r"') and pattern.endswith('"')):
+                self.pattern = pattern[2:-1]
         self.repl = repl
 
     def process(self, sample):