edit cleaners

kdrkdrkdr · Jun 27, 2023 · 84ee959 · 84ee959
1 parent d4ed8b3
commit 84ee959
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 35 deletions.
diff --git a/inference.ipynb b/inference.ipynb
diff --git a/text/cleaners.py b/text/cleaners.py
@@ -23,5 +23,6 @@ def korean_cleaners(text):
 def jk_cleaners(text):
     text = re.sub(r'\[JA\](.*?)\[JA\]', lambda x: japanese_cleaners(x.group(1))+' ', text)
     text = re.sub(r'\[KO\](.*?)\[KO\]', lambda x: korean_cleaners(x.group(1))+' ', text)
+    text = re.sub(r'\[PREPROCESSED\](.*?)\[PREPROCESSED\]', lambda x: x.group(1)+' ', text)
     text = ''.join(_cleaner_cleans.findall(text))
     return text
diff --git a/text/j2k.py b/text/j2k.py
@@ -1,7 +1,6 @@
-from .cleaners import japanese_to_romaji_with_accent
+from .cleaners import japanese_to_romaji_with_accent, korean_cleaners
 from .korean import join_jamos
 
-
 repl_lst = {
     '.': '. ',
     '↓': '',
@@ -14,10 +13,15 @@
     'Q': ' |Q ',
     'N': ' |N ',
     'U': 'u ',
+    'I': 'i ',
+    'A': 'a ',
+    'E': 'e ',
+    'O': 'o ',
 }
 
 repl_lst2 = {
     'ʧu': '츠',
+    'tsu': '츠',
     'zu': '즈',
     'su': '스',
 
@@ -35,6 +39,7 @@
     's': 'ㅅ',
     'j': 'ㅈ',
     'ʧ': 'ㅊ',
+    'ts': 'ㅊ',
     'k': 'ㅋ',
     't': 'ㅌ',
     'p': 'ㅍ',
@@ -80,20 +85,25 @@
     '|Nㅇ': 'ㅇㅇ',
     '|Nㅎ': 'ㅇㅎ',
 
-    '|Q': 'ㅅ'
+    '|Q': 'ㅅ',
+    '|N': 'ㄴ',
 }
 
 
 def japanese2korean(text):
-    text = japanese_to_romaji_with_accent(text).strip()
+    text = japanese_to_romaji_with_accent(text).strip().replace('^', '').replace(' ', '^ ')
 
     for k, v in repl_lst.items():
         text = text.replace(k, v)
     for k, v in repl_lst2.items():
         text = text.replace(k, v)
 
     text = ' '.join([i.replace('*', 'ㅇ') if i.startswith('*') else i.replace('*', '') for i in text.strip().split(' ')])
+
     for k, v in repl_lst3.items():
         text = text.replace(k, v)
 
-    return join_jamos(text.replace('  ', '_').replace(' ', ''))
+    text = join_jamos(text.replace('  ', ' ')).replace(' ', '').replace('^', ' ')
+    return f"[PREPROCESSED]{korean_cleaners(text)}[PREPROCESSED]"
+
+
diff --git a/text/k2j.py b/text/k2j.py
@@ -5,6 +5,7 @@
     number_to_hangul,
     g2pk,
 )
+from .cleaners import japanese_cleaners
 import re
 import jaconv
 
@@ -47,7 +48,8 @@ def get_word_list(text):
 
 def korean2katakana(text):
     word_lst = get_word_list(text)
-    text = '/' + text.replace('/', ' ').replace('|', ' ').replace('^', ' ').replace('  ', ' ').replace(' ', '^')
+    text = '/' + text.replace('/', ' ').replace('|', ' ').replace('^', ' ').replace('  ', ' ')
+    print(text)
     new_lst = []
 
     for i, s in enumerate(word_lst):
@@ -70,17 +72,12 @@ def korean2katakana(text):
         new_lst.extend(dh)
 
     kr = ''.join(new_lst)
-
     for k, v in repl_lst.items():
         kr = kr.replace(k, v)
-
     kr2ro = japanese_to_romaji_with_accent(kr).replace('si', 'shi').replace('c', 'ts') \
                                               .replace('ti', 'ティ').replace('tu', 'トゥ') \
                                               .replace('di', 'ディ').replace('du', 'ドゥ')
-    result = jaconv.alphabet2kata(kr2ro)
-    result = result.replace('/', '').replace('|', 'ー').replace('^', '_')
-    print(result)
-    return result
-
 
-# print(korean2katakana("안녕하세요.")) -> アンニョンーハセヨ
+    result = jaconv.alphabet2kata(kr2ro).replace('|', 'ー').replace('/', '').replace('^', '')
+    result = result if result[-1] == '.' else result + '.'
+    return f'[PREPROCESSED]{japanese_cleaners(result).replace(" ", "")}[PREPROCESSED]'