Add radical normalization preprocessor to Japanese, Chinese, and Cant…

…onese * Add radical normalization to japanese * Add radical normalization to chinese * Add CJK Strokes Range * Add to yue * Move normalizeRadicalCharacters to CJK-util * Fix tests <rikaitan.link>OTJiYWM0ZTczMDgyZmE1YjUyNGM1YWQ0OWU3YjU0MTUyOWY4NDIyZAo=</rikaitan.link>
Ajatt-Tools · Nov 4, 2024 · 23f1de8 · 23f1de8
1 parent e02289c
commit 23f1de8
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 2 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -560,6 +560,7 @@
                 "ext/js/dom/css-style-applier.js",
                 "ext/js/language/CJK-util.js",
                 "ext/js/language/ja/japanese.js",
+                "ext/js/language/text-processors.js",
                 "ext/js/language/text-utilities.js",
                 "ext/js/templates/anki-template-renderer-content-manager.js",
                 "ext/js/templates/anki-template-renderer.js",

diff --git a/ext/js/language/CJK-util.js b/ext/js/language/CJK-util.js
@@ -15,6 +15,8 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import {basicTextProcessorOptions} from './text-processors.js';
+
 /** @type {import('CJK-util').CodepointRange} */
 const CJK_UNIFIED_IDEOGRAPHS_RANGE = [0x4e00, 0x9fff];
 /** @type {import('CJK-util').CodepointRange} */
@@ -94,3 +96,40 @@ export function isCodePointInRanges(codePoint, ranges) {
     }
     return false;
 }
+
+/** @type {import('CJK-util').CodepointRange} */
+export const KANGXI_RADICALS_RANGE = [0x2f00, 0x2fdf];
+
+/** @type {import('CJK-util').CodepointRange} */
+export const CJK_RADICALS_SUPPLEMENT_RANGE = [0x2e80, 0x2eff];
+
+/** @type {import('CJK-util').CodepointRange} */
+export const CJK_STROKES_RANGE = [0x31c0, 0x31ef];
+
+/** @type {import('CJK-util').CodepointRange[]} */
+export const CJK_RADICALS_RANGES = [
+    KANGXI_RADICALS_RANGE,
+    CJK_RADICALS_SUPPLEMENT_RANGE,
+    CJK_STROKES_RANGE,
+];
+
+/**
+ * @param {string} text
+ * @returns {string}
+ */
+export function normalizeRadicals(text) {
+    let result = '';
+    for (let i = 0; i < text.length; i++) {
+        const codePoint = text[i].codePointAt(0);
+        result += codePoint && (isCodePointInRanges(codePoint, CJK_RADICALS_RANGES)) ? text[i].normalize('NFKD') : text[i];
+    }
+    return result;
+}
+
+/** @type {import('language').TextProcessor<boolean>} */
+export const normalizeRadicalCharacters = {
+    name: 'Normalize radical characters',
+    description: '⼀ → 一 (U+2F00 → U+4E00)',
+    options: basicTextProcessorOptions,
+    process: (str, setting) => (setting ? normalizeRadicals(str) : str),
+};
diff --git a/ext/js/language/language-descriptors.js b/ext/js/language/language-descriptors.js
@@ -16,6 +16,7 @@
  */
 
 import {removeArabicScriptDiacritics} from './ar/arabic-text-preprocessors.js';
+import {normalizeRadicalCharacters} from './CJK-util.js';
 import {eszettPreprocessor} from './de/german-text-preprocessors.js';
 import {germanTransforms} from './de/german-transforms.js';
 import {englishTransforms} from './en/english-transforms.js';
@@ -212,6 +213,7 @@ const languageDescriptors = [
             convertHalfWidthCharacters,
             alphabeticToHiragana,
             normalizeCombiningCharacters,
+            normalizeRadicalCharacters,
             alphanumericWidthVariants,
             convertHiraganaToKatakana,
             collapseEmphaticSequences,
@@ -374,6 +376,9 @@ const languageDescriptors = [
         iso639_3: 'yue',
         name: 'Cantonese',
         exampleText: '讀',
+        textPreprocessors: {
+            normalizeRadicalCharacters,
+        },
     },
     {
         iso: 'zh',
@@ -382,6 +387,9 @@ const languageDescriptors = [
         exampleText: '读',
         isTextLookupWorthy: isStringPartiallyChinese,
         readingNormalizer: normalizePinyin,
+        textPreprocessors: {
+            normalizeRadicalCharacters,
+        },
     },
 ];
 

diff --git a/types/ext/language-descriptors.d.ts b/types/ext/language-descriptors.d.ts
@@ -135,6 +135,7 @@ type AllTextProcessors = {
             convertHalfWidthCharacters: TextProcessor<boolean>;
             alphabeticToHiragana: TextProcessor<boolean>;
             normalizeCombiningCharacters: TextProcessor<boolean>;
+            normalizeRadicalCharacters: TextProcessor<boolean>;
             alphanumericWidthVariants: BidirectionalConversionPreprocessor;
             convertHiraganaToKatakana: BidirectionalConversionPreprocessor;
             collapseEmphaticSequences: TextProcessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>;
@@ -200,6 +201,14 @@ type AllTextProcessors = {
             normalizeDiacritics: TextProcessor<'old' | 'new' | 'off'>;
         };
     };
-    yue: Record<string, never>;
-    zh: Record<string, never>;
+    yue: {
+        pre: {
+            normalizeRadicalCharacters: TextProcessor<boolean>;
+        };
+    };
+    zh: {
+        pre: {
+            normalizeRadicalCharacters: TextProcessor<boolean>;
+        };
+    };
 };