[WJ-1191] Implement fallback locales

.github/workflows/deepwell.yaml

@@ -129,4 +129,10 @@ jobs:
         run: cd deepwell && cargo fmt --all -- --check
 
       - name: Clippy
-        run: cd deepwell && cargo clippy --no-deps
+        run: cd deepwell && cargo clippy --no-deps -- -A unused_imports
+
+        # clippy is over aggressive with "unused import" warnings, reporting it for
+        # prelude modules and common export patterns, which is noisy and unhelpful.
+        #
+        # Since regular (i.e. actual) unused imports will fail the normal build, we
+        # can just suppress all unused import warnings in Clippy.

deepwell/.clippy.toml

@@ -0,0 +1,2 @@
+# Code smells
+disallowed-names = ["foo", "bar", "baz", "todo"]

deepwell/Cargo.toml

@@ -31,6 +31,7 @@ either = "1"
 femme = "2"
 filemagic = "0.12"
 fluent = "0.16"
+fluent-syntax = "0"
 ftml = { version = "1.22", features = ["mathml"] }
 futures = { version = "0.3", features = ["async-await"], default-features = false }
 hex = { version = "0.4", features = ["serde"] }

deepwell/src/endpoints/locale.rs

@@ -32,9 +32,9 @@ pub struct LocaleOutput {
 }
 
 #[derive(Deserialize, Debug, Clone)]
-pub struct TranslateInput<'a> {
-    locale: &'a str,
-    messages: HashMap<String, MessageArguments<'a>>,
+pub struct TranslateInput {
+    locales: Vec<String>,
+    messages: HashMap<String, MessageArguments<'static>>,
 }
 
 type TranslateOutput = HashMap<String, String>;
@@ -58,18 +58,29 @@ pub async fn translate_strings(
     ctx: &ServiceContext<'_>,
     params: Params<'static>,
 ) -> Result<TranslateOutput> {
-    let TranslateInput {
-        locale: locale_str,
-        messages,
-    } = params.parse()?;
+    let TranslateInput { locales, messages } = params.parse()?;
+
+    if locales.is_empty() {
+        error!("No locales specified in translate call");
+        return Err(ServiceError::NoLocalesSpecified);
+    }
 
     info!(
-        "Translating {} message keys in locale {locale_str}",
+        "Translating {} message keys in locale {} (or {} fallbacks)",
         messages.len(),
+        &locales[0],
+        locales.len() - 1,
     );
 
-    let locale = LanguageIdentifier::from_bytes(locale_str.as_bytes())?;
     let mut output: TranslateOutput = HashMap::new();
+    let locales = {
+        let mut langids = Vec::new();
+        for locale in locales {
+            let langid = LanguageIdentifier::from_bytes(locale.as_bytes())?;
+            langids.push(langid);
+        }
+        langids
+    };
 
     for (message_key, arguments_raw) in messages {
         info!(
@@ -80,7 +91,7 @@ pub async fn translate_strings(
         let arguments = arguments_raw.into_fluent_args();
         let translation =
             ctx.localization()
-                .translate(&locale, &message_key, &arguments)?;
+                .translate(&locales, &message_key, &arguments)?;
 
         output.insert(message_key, translation.to_string());
     }

deepwell/src/locales/fallback.rs

@@ -0,0 +1,145 @@
+/*
+ * locales/fallback.rs
+ *
+ * DEEPWELL - Wikijump API provider and database manager
+ * Copyright (C) 2019-2023 Wikijump Team
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//! Module to implement locale fallbacks.
+//!
+//! This is different than having a list of locales and simply trying each one.
+//! Beyond that, this is another important component to finding a proper locale.
+//!
+//! Given some locale, it iterates through increasingly generic forms of it
+//! until a match can be found (or not).
+//!
+//! The order followed is:
+//! * Language, script, region, and variant (unmodified)
+//! * Language, script, and region
+//! * Language and script
+//! * Language, region, and variant
+//! * Language and region
+//! * Language only
+//!
+//! The logic here will skip a locale variant if it's already been outputted.
+//! So for a locale like `ko`, it will only emit one item, `ko`. For something like `en-CA`,
+//! it will emit `en-CA` then `en`.
+//!
+//! If `Some(_)` or `Err(_)` is returned, then iteration will end prematurely.
+
+use unic_langid::LanguageIdentifier;
+
+pub fn iterate_locale_fallbacks<F, T>(
+    mut locale: LanguageIdentifier,
+    mut f: F,
+) -> Option<(LanguageIdentifier, T)>
+where
+    F: FnMut(&LanguageIdentifier) -> Option<T>,
+{
+    debug!("Iterating through locale fallbacks for {locale}");
+
+    macro_rules! try_iter {
+        () => {
+            if let Some(result) = f(&locale) {
+                return Some((locale, result));
+            }
+        };
+    }
+
+    // Storage of temporarily removed fields.
+    let variants: Vec<_> = locale.variants().cloned().collect();
+
+    // Unmodified locale
+    // Language, script, region, variant
+    try_iter!();
+
+    if !variants.is_empty() {
+        // Remove variant
+        // Language, script, region
+        locale.clear_variants();
+        try_iter!();
+    }
+
+    // Remove region
+    // Language, script
+    let region = locale.region.take();
+    if region.is_some() {
+        try_iter!();
+    }
+
+    if locale.script.take().is_some() {
+        // Re-add region and variant, remove script
+        // Language, region, variant
+        locale.region = region;
+        locale.set_variants(&variants);
+        try_iter!();
+
+        if !variants.is_empty() {
+            // Remove variant
+            // Language, region
+            locale.clear_variants();
+            try_iter!();
+        }
+
+        if locale.region.is_some() {
+            // Remove region
+            // Language only
+            locale.region = None;
+            try_iter!();
+        }
+    }
+
+    // No results
+    None
+}
+
+#[test]
+fn fallbacks() {
+    fn check(locale: &str, expected: &[&str]) {
+        let locale = locale.parse().expect("Unable to parse locale");
+        let mut actual = Vec::new();
+
+        iterate_locale_fallbacks::<_, ()>(locale, |locale| {
+            actual.push(str!(locale));
+            None
+        });
+
+        assert!(
+            actual.iter().eq(expected),
+            "Actual fallback locale list doesn't match expected\nactual:   {:?}\nexpected: {:?}",
+            actual,
+            expected,
+        );
+    }
+
+    check("en", &["en"]);
+    check("fr-be", &["fr-BE", "fr"]);
+    check("es-Latn", &["es-Latn", "es"]);
+    check("en-Latn-US", &["en-Latn-US", "en-Latn", "en-US", "en"]);
+    check("en-Valencia", &["en-valencia", "en"]);
+    check("en_CA_valencia", &["en-CA-valencia", "en-CA", "en"]);
+    check(
+        "en_Latn-CA_valencia",
+        &[
+            "en-Latn-CA-valencia",
+            "en-Latn-CA",
+            "en-Latn",
+            "en-CA-valencia",
+            "en-CA",
+            "en",
+        ],
+    );
+}