Use ICU to determine UI language

Switch from relying on wxTranslations to using ICU to determine which
language to use.

This fixes some subtle bugs (e.g. incorrect Chinese choice on macOS) and
simplifies the surrounding code.

src/edapp.cpp

@@ -550,33 +550,26 @@ void PoeditApp::SetupLanguage()
     wxTranslations *trans = new wxTranslations();
     wxTranslations::Set(trans);
 
-    // workaround wx bug, see https://github.com/wxWidgets/wxWidgets/pull/24297
-    class PoeditTranslationsLoader : public wxFileTranslationsLoader
-    {
-    public:
-        wxArrayString GetAvailableTranslations(const wxString& domain) const override
-        {
-            auto all = wxFileTranslationsLoader::GetAvailableTranslations(domain);
-            all.push_back("en");
-            return all;
-        }
-    };
-    trans->SetLoader(new PoeditTranslationsLoader);
+    auto loader = new PoeditTranslationsLoader;
+    trans->SetLoader(loader);
 
     int language = wxLANGUAGE_DEFAULT;
+    Language uilang;
 
 #if NEED_CHOOSELANG_UI
-    auto uilang = GetUILanguage();
-    if (!uilang.empty())
+    uilang = GetUILanguage();
+    if (uilang)
     {
-        auto langinfo = wxLocale::FindLanguageInfo(uilang);
+        auto langinfo = wxLocale::FindLanguageInfo(uilang.Code());
         if (langinfo)
-        {
             language = langinfo->Language;
-            // keep 'uilang' for use below 
-        }
     }
+    else
 #endif
+    {
+        // this returns always valid language, possibly English
+        uilang = loader->DetermineBestUILanguage();
+    }
 
     // Properly set locale is important for some aspects of GTK+ as well as
     // other things. It's also the common thing to do, so don't break
@@ -588,67 +581,42 @@ void PoeditApp::SetupLanguage()
         m_locale.reset(new wxLocale());
         if (!m_locale->Init(language, wxLOCALE_DONT_LOAD_DEFAULT))
             m_locale.reset();
-
-#if NEED_CHOOSELANG_UI
-        if (!uilang.empty())
-            trans->SetLanguage(uilang);
-#endif
     }
 
+    trans->SetLanguage(uilang.LanguageTag());
     trans->AddStdCatalog();
     trans->AddCatalog("poedit");
 
-    wxString bestTrans = trans->GetBestTranslation("poedit");
-    Language uiLang = Language::TryParse(bestTrans.ToStdWstring());
+    g_layoutDirection = uilang.IsRTL() ? wxLayout_RightToLeft : wxLayout_LeftToRight;
+
     UErrorCode err = U_ZERO_ERROR;
-    uloc_setDefault(uiLang.IcuLocaleName().c_str(), &err);
+    uloc_setDefault(uilang.IcuLocaleName().c_str(), &err);
 #if defined(HAVE_HTTP_CLIENT) && !defined(__WXOSX__)
-    http_client::set_ui_language(uiLang.LanguageTag());
+    http_client::set_ui_language(uilang.LanguageTag());
 #endif
 
     char icuVerStr[U_MAX_VERSION_STRING_LENGTH] = {0};
     UVersionInfo icuVer;
     u_getVersion(icuVer);
     u_versionToString(icuVer, icuVerStr);
-    wxLogTrace("poedit", "ICU version %s, using UI language '%s'", icuVerStr, uiLang.LanguageTag());
-
-    const wxLanguageInfo *info = wxLocale::FindLanguageInfo(bestTrans);
-    g_layoutDirection = info ? info->LayoutDirection : wxLayout_Default;
+    wxLogTrace("poedit", "ICU version %s, using UI language '%s'", icuVerStr, uilang.LanguageTag());
 
 #ifdef __WXMSW__
-    AppUpdates::Get().SetLanguage(bestTrans.utf8_string());
+    AppUpdates::Get().SetLanguage(uilang.Code());
 #endif
 
 #ifdef SUPPORTS_OTA_UPDATES
-    SetupOTALanguageUpdate(trans, bestTrans);
+    SetupOTALanguageUpdate(trans, uilang);
 #endif
 }
 
 #ifdef SUPPORTS_OTA_UPDATES
-void PoeditApp::SetupOTALanguageUpdate(wxTranslations *trans, const wxString& lang)
+void PoeditApp::SetupOTALanguageUpdate(wxTranslations *trans, const Language& lang)
 {
-    if (lang == "en" || lang == "en_US")
-        return;
-
-    // normalize language code for requests
-    wxString langMO(lang);
-    if (langMO == "zh-Hans")
-        langMO = "zh_CN";
-    else if (langMO == "zh-Hant")
-        langMO = "zh_TW";
-    else
-        langMO.Replace("-", "_");
+    auto langMO = lang.Code();
 
-#if defined(__UNIX__) && !defined(__WXOSX__)
-    // GetBestTranslation() can fall back to the locale there, so check if we ship this translation
-    auto avail = trans->GetAvailableTranslations("poedit");
-    if (std::find(avail.begin(), avail.end(), lang) == avail.end())
-    {
-        langMO = lang.BeforeFirst('_');
-        if (std::find(avail.begin(), avail.end(), langMO) == avail.end())
-            return;
-    }
-#endif
+    if (langMO == "en" || langMO == "en_US")
+        return;
 
     auto version = str::to_utf8(GetMajorAppVersion());
 
@@ -664,7 +632,7 @@ void PoeditApp::SetupOTALanguageUpdate(wxTranslations *trans, const wxString& la
         return;
     Config::OTATranslationLastCheck(now);
 
-    wxFileName mofile(dir + "/" + lang + "/poedit-ota.mo");
+    wxFileName mofile(dir + "/" + langMO + "/poedit-ota.mo");
     wxFileName::Mkdir(mofile.GetPath(), wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL);
 
     http_client::headers hdrs;

src/edapp.h

@@ -36,6 +36,7 @@
 
 #include "prefsdlg.h"
 
+class Language;
 class WXDLLIMPEXP_FWD_BASE wxConfigBase;
 class WXDLLIMPEXP_FWD_BASE wxSingleInstanceChecker;
 
@@ -108,7 +109,7 @@ class PoeditApp : public wxApp, public MenusManager
 
         void SetupLanguage();
 #ifdef SUPPORTS_OTA_UPDATES
-        void SetupOTALanguageUpdate(wxTranslations *trans, const wxString& lang);
+        void SetupOTALanguageUpdate(wxTranslations *trans, const Language& lang);
 #endif
 
         // App-global menu commands:

src/language.h

@@ -50,6 +50,8 @@ class Language
     Language() : m_direction(TextDirection::LTR) {}
 
     bool IsValid() const { return !m_code.empty(); }
+    explicit operator bool() const { return IsValid(); }
+
     const std::string& Code() const { return m_code; }
     std::wstring WCode() const { return std::wstring(m_code.begin(), m_code.end()); }
 

src/uilang.cpp

@@ -25,31 +25,150 @@
 
 #include "uilang.h"
 
-#if NEED_CHOOSELANG_UI
-
 #include "language.h"
+#include "str_helpers.h"
 
-#include <wx/wx.h>
 #include <wx/config.h>
-#include <wx/translation.h>
+#include <wx/uilocale.h>
+
+#include <unicode/uloc.h>
+
+
+namespace
+{
+
+template<typename T>
+inline void to_vector(T&& list, std::vector<std::string>& strings, std::vector<const char*>& cstrings)
+{
+    strings.reserve(list.size());
+    cstrings.reserve(list.size());
+    for (const auto& s: list)
+    {
+        strings.push_back(str::to_utf8(s));
+        cstrings.push_back(strings.back().c_str());
+    }
+}
+
+// converts to POSIX-like locale, is idempotent
+inline wxString as_posix(const wxString& tag)
+{
+    wxString s(tag);
+    s.Replace("-", "_");
+    s.Replace("zh_Hans", "zh_CN");
+    s.Replace("zh_Hant", "zh_TW");
+    s.Replace("_Latn", "@latin");
+    return s;
+}
+
+// converts to language tag, is idempotent
+inline wxString as_tag(const wxString& posix)
+{
+    wxString s(posix);
+    s.Replace("_", "-");
+    s.Replace("zh-CN", "zh-Hans");
+    s.Replace("zh-TW", "zh-Hant");
+    s.Replace("@latin", "-Latn");
+    return s;
+}
+
+} // anonymous namespace
+
+
+/**
+    Customized loader for translations.
+
+    The primary purpose of this class is to overcome wx bugs or shortcomings:
+
+    - https://github.com/wxWidgets/wxWidgets/pull/24297
+    - https://github.com/wxWidgets/wxWidgets/pull/24804
+
+    Note that this relies on specific knowledge of Poedit's shipping data, it
+    is _not_ a universal replacement!
+ */
+Language PoeditTranslationsLoader::DetermineBestUILanguage() const
+{
+    std::vector<std::string> available, preferred;
+    std::vector<const char*> cavailable, cpreferred;
+    to_vector(GetAvailableTranslations("poedit"), available, cavailable);
+    to_vector(wxUILocale::GetPreferredUILanguages(), preferred, cpreferred);
+
+    char best[ULOC_FULLNAME_CAPACITY];
+    UAcceptResult result;
+    UErrorCode status = U_ZERO_ERROR;
+    UEnumeration *enumLangs = uenum_openCharStringsEnumeration(cavailable.data(), (int32_t)cavailable.size(), &status);
+    if (U_FAILURE(status))
+        return Language::English();
+
+    status = U_ZERO_ERROR;
+    uloc_acceptLanguage(best, std::size(best), &result, cpreferred.data(), (int32_t)cpreferred.size(), enumLangs, &status);
+    uenum_close(enumLangs);
+    if (U_FAILURE(status) || result == ULOC_ACCEPT_FAILED)
+        return Language::English();
+
+    char tag[ULOC_FULLNAME_CAPACITY];
+    status = U_ZERO_ERROR;
+    uloc_toLanguageTag(best, tag, std::size(tag), false, &status);
+    if (U_FAILURE(status))
+        return Language::English();
+
+    return Language::FromLanguageTag(tag);
+}
+
+
+wxArrayString PoeditTranslationsLoader::GetAvailableTranslations(const wxString& domain) const
+{
+    auto all = wxFileTranslationsLoader::GetAvailableTranslations(domain);
+
+    for (auto& lang: all)
+        lang = as_tag(lang);
+    all.push_back("en");
+    all.push_back("be_Latn");
+
+    return all;
+}
+
+
+wxMsgCatalog *PoeditTranslationsLoader::LoadCatalog(const wxString& domain, const wxString& lang_)
+{
+#ifdef __WXOSX__
+    auto lang = (domain == "poedit-ota") ? as_posix(lang_) : as_tag(lang_);
+#else
+    auto lang = as_posix(lang_);
+#endif
+
+    return wxFileTranslationsLoader::LoadCatalog(domain, lang);
+}
+
+
+#if NEED_CHOOSELANG_UI
 
 static void SaveUILanguage(const wxString& lang)
 {
     if (lang.empty())
         wxConfig::Get()->Write("ui_language", "default");
     else
-        wxConfig::Get()->Write("ui_language", lang);
+        wxConfig::Get()->Write("ui_language", as_tag(lang));
 }
 
-wxString GetUILanguage()
+
+Language GetUILanguage()
 {
-    wxString lng = wxConfig::Get()->Read("ui_language");
-    if (!lng.empty() && lng != "default")
-        return lng;
-    else
-        return "";
+    std::string lng = str::to_utf8(as_tag(wxConfig::Get()->Read("ui_language")));
+    if (lng.empty() || lng == "default")
+        return Language();
+
+    auto lang = Language::FromLanguageTag(lng);
+    if (!lang)
+        lang = Language::TryParse(lng); // backward compatibility
+
+    auto all = wxTranslations::Get()->GetAvailableTranslations("poedit");
+    if (all.Index(lang.LanguageTag(), false) == wxNOT_FOUND)
+        return Language();
+
+    return lang;
 }
 
+
 static bool ChooseLanguage(wxString *value)
 {
     wxArrayString langs;
@@ -58,7 +177,6 @@ static bool ChooseLanguage(wxString *value)
     {
         wxBusyCursor bcur;
         langs = wxTranslations::Get()->GetAvailableTranslations("poedit");
-        langs.insert(langs.begin(), "en");
         langs.Sort();
 
         arr.push_back(_("(Use default language)"));
@@ -70,7 +188,7 @@ static bool ChooseLanguage(wxString *value)
     }
 
     auto current = GetUILanguage();
-    int choice = current.empty() ? 0 : langs.Index(current) + 1;
+    int choice = current ? 0 : langs.Index(current.LanguageTag()) + 1;
 
     choice = wxGetSingleChoiceIndex(_("Select your preferred language"), _("Language selection"), arr, choice);
     if ( choice == -1 )

src/uilang.h

@@ -26,22 +26,53 @@
 #ifndef Poedit_uilang_h
 #define Poedit_uilang_h
 
-#include <wx/string.h>
+#include "language.h"
+
 #include <wx/intl.h>
+#include <wx/string.h>
+#include <wx/translation.h>
 
 #ifdef __WXMSW__
     #define NEED_CHOOSELANG_UI 1
 #else
     #define NEED_CHOOSELANG_UI 0
 #endif
 
+
+/**
+    Customized loader for translations.
+
+    The primary purpose of this class is to overcome wx bugs or shortcomings:
+
+    - https://github.com/wxWidgets/wxWidgets/pull/24297
+    - https://github.com/wxWidgets/wxWidgets/pull/24804
+
+    Note that this relies on specific knowledge of Poedit's shipping data, it
+    is _not_ a universal replacement!
+ */
+class PoeditTranslationsLoader : public wxFileTranslationsLoader
+{
+public:
+    /**
+        Use ICU to determine UI languages; replaces wxTranslations::GetBestTranslation().
+
+        Always returns a valid language (using English as fallback).
+     */
+    Language DetermineBestUILanguage() const;
+
+    // overrides to use language tags:
+    wxArrayString GetAvailableTranslations(const wxString& domain) const override;
+    wxMsgCatalog *LoadCatalog(const wxString& domain, const wxString& lang_) override;
+};
+
+
 #if NEED_CHOOSELANG_UI
 
 /// Let the user change UI language
 void ChangeUILanguage();
 
 /** Return currently chosen language. Calls ChooseLanguage if necessary. */
-wxString GetUILanguage();
+Language GetUILanguage();
 
 #endif