noswhere translation backend

README.md

@@ -11,9 +11,14 @@ The Damus API backend for Damus Purple and other functionality.
 #### Essential
 
 - `DB_PATH`: Path to the folder where to save mdb files.
-- `DEEPL_KEY`: API key for DeepL translation service (Can be set to something bogus for local testing with mock translations)
 - `TESTFLIGHT_URL`: URL for the TestFlight app (optional)
 
+#### Translations
+
+- `TRANSLATION_PROVIDER`: The translation provider to use, can be: `mock`, `deepl`, `noswhere`
+- `DEEPL_KEY`: The DeepL key to use for DeepL translations if enabled.
+- `NOSWHERE_KEY`: The Noswhere key to use for Noswhere translations if enabled.
+
 #### Apple In-App Purchase (IAP)
 
 - `ENABLE_IAP_PAYMENTS`: Set to `"true"` to enable Apple In-App Purchase payment endpoints.

package.json

@@ -10,14 +10,13 @@
     "damus-api": "./src/index.js"
   },
   "scripts": {
-    "test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 tap test/*.test.js",
+    "test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 TRANSLATION_PROVIDER=mock tap test/*.test.js",
     "test-translate": "ALLOW_HTTP_AUTH=\"true\" TRANSLATION_PROVIDER=mock tap test/translate.test.js",
+    "test-noswhere": "ALLOW_HTTP_AUTH=\"true\" TRANSLATION_PROVIDER=noswhere tap test/translate.test.js",
     "debug-test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 tap test/*.test.js --timeout=2400",
     "start": "node src/index.js",
-    "mock_deepl": "node test_utils/mock_deepl.js",
-    "start_with_mock": "DEEPL_KEY=123 DEEPL_URL=http://localhost:8990 ENABLE_HTTP_AUTH=\"true\" node src/index.js",
-    "dev": "npm run mock_deepl & npm run start_with_mock",
-    "dev-debug": "npm run mock_deepl & DEEPL_KEY=123 DEEPL_URL=http://localhost:8990 ENABLE_HTTP_AUTH=\"true\" node inspect src/index.js",
+    "dev": "TRANSLATION_PROVIDER=mock ENABLE_HTTP_AUTH=\"true\" node src/index.js",
+    "dev-debug": "TRANSLATION_PROVIDER=mock ENABLE_HTTP_AUTH=\"true\" node --inspect src/index.js",
     "type-check": "tsc --checkJs --allowJs src/*.js --noEmit --skipLibCheck",
     "type-check-path": "tsc --checkJs --allowJs --noEmit --skipLibCheck"
   },

src/translate.js

@@ -2,71 +2,52 @@
 const util = require('./server_helpers')
 const crypto = require('crypto')
 const current_time = require('./utils').current_time
+const SUPPORTED_TRANSLATION_PROVIDERS = new Set(["mock", "noswhere", "deepl"])
+let translation_provider = null
 
-const translate_sources = new Set(['BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
-const translate_targets = new Set(['BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'PT-BR', 'PT-PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
+if (!process.env.TRANSLATION_PROVIDER) {
+  throw new Error("expected TRANSLATION_PROVIDER")
+}
 
-const DEEPL_KEY = process.env.DEEPL_KEY
-const DEEPL_URL = process.env.DEEPL_URL || 'https://api.deepl.com/v2/translate'
+if (!SUPPORTED_TRANSLATION_PROVIDERS.has(process.env.TRANSLATION_PROVIDER)) {
+  throw new Error("translation provider not supported")
+}
 
-if (!DEEPL_KEY)
-  throw new Error("expected DEEPL_KEY env var")
+// this is safe, as the input value is restricted to known good ones.
+translation_provider = new (require("./translate/" + process.env.TRANSLATION_PROVIDER + ".js"))()
 
 async function validate_payload(payload) {
-  if (!payload.source)
-    return { ok: false, message: 'missing source' }
-  if (!payload.target)
-    return { ok: false, message: 'missing target' }
-  if (!payload.q)
-    return { ok: false, message: 'missing q' }
-  if (!translate_sources.has(payload.source))
-    return { ok: false, message: 'invalid translation source' }
-  if (!translate_targets.has(payload.target))
-    return { ok: false, message: 'invalid translation target' }
+  if (typeof payload.source !== "string")
+    return { ok: false, message: 'bad source' }
+  if (typeof payload.target !== "string")
+    return { ok: false, message: 'bad target' }
+  if (typeof payload.q !== "string")
+    return { ok: false, message: 'bad q' }
+  if (!translation_provider.canTranslate(payload.source, payload.target))
+    return { ok: false, message: 'invalid translation source/target' }
 
   return { ok: true, message: 'valid' }
 }
 
 function hash_payload(payload) {
   const hash = crypto.createHash('sha256')
   hash.update(payload.q)
-  hash.update(payload.source)
-  hash.update(payload.target)
+  hash.update(payload.source.toUpperCase())
+  hash.update(payload.target.toUpperCase())
   return hash.digest()
 }
 
-async function deepl_translate_text(payload) {
-  let resp = await fetch(DEEPL_URL, {
-    method: 'POST',
-    headers: {
-      'Authorization': `DeepL-Auth-Key ${DEEPL_KEY}`,
-      'Content-Type': 'application/json'
-    },
-    body: JSON.stringify({
-      text: [payload.q],
-      source_lang: payload.source,
-      target_lang: payload.target,
-    })
-  })
-
-  let data = await resp.json()
-
-  if (data.translations && data.translations.length > 0) {
-    return data.translations[0].text;
-  }
-
-  return null
-}
-
 async function translate_payload(api, res, payload, trans_id) {
   // we might already be translating this
   const job = api.translation.queue[trans_id]
   if (job) {
-    let text = await job
-    if (text === null)
-      return util.error_response(res, 'deepl translation error')
-
-    return util.json_response(res, { text })
+    try {
+      let result = await job
+      return util.json_response(res, { text: result.text })
+    } catch(e) {
+      console.error("translation error: %o", e)
+      return util.error_response(res, 'translation error')
+    }
   }
 
   // we might have it in the database already
@@ -78,26 +59,28 @@ async function translate_payload(api, res, payload, trans_id) {
     return util.json_response(res, { text })
   }
 
-  const new_job = deepl_translate_text(payload)
+  const new_job = translation_provider.translate(payload.source, payload.target, payload.q)
   api.translation.queue[trans_id] = new_job
 
-  let text = await new_job
-  if (text === null) {
+  let result
+
+  try {
+    result = await new_job
+  } catch(e) {
+    console.error("translation error: %o", e)
+    return util.error_response(res, 'translation error')
+  } finally {
     delete api.translation.queue[trans_id]
-    return util.error_response(res, 'deepl translation error')
   }
-
   // return results immediately
-  util.json_response(res, { text })
+  util.json_response(res, { text: result.text })
 
   // write result to db
   await api.dbs.translations.put(trans_id, {
-    text: text,
+    text: result.text,
     translated_at: current_time(),
     payload: payload
   })
-
-  delete api.translation.queue[trans_id]
 }
 
 function payload_is_data(q) {
@@ -111,8 +94,8 @@ function payload_is_data(q) {
 async function handle_translate(api, req, res) {
   let id
   try {
-    const source = req.query.source.toUpperCase()
-    const target = req.query.target.toUpperCase()
+    const source = req.query.source.toLowerCase()
+    const target = req.query.target.toLowerCase()
     const q = req.query.q
     if (payload_is_data(q))
       return util.invalid_request(res, `payload is data`)

src/translate/deepl.js

@@ -0,0 +1,57 @@
+const translate_sources = new Set([
+    'bg', 'cs', 'da', 'de', 'el',
+    'en', 'es', 'et', 'fi', 'fr',
+    'hu', 'id', 'it', 'ja', 'ko',
+    'lt', 'lv', 'nb', 'nl', 'pl',
+    'pt', 'ro', 'ru', 'sk', 'sl',
+    'sv', 'tr', 'uk', 'zh'
+])
+const translate_targets = new Set([
+    'bg', 'cs', 'da', 'de',
+    'el', 'en', 'en-gb', 'en-us',
+    'es', 'et', 'fi', 'fr',
+    'hu', 'id', 'it', 'ja',
+    'ko', 'lt', 'lv', 'nb',
+    'nl', 'pl', 'pt', 'pt-br',
+    'pt-pt', 'ro', 'ru', 'sk',
+    'sl', 'sv', 'tr', 'uk',
+    'zh'
+])
+
+module.exports = class DeepLTranslator {
+    #deeplURL = process.env.DEEPL_URL || 'https://api.deepl.com/v2/translate'
+    #deeplKey = process.env.DEEPL_KEY
+    constructor() {
+        if (!this.#deeplKey)
+            throw new Error("expected DEEPL_KEY env var")
+    }
+    canTranslate(from_lang, to_lang) {
+        return translate_sources.has(from_lang) && translate_targets.has(to_lang)
+    }
+    async translate(from_lang, to_lang, text) {
+        let resp = await fetch(this.#deeplURL, {
+            method: 'POST',
+            headers: {
+                'Authorization': `DeepL-Auth-Key ${this.#deeplKey}`,
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+                text: [text],
+                source_lang: from_lang.toUpperCase(),
+                target_lang: to_lang.toUpperCase(),
+            })
+        })
+
+        if (!resp.ok) throw new Error("error translating: API failed with " + resp.status + " " + resp.statusText)
+
+        let data = await resp.json()
+
+        if (data.translations && data.translations.length > 0) {
+            return {
+                text: data.translations[0].text
+            }
+        }
+
+        throw new Error("error translating: no response")
+    }
+}

src/translate/mock.js

@@ -0,0 +1,13 @@
+module.exports = class MockTranslator {
+    constructor() {
+
+    }
+    canTranslate(from_lang, to_lang) {
+        return true
+    }
+    async translate(from_lang, to_lang, text) {
+        return {
+            text: "Mock translation"
+        }
+    }
+}

src/translate/noswhere.js

@@ -0,0 +1,62 @@
+module.exports = class NoswhereTranslator {
+    #noswhereURL = process.env.NOSWHERE_URL || 'https://translate.api.noswhere.com/api'
+    #noswhereKey = process.env.NOSWHERE_KEY
+    #type = "default"
+    #fromLangs = new Set()
+    #toLangs = new Set()
+    constructor() {
+        if (!this.#noswhereKey)
+            throw new Error("expected NOSWHERE_KEY env var")
+        this.#loadTranslationLangs()
+    }
+    async #loadTranslationLangs() {
+        let resp = await fetch(this.#noswhereURL + "/langs", {
+            method: 'GET',
+            timeout: 5000,
+            headers: {
+                'X-Noswhere-Key': this.#noswhereKey,
+                'Content-Type': 'application/json'
+            }
+        })
+        let data = await resp.json()
+        if (!resp.ok) {
+            throw new Error(`error getting translation langs: API failed with ${resp.status} ${data.error} (request: ${resp.headers.get("x-noswhere-request")})`)
+        }
+        if (!data[this.#type]) {
+            throw new Error(`type ${this.#type} not supported for translation`)
+        }
+        this.#fromLangs = new Set(data[this.#type].from)
+        this.#toLangs = new Set(data[this.#type].to)
+    }
+    canTranslate(from_lang, to_lang) {
+        if (this.#fromLangs.size === 0) return true // assume true until we get the list of languages
+        return this.#fromLangs.has(from_lang) && this.#toLangs.has(to_lang)
+    }
+    async translate(from_lang, to_lang, text) {
+        let resp = await fetch(this.#noswhereURL + "/translate", {
+            method: 'POST',
+            headers: {
+                'X-Noswhere-Key': this.#noswhereKey,
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+                text: text,
+                src_lang: from_lang,
+                dst_lang: to_lang,
+            })
+        })
+
+        let data = await resp.json()
+        if (!resp.ok) {
+            throw new Error(`error translating: API failed with ${resp.status} ${data.error} (request: ${resp.headers.get("x-noswhere-request")})`)
+        }
+
+        if (data.result) {
+            return {
+                text: data.result
+            }
+        }
+
+        throw new Error("error translating: no response")
+    }
+}