✨ Add a cache system that recognizes changes to key-based translation…

… files

.gitignore

@@ -19,6 +19,7 @@ service-account.json
 
 # Test Data
 locales/
+.json-autotranslate-cache
 
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov

package.json

@@ -35,11 +35,13 @@
     "@types/node": "^11.10.4",
     "chalk": "^2.4.2",
     "commander": "^2.19.0",
+    "deep-object-diff": "^1.1.0",
     "dotenv": "^6.2.0",
     "flattenjs": "^1.0.4",
     "inquirer": "^6.2.2",
     "lodash": "^4.17.11",
     "messageformat-parser": "^4.1.0",
+    "ncp": "^2.0.0",
     "node-fetch": "^2.3.0",
     "ts-node": "^8.0.2",
     "tslint": "^5.13.1",
@@ -50,6 +52,7 @@
     "@types/inquirer": "^0.0.44",
     "@types/jest": "^24.0.10",
     "@types/lodash": "^4.14.122",
+    "@types/ncp": "^2.0.3",
     "@types/node-fetch": "^2.1.6",
     "jest": "^24.3.1",
     "ts-jest": "^24.0.0"

src/index.ts

@@ -6,6 +6,8 @@ import * as flatten from 'flattenjs';
 import * as fs from 'fs';
 import { omit } from 'lodash';
 import * as path from 'path';
+import { diff } from 'deep-object-diff';
+import ncp from 'ncp';
 
 import { serviceMap } from './services';
 import {
@@ -24,6 +26,11 @@ commander
     'the directory containing language directories',
     '.',
   )
+  .option(
+    '--cache <cacheDir>',
+    'set the cache directory',
+    '.json-autotranslate-cache',
+  )
   .option(
     '-l, --source-language <sourceLang>',
     'specify the source language',
@@ -63,6 +70,7 @@ commander
 
 const translate = async (
   inputDir: string = '.',
+  cacheDir: string = '.json-autotranslate-cache',
   sourceLang: string = 'en',
   deleteUnusedStrings = false,
   fileType: FileType = 'auto',
@@ -72,9 +80,15 @@ const translate = async (
   config?: string,
 ) => {
   const workingDir = path.resolve(process.cwd(), inputDir);
+  const resolvedCacheDir = path.resolve(process.cwd(), cacheDir);
   const languageFolders = getAvailableLanguages(workingDir);
   const targetLanguages = languageFolders.filter(f => f !== sourceLang);
 
+  if (!fs.existsSync(resolvedCacheDir)) {
+    fs.mkdirSync(resolvedCacheDir);
+    console.log(`🗂 Created the cache directory.`);
+  }
+
   if (!languageFolders.includes(sourceLang)) {
     throw new Error(`The source language ${sourceLang} doesn't exist.`);
   }
@@ -127,9 +141,7 @@ const translate = async (
 
   if (!availableLanguages.includes(sourceLang)) {
     throw new Error(
-      `${
-        translationService.name
-      } doesn't support the source language ${sourceLang}`,
+      `${translationService.name} doesn't support the source language ${sourceLang}`,
     );
   }
 
@@ -162,7 +174,10 @@ const translate = async (
 
     if (fixInconsistencies) {
       console.log(`💚 Fixing inconsistencies...`);
-      fixSourceInconsistencies(path.resolve(workingDir, sourceLang));
+      fixSourceInconsistencies(
+        path.resolve(workingDir, sourceLang),
+        path.resolve(resolvedCacheDir, sourceLang),
+      );
       console.log(chalk`└── {green.bold Fixed all inconsistencies.}`);
     } else {
       console.log(
@@ -213,12 +228,13 @@ const translate = async (
   }
   console.log();
 
+  let addedTranslations = 0;
+  let removedTranslations = 0;
+
   for (const language of targetLanguages) {
     if (!availableLanguages.includes(language)) {
       console.log(
-        chalk`🙈 {yellow.bold ${
-          translationService.name
-        } doesn't support} {red.bold ${language}}{yellow.bold . Skipping this language.}`,
+        chalk`🙈 {yellow.bold ${translationService.name} doesn't support} {red.bold ${language}}{yellow.bold . Skipping this language.}`,
       );
       console.log();
       continue;
@@ -241,12 +257,15 @@ const translate = async (
 
       for (const file of deletableFiles) {
         console.log(
-          chalk`├── {red.bold ${
-            file.name
-          } is no longer used and will be deleted.}`,
+          chalk`├── {red.bold ${file.name} is no longer used and will be deleted.}`,
         );
 
         fs.unlinkSync(path.resolve(workingDir, language, file.name));
+
+        const cacheFile = path.resolve(workingDir, language, file.name);
+        if (fs.existsSync(cacheFile)) {
+          fs.unlinkSync(cacheFile);
+        }
       }
     }
 
@@ -261,9 +280,31 @@ const translate = async (
         : [];
       const existingTranslations = languageFile ? languageFile.content : {};
 
+      const cachePath = path.resolve(
+        resolvedCacheDir,
+        sourceLang,
+        languageFile.name,
+      );
+      let cacheDiff: string[] = [];
+      if (fs.existsSync(cachePath)) {
+        const cachedFile = flatten.convert(
+          JSON.parse(fs.readFileSync(cachePath).toString()),
+        );
+        const cDiff = diff(cachedFile, templateFile.content);
+        cacheDiff = Object.keys(cDiff).filter(k => cDiff[k]);
+        console.log();
+        console.log('Cached File:', cachedFile);
+        console.log('Current File:', templateFile.content);
+        console.log('Cache Diff:', cacheDiff);
+        const changedItems = Object.keys(cacheDiff).length.toString();
+        process.stdout.write(
+          chalk` ({green.bold ${changedItems}} changes from cache)`,
+        );
+      }
+
       const templateStrings = Object.keys(templateFile.content);
       const stringsToTranslate = templateStrings
-        .filter(key => !existingKeys.includes(key))
+        .filter(key => !existingKeys.includes(key) || cacheDiff.includes(key))
         .map(key => ({
           key,
           value:
@@ -285,7 +326,10 @@ const translate = async (
         {} as { [k: string]: string },
       );
 
-      if (service !== 'dryRun') {
+      addedTranslations += translatedStrings.length;
+      removedTranslations += deleteUnusedStrings ? unusedStrings.length : 0;
+
+      if (service !== 'dry-run') {
         const translatedFile = {
           ...omit(
             existingTranslations,
@@ -294,15 +338,27 @@ const translate = async (
           ...newKeys,
         };
 
-        fs.writeFileSync(
-          path.resolve(workingDir, language, templateFile.name),
+        const newContent =
           JSON.stringify(
             templateFile.type === 'key-based'
               ? flatten.undo(translatedFile)
               : translatedFile,
             null,
             2,
-          ) + `\n`,
+          ) + `\n`;
+
+        fs.writeFileSync(
+          path.resolve(workingDir, language, templateFile.name),
+          newContent,
+        );
+
+        const languageCachePath = path.resolve(resolvedCacheDir, language);
+        if (!fs.existsSync(languageCachePath)) {
+          fs.mkdirSync(languageCachePath);
+        }
+        fs.writeFileSync(
+          path.resolve(languageCachePath, templateFile.name),
+          JSON.stringify(translatedFile, null, 2) + '\n',
         );
       }
 
@@ -319,7 +375,30 @@ const translate = async (
     console.log();
   }
 
-  console.log(chalk.green.bold('All new strings have been translated!'));
+  if (service !== 'dry-run') {
+    console.log('🗂 Caching source translation files...');
+    await new Promise((res, rej) =>
+      ncp(
+        path.resolve(workingDir, sourceLang),
+        path.resolve(resolvedCacheDir, sourceLang),
+        err => (err ? rej() : res()),
+      ),
+    );
+    console.log(chalk`└── {green.bold Translation files have been cached.}`);
+    console.log();
+  }
+
+  console.log(
+    chalk.green.bold(`${addedTranslations} new translations have been added!`),
+  );
+
+  if (removedTranslations > 0) {
+    console.log(
+      chalk.green.bold(
+        `${removedTranslations} translations have been removed!`,
+      ),
+    );
+  }
 };
 
 if (commander.listServices) {
@@ -336,6 +415,7 @@ if (commander.listMatchers) {
 
 translate(
   commander.input,
+  commander.cacheDir,
   commander.sourceLanguage,
   commander.deleteUnusedStrings,
   commander.type,

src/util/file-system.ts

@@ -41,7 +41,10 @@ export const loadTranslations = (
       };
     });
 
-export const fixSourceInconsistencies = (directory: string) => {
+export const fixSourceInconsistencies = (
+  directory: string,
+  cacheDir: string,
+) => {
   const files = loadTranslations(directory).filter(f => f.type === 'natural');
 
   for (const file of files) {
@@ -54,5 +57,10 @@ export const fixSourceInconsistencies = (directory: string) => {
       path.resolve(directory, file.name),
       JSON.stringify(fixedContent, null, 2) + '\n',
     );
+
+    fs.writeFileSync(
+      path.resolve(cacheDir, file.name),
+      JSON.stringify(fixedContent, null, 2) + '\n',
+    );
   }
 };

yarn.lock

@@ -405,6 +405,13 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.122.tgz#3e31394c38cf1e5949fb54c1192cbc406f152c6c"
   integrity sha512-9IdED8wU93ty8gP06ninox+42SBSJHp2IAamsSYMUY76mshRTeUsid/gtbl8ovnOwy8im41ib4cxTiIYMXGKew==
 
+"@types/ncp@^2.0.3":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@types/ncp/-/ncp-2.0.3.tgz#68429472698a68f2c85a984932e9d8014d5f2ed0"
+  integrity sha512-eZatZMWhPHUHY/1hUUqwGrWzBAE9deYR3L0QJMicpVkBUOrQslhWblw5Ye+rKuzvFG/UQ3jDolMcjhC2WKUQ5w==
+  dependencies:
+    "@types/node" "*"
+
 "@types/node-fetch@^2.1.6":
   version "2.1.6"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.1.6.tgz#4326288b49f352a142f03c63526ebce0f4c50877"
@@ -1146,6 +1153,11 @@ deep-is@~0.1.3:
   resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34"
   integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=
 
+deep-object-diff@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/deep-object-diff/-/deep-object-diff-1.1.0.tgz#d6fabf476c2ed1751fc94d5ca693d2ed8c18bc5a"
+  integrity sha512-b+QLs5vHgS+IoSNcUE4n9HP2NwcHj7aqnJWsjPtuG75Rh5TOaGt0OjAYInh77d5T16V5cRDC+Pw/6ZZZiETBGw==
+
 default-require-extensions@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/default-require-extensions/-/default-require-extensions-2.0.0.tgz#f5f8fbb18a7d6d50b21f641f649ebb522cfe24f7"
@@ -2810,6 +2822,11 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
+ncp@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3"
+  integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=
+
 nice-try@^1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"