✨ ability to add migrations (actualbudget#267)

.gitignore

@@ -15,6 +15,8 @@ build/
 *.pem
 *.key
 artifacts.json
+.migrate
+.migrate-test
 
 # Yarn
 .pnp.*

app.js

@@ -1,6 +1,9 @@
 import run from './src/app.js';
+import runMigrations from './src/migrations.js';
 
-run().catch((err) => {
-  console.log('Error starting app:', err);
-  process.exit(1);
-});
+runMigrations()
+  .then(run)
+  .catch((err) => {
+    console.log('Error starting app:', err);
+    process.exit(1);
+  });

jest.config.json

@@ -1,5 +1,6 @@
 {
-  "setupFiles": ["./jest.setup.js"],
+  "globalSetup": "./jest.global-setup.js",
+  "globalTeardown": "./jest.global-teardown.js",
   "testPathIgnorePatterns": ["dist", "/node_modules/", "/build/"],
   "roots": ["<rootDir>"],
   "moduleFileExtensions": ["ts", "js", "json"],

jest.global-setup.js

@@ -0,0 +1,10 @@
+import getAccountDb from './src/account-db.js';
+import runMigrations from './src/migrations.js';
+
+export default async function setup() {
+  await runMigrations();
+
+  // Insert a fake "valid-token" fixture that can be reused
+  const db = getAccountDb();
+  await db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);
+}

jest.global-teardown.js

@@ -0,0 +1,5 @@
+import runMigrations from './src/migrations.js';
+
+export default async function teardown() {
+  await runMigrations('down');
+}

migrations/1694360000000-create-folders.js

@@ -0,0 +1,24 @@
+import fs from 'node:fs/promises';
+import config from '../src/load-config.js';
+
+async function ensureExists(path) {
+  try {
+    await fs.mkdir(path);
+  } catch (err) {
+    if (err.code == 'EEXIST') {
+      return null;
+    }
+
+    throw err;
+  }
+}
+
+export const up = async function () {
+  await ensureExists(config.serverFiles);
+  await ensureExists(config.userFiles);
+};
+
+export const down = async function () {
+  await fs.rm(config.serverFiles, { recursive: true, force: true });
+  await fs.rm(config.userFiles, { recursive: true, force: true });
+};

migrations/1694360479680-create-account-db.js

@@ -0,0 +1,30 @@
+import getAccountDb from '../src/account-db.js';
+
+export const up = async function () {
+  await getAccountDb().exec(`
+    CREATE TABLE IF NOT EXISTS auth
+      (password TEXT PRIMARY KEY);
+
+    CREATE TABLE IF NOT EXISTS sessions
+      (token TEXT PRIMARY KEY);
+
+    CREATE TABLE IF NOT EXISTS files
+      (id TEXT PRIMARY KEY,
+       group_id TEXT,
+       sync_version SMALLINT,
+       encrypt_meta TEXT,
+       encrypt_keyid TEXT,
+       encrypt_salt TEXT,
+       encrypt_test TEXT,
+       deleted BOOLEAN DEFAULT FALSE,
+       name TEXT);
+  `);
+};
+
+export const down = async function () {
+  await getAccountDb().exec(`
+    DROP TABLE auth;
+    DROP TABLE sessions;
+    DROP TABLE files;
+  `);
+};

migrations/1694362247011-create-secret-table.js

@@ -0,0 +1,16 @@
+import getAccountDb from '../src/account-db.js';
+
+export const up = async function () {
+  await getAccountDb().exec(`
+    CREATE TABLE IF NOT EXISTS secrets (
+      name TEXT PRIMARY KEY,
+      value BLOB
+    );
+  `);
+};
+
+export const down = async function () {
+  await getAccountDb().exec(`
+    DROP TABLE secrets;
+  `);
+};

package.json

@@ -28,6 +28,7 @@
     "express-rate-limit": "^6.7.0",
     "express-response-size": "^0.0.3",
     "jws": "^4.0.0",
+    "migrate": "^2.0.0",
     "nordigen-node": "^1.2.6",
     "uuid": "^9.0.0"
   },

src/account-db.js

@@ -1,34 +1,15 @@
-import fs from 'node:fs';
 import { join } from 'node:path';
 import openDatabase from './db.js';
-import config, { sqlDir } from './load-config.js';
-import createDebug from 'debug';
+import config from './load-config.js';
 import * as uuid from 'uuid';
 import * as bcrypt from 'bcrypt';
 
-const debug = createDebug('actual:account-db');
-
 let _accountDb = null;
 
 export default function getAccountDb() {
   if (_accountDb == null) {
-    if (!fs.existsSync(config.serverFiles)) {
-      debug(`creating server files directory: '${config.serverFiles}'`);
-      fs.mkdirSync(config.serverFiles);
-    }
-
-    let dbPath = join(config.serverFiles, 'account.sqlite');
-    let needsInit = !fs.existsSync(dbPath);
-
+    const dbPath = join(config.serverFiles, 'account.sqlite');
     _accountDb = openDatabase(dbPath);
-
-    if (needsInit) {
-      debug(`initializing account database: '${dbPath}'`);
-      let initSql = fs.readFileSync(join(sqlDir, 'account.sql'), 'utf8');
-      _accountDb.exec(initSql);
-    } else {
-      debug(`opening account database: '${dbPath}'`);
-    }
   }
 
   return _accountDb;

src/app-account.js

@@ -13,10 +13,6 @@ app.use(errorMiddleware);
 
 export { app as handlers };
 
-export function init() {
-  // eslint-disable-previous-line @typescript-eslint/no-empty-function
-}
-
 // Non-authenticated endpoints:
 //
 // /needs-bootstrap

src/app-gocardless/services/gocardless-service.js

@@ -17,17 +17,12 @@ import jwt from 'jws';
 import { SecretName, secretsService } from '../../services/secrets-service.js';
 
 const GoCardlessClient = nordigenNode.default;
-const goCardlessClient = new GoCardlessClient({
-  secretId: secretsService.get(SecretName.nordigen_secretId),
-  secretKey: secretsService.get(SecretName.nordigen_secretKey),
-});
-
-secretsService.onUpdate(SecretName.nordigen_secretId, (newSecret) => {
-  goCardlessClient.secretId = newSecret;
-});
-secretsService.onUpdate(SecretName.nordigen_secretKey, (newSecret) => {
-  goCardlessClient.secretKey = newSecret;
-});
+
+const getGocardlessClient = () =>
+  new GoCardlessClient({
+    secretId: secretsService.get(SecretName.nordigen_secretId),
+    secretKey: secretsService.get(SecretName.nordigen_secretKey),
+  });
 
 export const handleGoCardlessError = (response) => {
   switch (response.status_code) {
@@ -58,7 +53,9 @@ export const goCardlessService = {
    * @returns {boolean}
    */
   isConfigured: () => {
-    return !!(goCardlessClient.secretId && goCardlessClient.secretKey);
+    return !!(
+      getGocardlessClient().secretId && getGocardlessClient().secretKey
+    );
   },
 
   /**
@@ -76,7 +73,7 @@ export const goCardlessService = {
       return clockTimestamp >= payload.exp;
     };
 
-    if (isExpiredJwtToken(goCardlessClient.token)) {
+    if (isExpiredJwtToken(getGocardlessClient().token)) {
       // Generate new access token. Token is valid for 24 hours
       // Note: access_token is automatically injected to other requests after you successfully obtain it
       const tokenData = await client.generateToken();
@@ -479,25 +476,25 @@ export const goCardlessService = {
  */
 export const client = {
   getBalances: async (accountId) =>
-    await goCardlessClient.account(accountId).getBalances(),
+    await getGocardlessClient().account(accountId).getBalances(),
   getTransactions: async ({ accountId, dateFrom, dateTo }) =>
-    await goCardlessClient.account(accountId).getTransactions({
+    await getGocardlessClient().account(accountId).getTransactions({
       dateFrom,
       dateTo,
       country: undefined,
     }),
   getInstitutions: async (country) =>
-    await goCardlessClient.institution.getInstitutions({ country }),
+    await getGocardlessClient().institution.getInstitutions({ country }),
   getInstitutionById: async (institutionId) =>
-    await goCardlessClient.institution.getInstitutionById(institutionId),
+    await getGocardlessClient().institution.getInstitutionById(institutionId),
   getDetails: async (accountId) =>
-    await goCardlessClient.account(accountId).getDetails(),
+    await getGocardlessClient().account(accountId).getDetails(),
   getMetadata: async (accountId) =>
-    await goCardlessClient.account(accountId).getMetadata(),
+    await getGocardlessClient().account(accountId).getMetadata(),
   getRequisitionById: async (requisitionId) =>
-    await goCardlessClient.requisition.getRequisitionById(requisitionId),
+    await getGocardlessClient().requisition.getRequisitionById(requisitionId),
   deleteRequisition: async (requisitionId) =>
-    await goCardlessClient.requisition.deleteRequisition(requisitionId),
+    await getGocardlessClient().requisition.deleteRequisition(requisitionId),
   initSession: async ({
     redirectUrl,
     institutionId,
@@ -509,7 +506,7 @@ export const client = {
     redirectImmediate,
     accountSelection,
   }) =>
-    await goCardlessClient.initSession({
+    await getGocardlessClient().initSession({
       redirectUrl,
       institutionId,
       referenceId,
@@ -520,7 +517,7 @@ export const client = {
       redirectImmediate,
       accountSelection,
     }),
-  generateToken: async () => await goCardlessClient.generateToken(),
+  generateToken: async () => await getGocardlessClient().generateToken(),
   exchangeToken: async ({ refreshToken }) =>
-    await goCardlessClient.exchangeToken({ refreshToken }),
+    await getGocardlessClient().exchangeToken({ refreshToken }),
 };

src/app-sync.js

@@ -15,9 +15,6 @@ const app = express();
 app.use(errorMiddleware);
 export { app as handlers };
 
-// eslint-disable-next-line
-export async function init() {}
-
 // This is a version representing the internal format of sync
 // messages. When this changes, all sync files need to be reset. We
 // will check this version when syncing and notify the user if they

src/app.js

@@ -71,17 +71,6 @@ function parseHTTPSConfig(value) {
 }
 
 export default async function run() {
-  if (!fs.existsSync(config.serverFiles)) {
-    fs.mkdirSync(config.serverFiles);
-  }
-
-  if (!fs.existsSync(config.userFiles)) {
-    fs.mkdirSync(config.userFiles);
-  }
-
-  await accountApp.init();
-  await syncApp.init();
-
   if (config.https) {
     const https = await import('node:https');
     const httpsOptions = {

src/db.js

@@ -27,7 +27,7 @@ class WrappedDatabase {
    * @param {string} sql
    */
   exec(sql) {
-    this.db.exec(sql);
+    return this.db.exec(sql);
   }
 
   /**

src/migrations.js

@@ -0,0 +1,30 @@
+import migrate from 'migrate';
+import config from './load-config.js';
+
+export default function run(direction = 'up') {
+  console.log(
+    `Checking if there are any migrations to run for direction "${direction}"...`,
+  );
+
+  return new Promise((resolve) =>
+    migrate.load(
+      {
+        stateStore: `.migrate${config.mode === 'test' ? '-test' : ''}`,
+      },
+      (err, set) => {
+        if (err) {
+          throw err;
+        }
+
+        set[direction]((err) => {
+          if (err) {
+            throw err;
+          }
+
+          console.log('Migrations: DONE');
+          resolve();
+        });
+      },
+    ),
+  );
+}