all tests passing

.env-example

@@ -21,6 +21,13 @@ DB_PASSWORD=postgres
 DB_DATABASE=statswales-backend
 DB_SSL=false
 
+# test database - port must be different from app database if both are run locally
+TEST_DB_HOST=localhost
+TEST_DB_PORT=5433
+TEST_DB_USERNAME=postgres
+TEST_DB_PASSWORD=postgres
+TEST_DB_DATABASE=statswales-backend-test
+
 # storage providers
 AZURE_DATALAKE_STORAGE_ACCOUNT_NAME=
 AZURE_DATALAKE_STORAGE_ACCOUNT_KEY=
@@ -31,6 +38,8 @@ AZURE_BLOB_STORAGE_ACCOUNT_KEY=
 AZURE_BLOB_STORAGE_CONTAINER_NAME=
 
 # authentication providers
+AUTH_PROVIDERS=google,onelogin
+
 ONELOGIN_URL=https://oidc.integration.account.gov.uk
 ONELOGIN_CLIENT_ID=
 ONELOGIN_CLIENT_SECRET=

.eslintrc

@@ -23,7 +23,8 @@
     "no-console": 0,
     "no-process-env": 0,
     "no-inline-comments": 0,
-    "line-comment-position": 0
+    "line-comment-position": 0,
+    "no-warning-comments": 1
   },
   "globals": {
     "NodeJS": true

.github/workflows/node-test.yml

@@ -19,6 +19,22 @@ jobs:
         node-version: [20.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
+    services:
+      # postgres config should match the env vars set in test/helpers/jest-setup.ts
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: statswales-backend-test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5433:5432
+
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

README.md

@@ -70,7 +70,7 @@ npm run migration:generate -- ./src/migration/<name>
 
 e.g.
 ```bash
-npm run migration:generate -- ./src/migration/create_user_table
+npm run migration:generate -- ./src/migration/initial-schema
 ```
 
 ## Deploying the service

docker-compose.yml

@@ -1,5 +1,5 @@
 services:
-  postgres:
+  db-dev:
     image: postgres:latest
     ports:
       - "5432:5432"
@@ -9,3 +9,12 @@ services:
       POSTGRES_DB: statswales-backend
     volumes:
       - .docker/postgres/data:/var/lib/postgresql/data
+
+  db-test:
+    image: postgres:latest
+    ports:
+      - "5433:5432"
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: statswales-backend-test

jest.config.ts

@@ -8,7 +8,9 @@ const config: Config = {
   coverageDirectory: './coverage',
   collectCoverage: true,
   coverageReporters: ['cobertura', 'lcov', 'html', 'text'],
-  coveragePathIgnorePatterns: ['/node_modules/', '/test/', '/src/controllers/datalake.ts']
+  coveragePathIgnorePatterns: ['/node_modules/', '/test/', '/src/controllers/datalake.ts'],
+  setupFiles: ['<rootDir>/test/helpers/jest-setup.ts'],
+  maxWorkers: 1 // TODO: temporary solution to test parallelism issue
 };
 
 export default config;

package.json

@@ -12,19 +12,20 @@
     "lint:ci": "eslint . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
     "build": "npm-run-all clean tsc copy-assets",
-    "test": "jest --coverage",
-    "test:ci": "jest --ci --coverage --config=jest.config.ts",
+    "pretest": "docker compose up -d db-test",
+    "test": "NODE_ENV=test jest --coverage",
+    "test:ci": "NODE_ENV=test jest --ci --coverage --config=jest.config.ts",
     "check": "npm-run-all prettier:fix lint:fix test build",
-    "predev": "docker compose up -d",
+    "predev": "docker compose up -d db-dev",
     "dev:check": "npm-run-all check dev",
     "dev": "NODE_ENV=dev nodemon --watch src -e ts,ejs --exec npm run start",
     "start": "npm run build && node dist/server.js | pino-colada",
     "start:container": "node dist/server.js",
     "typeorm": "typeorm-ts-node-commonjs",
-    "migration:show": "npm run typeorm migration:show -- --dataSource=./src/data-source.ts",
-    "migration:run": "npm run typeorm migration:run -- --dataSource=./src/data-source.ts",
-    "migration:revert": "npm run typeorm migration:revert -- --dataSource=./src/data-source.ts",
-    "migration:generate": "npm run typeorm migration:generate -- --dataSource=./src/data-source.ts"
+    "migration:show": "npm run typeorm migration:show -- --dataSource=./src/db/data-source.ts",
+    "migration:run": "npm run typeorm migration:run -- --dataSource=./src/db/data-source.ts",
+    "migration:revert": "npm run typeorm migration:revert -- --dataSource=./src/db/data-source.ts",
+    "migration:generate": "npm run typeorm migration:generate -- --dataSource=./src/db/data-source.ts"
   },
   "keywords": [],
   "author": "",

src/app.ts

@@ -4,28 +4,22 @@ import express, { Application, Request, Response } from 'express';
 import passport from 'passport';
 
 import { logger, httpLogger } from './utils/logger';
-import { dataSource } from './data-source';
-import DatabaseManager from './database-manager';
+import DatabaseManager from './db/database-manager';
 import { i18next, i18nextMiddleware } from './middleware/translation';
 import { initPassport } from './middleware/passport-auth';
 import { rateLimiter } from './middleware/rate-limiter';
-import { apiRoute as datasetRoutes } from './route/dataset-route';
-import { healthcheck as healthCheckRoutes } from './route/healthcheck';
-import { test as testRoutes } from './route/test';
-import { auth as authRoutes } from './route/auth';
+import { authRouter } from './route/auth';
+import { healthcheckRouter } from './route/healthcheck';
+import { datasetRouter } from './route/dataset';
 import session from './middleware/session';
 
-// eslint-disable-next-line import/no-mutable-exports
-export let dbManager: DatabaseManager;
-
-const connectToDb = async () => {
-    dbManager = new DatabaseManager(dataSource, logger);
+export const initDb = async (): Promise<DatabaseManager> => {
+    const dbManager = new DatabaseManager(logger);
     await dbManager.initializeDataSource();
     await initPassport(dbManager.getDataSource().getRepository('User'));
+    return dbManager;
 };
 
-connectToDb();
-
 const app: Application = express();
 
 app.disable('x-powered-by');
@@ -35,19 +29,17 @@ app.use(i18nextMiddleware.handle(i18next));
 app.use(express.json());
 app.use(session);
 
-app.use('/auth', rateLimiter, authRoutes);
-app.use('/test', rateLimiter, testRoutes);
-app.use('/healthcheck', rateLimiter, healthCheckRoutes);
-app.use('/:lang/dataset', rateLimiter, passport.authenticate('jwt'), datasetRoutes);
-app.use('/:lang/healthcheck', rateLimiter, healthCheckRoutes);
+app.use('/auth', rateLimiter, authRouter);
+app.use('/healthcheck', rateLimiter, healthcheckRouter);
+app.use('/:lang/dataset', rateLimiter, passport.authenticate('jwt', { session: false }), datasetRouter);
 
 app.get('/', (req: Request, res: Response) => {
     const lang = req.headers['accept-language'] || req.headers['Accept-Language'] || req.i18n.language || 'en-GB';
     if (lang.includes('cy')) {
         res.redirect('/cy-GB/api');
-    } else {
-        res.redirect('/en-GB/api');
+        return;
     }
+    res.redirect('/en-GB/api');
 });
 
 export default app;

src/controllers/auth.ts

@@ -3,20 +3,15 @@ import { URL } from 'node:url';
 import { RequestHandler } from 'express';
 import passport from 'passport';
 import jwt from 'jsonwebtoken';
-import { pick } from 'lodash';
 
 import { logger } from '../utils/logger';
-import { User } from '../entity/user';
+import { User } from '../entities/user';
+import { sanitiseUser } from '../utils/sanitise-user';
 
 const returnURL = `${process.env.FRONTEND_URL}/auth/callback`;
 const domain = new URL(process.env.BACKEND_URL || '').hostname;
 const DEFAULT_TOKEN_EXPIRY = '1d';
 
-// strip anything from the user object that we do not want to expose to the frontend app
-const sanitiseUser = (user: User): Partial<User> => {
-    return pick(user, ['id', 'email', 'givenName', 'familyName']);
-};
-
 export const loginGoogle: RequestHandler = (req, res, next) => {
     logger.debug('attempting to authenticate with google...');
 

src/controllers/csv-processor.ts

@@ -127,7 +127,7 @@ export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: str
     }
     const importRecord = new Import();
     importRecord.id = randomUUID();
-    importRecord.mime_type = filetype;
+    importRecord.mimeType = filetype;
     const extension = filetype === 'text/csv' ? 'csv' : 'zip';
     importRecord.filename = `${importRecord.id}.${extension}`;
     try {
@@ -141,7 +141,7 @@ export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: str
         await blobStorageService.uploadFile(`${importRecord.filename}`, fileStream);
         const resolvedHash = await promisedHash;
         if (resolvedHash) importRecord.hash = resolvedHash;
-        importRecord.uploaded_at = new Date(Date.now());
+        importRecord.uploadedAt = new Date(Date.now());
         importRecord.type = 'Draft';
         importRecord.location = 'BlobStorage';
         return importRecord;

src/db/data-source.ts

@@ -0,0 +1,37 @@
+import 'dotenv/config';
+import 'reflect-metadata';
+import { DataSource, DataSourceOptions } from 'typeorm';
+
+const { NODE_ENV, DB_HOST, DB_PORT, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_SSL } = process.env;
+
+const dataSourceOpts: DataSourceOptions = {
+    type: 'postgres',
+    host: DB_HOST,
+    port: parseInt(DB_PORT || '5432', 10),
+    username: DB_USERNAME,
+    password: DB_PASSWORD,
+    database: DB_DATABASE,
+    ssl: DB_SSL !== 'false' ?? true,
+    synchronize: false,
+    logging: false,
+    entities: [`${__dirname}/../entities/*.{ts,js}`],
+    migrations: [`${__dirname}/../migrations/*.{ts,js}`]
+};
+
+const { TEST_DB_HOST, TEST_DB_PORT, TEST_DB_USERNAME, TEST_DB_PASSWORD, TEST_DB_DATABASE } = process.env;
+
+const testDataSourceOpts: DataSourceOptions = {
+    type: 'postgres',
+    host: TEST_DB_HOST,
+    port: parseInt(TEST_DB_PORT || '5432', 10),
+    username: TEST_DB_USERNAME,
+    password: TEST_DB_PASSWORD,
+    database: TEST_DB_DATABASE,
+    ssl: false,
+    synchronize: true, // auto-sync test db schema
+    logging: false,
+    entities: [`${__dirname}/../entities/*.{ts,js}`],
+    migrations: [`${__dirname}/../migrations/*.{ts,js}`]
+};
+
+export const dataSource = new DataSource(NODE_ENV === 'test' ? testDataSourceOpts : dataSourceOpts);

src/database-manager.ts → src/db/database-manager.ts

@@ -3,14 +3,16 @@ import 'reflect-metadata';
 import { Logger } from 'pino';
 import { DataSource, EntityManager } from 'typeorm';
 
+import { dataSource } from './data-source';
+
 class DatabaseManager {
     private entityManager: EntityManager;
     private dataSource: DataSource;
     private logger: Logger;
 
-    constructor(dataSource: DataSource, logger: Logger) {
-        this.dataSource = dataSource;
+    constructor(logger: Logger) {
         this.logger = logger;
+        this.dataSource = dataSource;
     }
 
     getDataSource() {
@@ -25,13 +27,13 @@ class DatabaseManager {
     }
 
     async initializeDataSource() {
+        this.logger.debug(`DB '${this.dataSource.options.database}' initializing...`);
+
         if (!this.dataSource.isInitialized) {
-            await this.dataSource
-                .initialize()
-                .then(() => this.logger.info('Data source initialized'))
-                .catch((error) => this.logger.error(error));
+            await this.dataSource.initialize().catch((error) => this.logger.error(error));
         }
 
+        this.logger.info(`DB '${this.dataSource.options.database}' initialized`);
         this.entityManager = this.dataSource.createEntityManager();
     }
 }