From afa9c466a32737ca8bf4762cada24705313f76e0 Mon Sep 17 00:00:00 2001 From: Jamie Maynard <29251905+j-maynard@users.noreply.github.com> Date: Fri, 7 Jun 2024 14:24:27 +0100 Subject: [PATCH] Implement stories 40 and 242 from the user journey This implements sotries AB#40 AB#242. Provides the user with the ability to name a dataset and upload the inital fact table to the backend. --- jest.config.ts | 3 +- package-lock.json | 460 +++++++++++++++++- package.json | 1 + src/app.ts | 50 +- src/controllers/api.ts | 15 +- src/dtos/datafile-dto.ts | 7 + src/dtos/dataset-dto.ts | 33 ++ src/{models => dtos}/error.ts | 0 src/{models => dtos}/filelist.ts | 3 +- src/{models => dtos}/healthcehck.ts | 0 .../processedcsv-dto.ts} | 5 +- src/dtos/upload-dto.ts | 13 + src/dtos/view-dto.ts | 26 + src/resources/locales/en-GB.json | 10 + src/route/healthcheck.ts | 6 +- src/views/list.ejs | 2 +- src/views/publish/name.ejs | 46 ++ src/views/publish/start.ejs | 2 +- src/views/publish/upload.ejs | 40 +- test/.jest/setEnvVars.ts | 3 + test/app.test.ts | 212 +++++--- 21 files changed, 794 insertions(+), 143 deletions(-) create mode 100644 src/dtos/datafile-dto.ts create mode 100644 src/dtos/dataset-dto.ts rename src/{models => dtos}/error.ts (100%) rename src/{models => dtos}/filelist.ts (72%) rename src/{models => dtos}/healthcehck.ts (100%) rename src/{models/processedcsv.ts => dtos/processedcsv-dto.ts} (81%) create mode 100644 src/dtos/upload-dto.ts create mode 100644 src/dtos/view-dto.ts create mode 100644 src/views/publish/name.ejs create mode 100644 test/.jest/setEnvVars.ts diff --git a/jest.config.ts b/jest.config.ts index 9dafe53..d6cc55d 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -8,7 +8,8 @@ const config: Config = { coverageDirectory: './coverage', collectCoverage: true, coverageReporters: ['cobertura', 'lcov', 'html', 'text'], - coveragePathIgnorePatterns: ['/node_modules/', '/test/', '/src/controllers/datalake.ts'] + coveragePathIgnorePatterns: ['/node_modules/', '/test/', '/src/controllers/datalake.ts'], + setupFiles: ['/test/.jest/setEnvVars.ts'] }; export default config; diff --git a/package-lock.json b/package-lock.json index 0509695..5e509ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "jest": "^29.7.0", "jest-junit": "^16.0.0", "jest-junit-reporter": "^1.1.0", + "msw": "^2.3.1", "nodemon": "^3.1.0", "npm-run-all": "^4.1.5", "prettier": "^3.2.5", @@ -598,6 +599,33 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@bundled-es-modules/cookie": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.0.tgz", + "integrity": "sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==", + "dev": true, + "dependencies": { + "cookie": "^0.5.0" + } + }, + "node_modules/@bundled-es-modules/cookie/node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/@bundled-es-modules/statuses": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz", + "integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==", + "dev": true, + "dependencies": { + "statuses": "^2.0.1" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -792,6 +820,157 @@ "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", "dev": true }, + "node_modules/@inquirer/confirm": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.9.tgz", + "integrity": "sha512-UF09aejxCi4Xqm6N/jJAiFXArXfi9al52AFaSD+2uIHnhZGtd1d6lIGTRMPouVSJxbGEi+HkOWSYaiEY/+szUw==", + "dev": true, + "dependencies": { + "@inquirer/core": "^8.2.2", + "@inquirer/type": "^1.3.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/core": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-8.2.2.tgz", + "integrity": "sha512-K8SuNX45jEFlX3EBJpu9B+S2TISzMPGXZIuJ9ME924SqbdW6Pt6fIkKvXg7mOEOKJ4WxpQsxj0UTfcL/A434Ww==", + "dev": true, + "dependencies": { + "@inquirer/figures": "^1.0.3", + "@inquirer/type": "^1.3.3", + "@types/mute-stream": "^0.0.4", + "@types/node": "^20.12.13", + "@types/wrap-ansi": "^3.0.0", + "ansi-escapes": "^4.3.2", + "chalk": "^4.1.2", + "cli-spinners": "^2.9.2", + "cli-width": "^4.1.0", + "mute-stream": "^1.0.0", + "signal-exit": "^4.1.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^6.2.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/core/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@inquirer/core/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@inquirer/core/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/@inquirer/core/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/@inquirer/core/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@inquirer/core/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/core/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.3.tgz", + "integrity": "sha512-ErXXzENMH5pJt5/ssXV0DfWUZqly8nGzf0UcBV9xTnP+KyffE2mqyxIMBrZ8ijQck2nU0TQm40EQB53YreyWHw==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/type": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.3.3.tgz", + "integrity": "sha512-xTUt0NulylX27/zMx04ZYar/kr1raaiFTVvQ5feljQsiAgdm0WPj4S73/ye0fbslh+15QrIuDvfCXTek7pMY5A==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1576,6 +1755,32 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mswjs/cookies": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@mswjs/cookies/-/cookies-1.1.0.tgz", + "integrity": "sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@mswjs/interceptors": { + "version": "0.29.1", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.29.1.tgz", + "integrity": "sha512-3rDakgJZ77+RiQUuSK69t1F0m8BQKA8Vh5DCS5V0DWvNY67zob2JhhQrhCO0AKLGINTRSFd1tBaHcJTkhefoSw==", + "dev": true, + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.2.1", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz", @@ -1620,6 +1825,28 @@ "node": ">= 8" } }, + "node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "dev": true + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dev": true, + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "dev": true + }, "node_modules/@pkgr/core": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", @@ -2030,6 +2257,12 @@ "@types/node": "*" } }, + "node_modules/@types/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", + "dev": true + }, "node_modules/@types/cookiejar": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", @@ -2167,10 +2400,19 @@ "@types/express": "*" } }, + "node_modules/@types/mute-stream": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { - "version": "20.12.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.2.tgz", - "integrity": "sha512-zQ0NYO87hyN6Xrclcqp7f8ZbXNbRfoGWNcMvHTPQp9UUrwI0mI7XBz+cu7/W6/VClYo2g63B0cjull/srU7LgQ==", + "version": "20.14.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.2.tgz", + "integrity": "sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==", "dev": true, "dependencies": { "undici-types": "~5.26.4" @@ -2241,6 +2483,12 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, + "node_modules/@types/statuses": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.5.tgz", + "integrity": "sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==", + "dev": true + }, "node_modules/@types/superagent": { "version": "8.1.6", "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.6.tgz", @@ -2262,6 +2510,12 @@ "@types/superagent": "^8.1.0" } }, + "node_modules/@types/wrap-ansi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==", + "dev": true + }, "node_modules/@types/yargs": { "version": "17.0.32", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.32.tgz", @@ -3517,6 +3771,27 @@ "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", "dev": true }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "dev": true, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -4041,9 +4316,9 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/ejs": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", - "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "dependencies": { "jake": "^10.8.5" }, @@ -5864,6 +6139,15 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/graphql": { + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", + "dev": true, + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, "node_modules/has-bigints": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", @@ -5951,6 +6235,12 @@ "tslib": "^2.0.3" } }, + "node_modules/headers-polyfill": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", + "integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==", + "dev": true + }, "node_modules/hexoid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", @@ -6448,6 +6738,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "dev": true + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -8925,6 +9221,137 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/msw": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.3.1.tgz", + "integrity": "sha512-ocgvBCLn/5l3jpl1lssIb3cniuACJLoOfZu01e3n5dbJrpA5PeeWn28jCLgQDNt6d7QT8tF2fYRzm9JoEHtiig==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "@bundled-es-modules/cookie": "^2.0.0", + "@bundled-es-modules/statuses": "^1.0.1", + "@inquirer/confirm": "^3.0.0", + "@mswjs/cookies": "^1.1.0", + "@mswjs/interceptors": "^0.29.0", + "@open-draft/until": "^2.1.0", + "@types/cookie": "^0.6.0", + "@types/statuses": "^2.0.4", + "chalk": "^4.1.2", + "graphql": "^16.8.1", + "headers-polyfill": "^4.0.2", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.2", + "path-to-regexp": "^6.2.0", + "strict-event-emitter": "^0.5.1", + "type-fest": "^4.9.0", + "yargs": "^17.7.2" + }, + "bin": { + "msw": "cli/index.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/mswjs" + }, + "peerDependencies": { + "typescript": ">= 4.7.x" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/msw/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/msw/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/msw/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/msw/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/msw/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/msw/node_modules/path-to-regexp": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", + "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==", + "dev": true + }, + "node_modules/msw/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/msw/node_modules/type-fest": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.19.0.tgz", + "integrity": "sha512-CN2l+hWACRiejlnr68vY0/7734Kzu+9+TOslUXbSCQ1ruY9XIHDBSceVXCcHm/oXrdzhtLMMdJEKfemf1yXiZQ==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/multer": { "version": "1.4.5-lts.1", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.1.tgz", @@ -8953,6 +9380,15 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mute-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "dev": true, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -9432,6 +9868,12 @@ "node": ">= 0.8.0" } }, + "node_modules/outvariant": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.2.tgz", + "integrity": "sha512-Ou3dJ6bA/UJ5GVHxah4LnqDwZRwAmWxrG3wtrHrbGnP4RnLCtA64A4F+ae7Y8ww660JaddSoArUR5HjipWSHAQ==", + "dev": true + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -10622,6 +11064,12 @@ "node": ">=10.0.0" } }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", + "dev": true + }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", diff --git a/package.json b/package.json index a1bbe49..6f494ba 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "jest": "^29.7.0", "jest-junit": "^16.0.0", "jest-junit-reporter": "^1.1.0", + "msw": "^2.3.1", "nodemon": "^3.1.0", "npm-run-all": "^4.1.5", "prettier": "^3.2.5", diff --git a/src/app.ts b/src/app.ts index 1bbce13..be4d706 100644 --- a/src/app.ts +++ b/src/app.ts @@ -10,7 +10,7 @@ import multer from 'multer'; import { API } from './controllers/api'; import { healthcheck } from './route/healthcheck'; -import { FileList } from './models/filelist'; +import { FileList } from './dtos/filelist'; i18next .use(Backend) @@ -67,66 +67,74 @@ app.get('/:lang/publish', (req: Request, res: Response) => { res.render('publish/start'); }); -app.get('/:lang/publish/upload', (req: Request, res: Response) => { - res.render('publish/upload'); +app.get('/:lang/publish/name', (req: Request, res: Response) => { + res.render('publish/name'); }); -app.post('/:lang/publish/upload', upload.single('csv'), async (req: Request, res: Response) => { - const lang = req.params.lang; - logger.debug(`Filename is ${req.body?.filename}`); - if (!req.file) { +app.post('/:lang/publish/name', upload.none(), (req: Request, res: Response) => { + if (!req.body?.internal_name) { + logger.debug('Internal name was missing on request'); res.status(400); - res.render('publish/upload', { + res.render('publish/name', { success: false, headers: undefined, data: undefined, errors: [ { - field: 'csv', - message: 'No CSV data available' + field: 'internal_name', + message: 'No dataset name provided' } ] }); return; } - if (!req.body?.filename) { + const internalName: string = req.body.internal_name; + res.render('publish/upload', { internal_name: internalName }); +}); + +app.post('/:lang/publish/upload', upload.single('csv'), async (req: Request, res: Response) => { + const lang = req.params.lang; + if (!req.body?.internal_name) { + logger.debug('Internal name was missing on request'); res.status(400); - res.render('publish/upload', { + res.render('publish/name', { success: false, headers: undefined, data: undefined, errors: [ { - field: 'filename', - message: 'No datasetname provided' + field: 'internal_name', + message: 'No dataset name provided' } ] }); return; } - if (!req.body?.description) { + logger.debug(`Internal name: ${req.body.internal_name}`); + const internalName: string = req.body.internal_name; + if (!req.file) { + logger.debug('Attached file was missing on this request'); res.status(400); res.render('publish/upload', { success: false, headers: undefined, data: undefined, + internal_name: internalName, errors: [ { - field: 'description', - message: 'No datasetname provided' + field: 'csv', + message: 'No CSV data available' } ] }); return; } - const name: string = req.body?.filename; - const description: string = req.body?.description; const fileData = new Blob([req.file?.buffer]); - const processedCSV = await APIInstance.uploadCSV(lang, fileData, name, description); + const processedCSV = await APIInstance.uploadCSV(lang, fileData, internalName); if (processedCSV.success) { - res.redirect(`/${req.i18n.language}/data/?file=${processedCSV.datafile_id}`); + res.redirect(`/${req.i18n.language}/data/?file=${processedCSV.dataset?.id}`); } else { res.status(400); res.render('publish/upload', processedCSV); diff --git a/src/controllers/api.ts b/src/controllers/api.ts index d362a32..ee60625 100644 --- a/src/controllers/api.ts +++ b/src/controllers/api.ts @@ -2,9 +2,9 @@ import { env } from 'process'; import { Logger } from 'pino'; -import { FileList } from '../models/filelist'; -import { ProcessedCSV } from '../models/processedcsv'; -import { Healthcheck } from '../models/healthcehck'; +import { FileList } from '../dtos/filelist'; +import { ProcessedCSV } from '../dtos/processedcsv-dto'; +import { Healthcheck } from '../dtos/healthcehck'; export class API { private readonly backend_server: string; @@ -45,11 +45,10 @@ export class API { return file; } - public async uploadCSV(lang: string, file: Blob, filename: string, description: string) { + public async uploadCSV(lang: string, file: Blob, filename: string) { const formData = new FormData(); formData.append('csv', file, filename); - formData.append('filename', filename); - formData.append('description', description); + formData.append('internal_name', filename); const processedCSV: ProcessedCSV = await fetch( `${this.backend_protocol}://${this.backend_server}:${this.backend_port}/${lang}/dataset/`, @@ -58,10 +57,6 @@ export class API { body: formData } ) - .then((api_res) => { - this.logger.debug(api_res); - return api_res; - }) .then((api_res) => api_res.json()) .then((api_res) => { return api_res as ProcessedCSV; diff --git a/src/dtos/datafile-dto.ts b/src/dtos/datafile-dto.ts new file mode 100644 index 0000000..68dc31f --- /dev/null +++ b/src/dtos/datafile-dto.ts @@ -0,0 +1,7 @@ +export interface DatafileDTO { + id: string; + sha256hash: string; + created_by: string; + creation_date: Date; + csv_link: string; +} diff --git a/src/dtos/dataset-dto.ts b/src/dtos/dataset-dto.ts new file mode 100644 index 0000000..fe6ef59 --- /dev/null +++ b/src/dtos/dataset-dto.ts @@ -0,0 +1,33 @@ +export interface DatafileDTO { + id: string; + sha256hash: string; + created_by: string; + creation_date: string; +} + +export interface DatasetDescriptionDTO { + description: string; + language: string; +} + +export interface DatasetTitleDTO { + title: string; + language: string; +} + +export interface DatasetDTO { + id: string; + code: string; + internal_name: string; + title: DatasetTitleDTO[]; + description: DatasetDescriptionDTO[]; + creation_date: string; + created_by: string; + modification_date: string; + modified_by: string; + live: boolean; + datafiles: DatafileDTO[]; + csv_link: string; + xslx_link: string; + view_link: string; +} diff --git a/src/models/error.ts b/src/dtos/error.ts similarity index 100% rename from src/models/error.ts rename to src/dtos/error.ts diff --git a/src/models/filelist.ts b/src/dtos/filelist.ts similarity index 72% rename from src/models/filelist.ts rename to src/dtos/filelist.ts index 35c1b41..522c4ba 100644 --- a/src/models/filelist.ts +++ b/src/dtos/filelist.ts @@ -1,7 +1,6 @@ export interface FileDescription { id: string; - name: string; - description: string; + internal_name: string; } export interface FileList { diff --git a/src/models/healthcehck.ts b/src/dtos/healthcehck.ts similarity index 100% rename from src/models/healthcehck.ts rename to src/dtos/healthcehck.ts diff --git a/src/models/processedcsv.ts b/src/dtos/processedcsv-dto.ts similarity index 81% rename from src/models/processedcsv.ts rename to src/dtos/processedcsv-dto.ts index 39da69c..6000f2b 100644 --- a/src/models/processedcsv.ts +++ b/src/dtos/processedcsv-dto.ts @@ -1,3 +1,4 @@ +import { DatasetDTO } from './dataset-dto'; import { Error } from './error'; export interface PageInfo { @@ -8,9 +9,7 @@ export interface PageInfo { export interface ProcessedCSV { success: boolean; - datafile_id: string | undefined; - datafile_name: string | undefined; - datafile_description: string | undefined; + dataset: DatasetDTO | undefined; current_page: number | undefined; page_info: PageInfo | undefined; pages: Array | undefined; diff --git a/src/dtos/upload-dto.ts b/src/dtos/upload-dto.ts new file mode 100644 index 0000000..0ef4e4c --- /dev/null +++ b/src/dtos/upload-dto.ts @@ -0,0 +1,13 @@ +import { Error } from './error'; +import { DatasetDTO } from './dataset-dto'; + +export interface UploadDTO { + success: true; + dataset: DatasetDTO; +} + +export interface UploadErrDTO { + success: boolean; + dataset: DatasetDTO; + errors: Error[]; +} diff --git a/src/dtos/view-dto.ts b/src/dtos/view-dto.ts new file mode 100644 index 0000000..24ef285 --- /dev/null +++ b/src/dtos/view-dto.ts @@ -0,0 +1,26 @@ +import { Error } from './error'; +import { DatasetDTO } from './dataset-dto'; + +export interface PageInfo { + total_records: number | undefined; + start_record: number | undefined; + end_record: number | undefined; +} + +export interface ViewErrDTO { + success: boolean; + errors: Error[]; + dataset_id: string; +} + +export interface ViewDTO { + success: boolean; + dataset: DatasetDTO; + current_page: number; + page_info: PageInfo; + pages: Array; + page_size: number; + total_pages: number; + headers: Array | undefined; + data: Array>; +} diff --git a/src/resources/locales/en-GB.json b/src/resources/locales/en-GB.json index 2c1746d..51cb6b3 100644 --- a/src/resources/locales/en-GB.json +++ b/src/resources/locales/en-GB.json @@ -34,6 +34,15 @@ "data-table": "a data table containing data and references to all relevant dimensions", "lookup-table": "lookup tables for all relevant dimensions - using either common tables within this service or uploading them yourself", "metadata": "details about the dataset including description, quality and how the data was collected" + }, + "name": { + "title": "Name the dataset", + "description": "This name is for internal use only to identify this dataset. You'll be asked to add the title that will appear on the StatsWales website with the other mettadata later.", + "internal_name": "Internal name" + }, + "upload": { + "title": "Upload the data table", + "note": "The file must be in CSV format" } }, "upload": { @@ -82,6 +91,7 @@ "data": "data", "publish": { "start": "publish", + "name": "name", "upload": "upload" } } diff --git a/src/route/healthcheck.ts b/src/route/healthcheck.ts index bb04529..5dd8aad 100644 --- a/src/route/healthcheck.ts +++ b/src/route/healthcheck.ts @@ -12,16 +12,16 @@ const APIInstance = new API(logger); export const healthcheck = Router(); -healthcheck.get('/', (req, res) => { +healthcheck.get('/', async (req, res) => { const lang = req.i18n.language || 'en-GB'; logger.info(`Healthcheck requested in ${lang}`); const statusMsg = req.t('app-running'); - + const beConnected = await APIInstance.ping(); res.json({ status: statusMsg, notes: req.t('health-notes'), services: { - backend_connected: APIInstance.ping() + backend_connected: beConnected } }); }); diff --git a/src/views/list.ejs b/src/views/list.ejs index 7b646bc..33508bd 100644 --- a/src/views/list.ejs +++ b/src/views/list.ejs @@ -6,7 +6,7 @@

<%= t('list.title') %>

diff --git a/src/views/publish/name.ejs b/src/views/publish/name.ejs new file mode 100644 index 0000000..6e6df0e --- /dev/null +++ b/src/views/publish/name.ejs @@ -0,0 +1,46 @@ +<%- include("../partials/top"); %> + +
+ +
+

<%= t('publish.name.title') %>

+ + <% if (locals?.errors) { %> +
+
+

+ There is a problem +

+
+ +
+
+
+ <% } %> + +

<%= t('publish.name.description') %>

+
+
+

+ +

+ +
+ +
+
+
+ +<%- include("../partials/bottom"); %> diff --git a/src/views/publish/start.ejs b/src/views/publish/start.ejs index c70b61b..ab959a6 100644 --- a/src/views/publish/start.ejs +++ b/src/views/publish/start.ejs @@ -11,7 +11,7 @@
  • <%= t('publish.start.metadata') %>
    • - <%= t('buttons.continue') %> + <%= t('buttons.continue') %> <%= t('buttons.cancel') %>
    diff --git a/src/views/publish/upload.ejs b/src/views/publish/upload.ejs index 06fb75e..e8fb0b5 100644 --- a/src/views/publish/upload.ejs +++ b/src/views/publish/upload.ejs @@ -3,22 +3,7 @@
    -

    <%= t('upload.title') %>

    - <% if (locals?.success) { %> - - <% } %> +

    <%= t('publish.upload.title') %>

    <% if (locals?.errors) { %>
    @@ -38,29 +23,14 @@
    <% } %> -
    -
    -

    - -

    - -
    -
    -

    - -

    - -
    +
    -
    +