From 5e1940c88a8b001cf3e890774599ba222d9e2fc4 Mon Sep 17 00:00:00 2001 From: Jamie Maynard <29251905+j-maynard@users.noreply.github.com> Date: Fri, 17 May 2024 11:12:00 +0100 Subject: [PATCH] Changed the API route to Dataset, improved the API As part of improving the backend api layer it was time to rename CSV to dataset and remove the redundant API route as the backend service no longer provides any views just API endpoints. --- src/app.ts | 5 +-- src/models/datafile-dto.ts | 9 +++++ src/route/{api.ts => dataset.ts} | 52 +++++++++++++++++++++---- test/{api.test.ts => dataset.test.ts} | 56 +++++++++++++++++++-------- 4 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 src/models/datafile-dto.ts rename src/route/{api.ts => dataset.ts} (70%) rename test/{api.test.ts => dataset.test.ts} (78%) diff --git a/src/app.ts b/src/app.ts index c32b29b..2b003fe 100644 --- a/src/app.ts +++ b/src/app.ts @@ -7,7 +7,7 @@ import Backend from 'i18next-fs-backend'; import i18nextMiddleware from 'i18next-http-middleware'; import { DataSourceOptions } from 'typeorm'; -import { apiRoute } from './route/api'; +import { apiRoute } from './route/dataset'; import { healthcheck } from './route/healthcheck'; import DatabaseManager from './database-manager'; @@ -45,10 +45,9 @@ export const logger = pino({ }); app.use(i18nextMiddleware.handle(i18next)); -app.use('/:lang/api', apiRoute); +app.use('/:lang/dataset', apiRoute); app.use('/:lang/healthcheck', healthcheck); app.use('/healthcheck', healthcheck); -app.use('/public', express.static(`${__dirname}/public`)); app.get('/', (req: Request, res: Response) => { const lang = req.headers['accept-language'] || req.headers['Accept-Language'] || req.i18n.language || 'en-GB'; diff --git a/src/models/datafile-dto.ts b/src/models/datafile-dto.ts new file mode 100644 index 0000000..8c3f3d6 --- /dev/null +++ b/src/models/datafile-dto.ts @@ -0,0 +1,9 @@ +export interface DatafileDTO { + id: string; + name: string; + description: string; + creation_date: Date; + csv_link: string; + xslx_link: string; + view_link: string; +} diff --git a/src/route/api.ts b/src/route/dataset.ts similarity index 70% rename from src/route/api.ts rename to src/route/dataset.ts index a37873b..8131c5a 100644 --- a/src/route/api.ts +++ b/src/route/dataset.ts @@ -6,6 +6,7 @@ import { processCSV, uploadCSV, DEFAULT_PAGE_SIZE } from '../controllers/csv-pro import { DataLakeService } from '../controllers/datalake'; import { Datafile } from '../entity/Datafile'; import { FileDescription } from '../models/filelist'; +import { DatafileDTO } from '../models/datafile-dto'; const storage = multer.memoryStorage(); const upload = multer({ storage }); @@ -17,11 +18,7 @@ export const logger = pino({ export const apiRoute = Router(); -apiRoute.get('/', (req, res) => { - res.json({ message: req.t('api.available') }); -}); - -apiRoute.post('/csv', upload.single('csv'), async (req: Request, res: Response) => { +apiRoute.post('/', upload.single('csv'), async (req: Request, res: Response) => { if (!req.file) { res.status(400); res.json({ @@ -78,7 +75,7 @@ apiRoute.post('/csv', upload.single('csv'), async (req: Request, res: Response) res.json(processedCSV); }); -apiRoute.get('/csv/', async (req, res) => { +apiRoute.get('/', async (req, res) => { const datafiles = await Datafile.find(); const fileList: FileDescription[] = []; for (const datafile of datafiles) { @@ -91,7 +88,32 @@ apiRoute.get('/csv/', async (req, res) => { res.json({ filelist: fileList }); }); -apiRoute.get('/csv/:file', async (req, res) => { +apiRoute.get('/:file', async (req, res) => { + const fileId = req.params.file; + if (fileId === undefined || fileId === null) { + res.status(404); + res.json({ message: 'File not found... file is null or undefined' }); + return; + } + const datafile = await Datafile.findOneBy({ id: fileId }); + if (datafile === undefined || datafile === null) { + res.status(404); + res.json({ message: 'File not found... file ID not found in Database' }); + return; + } + const datafileDTO: DatafileDTO = { + id: datafile.id, + name: datafile.name, + description: datafile.description, + creation_date: datafile.creationDate, + csv_link: `/datafile/${fileId}/csv`, + xslx_link: `/datafile/${fileId}/xlsx`, + view_link: `/datafile/${fileId}/view` + }; + res.json(datafileDTO); +}); + +apiRoute.get('/:file/csv', async (req, res) => { const dataLakeService = new DataLakeService(); const filename = req.params.file; const file = await dataLakeService.downloadFile(`${filename}.csv`); @@ -107,7 +129,21 @@ apiRoute.get('/csv/:file', async (req, res) => { res.end(); }); -apiRoute.get('/csv/:file/view', async (req, res) => { +apiRoute.get('/:file/xlsx', async (req, res) => { + const dataLakeService = new DataLakeService(); + const filename = req.params.file; + const file = await dataLakeService.downloadFile(`${filename}.csv`); + if (file === undefined || file === null) { + res.status(404); + res.json({ message: 'File not found... file is null or undefined' }); + return; + } + res.json({ + message: 'Not implmented yet' + }); +}); + +apiRoute.get('/:file/view', async (req, res) => { const dataLakeService = new DataLakeService(); const filename = req.params.file; const file = await dataLakeService.downloadFile(`${filename}.csv`); diff --git a/test/api.test.ts b/test/dataset.test.ts similarity index 78% rename from test/api.test.ts rename to test/dataset.test.ts index 1be4f9d..02bdca5 100644 --- a/test/api.test.ts +++ b/test/dataset.test.ts @@ -31,14 +31,8 @@ beforeAll(async () => { }); describe('API Endpoints', () => { - test('Inital API endpoint works', async () => { - const res = await request(app).get('/en-GB/api/'); - expect(res.status).toBe(200); - expect(res.body).toEqual({ message: 'api.available' }); - }); - test('Upload returns 400 if no file attached', async () => { - const res = await request(app).post('/en-GB/api/csv').query({ filename: 'test-data-1.csv' }); + const res = await request(app).post('/en-GB/dataset').query({ filename: 'test-data-1.csv' }); expect(res.status).toBe(400); expect(res.body).toEqual({ success: false, @@ -53,7 +47,7 @@ describe('API Endpoints', () => { test('Upload returns 400 if no filename is given', async () => { const csvfile = path.resolve(__dirname, `./test-data-1.csv`); - const res = await request(app).post('/en-GB/api/csv').attach('csv', csvfile); + const res = await request(app).post('/en-GB/dataset').attach('csv', csvfile); expect(res.status).toBe(400); expect(res.body).toEqual({ success: false, @@ -70,7 +64,7 @@ describe('API Endpoints', () => { const csvfile = path.resolve(__dirname, `./test-data-1.csv`); const res = await request(app) - .post('/en-GB/api/csv') + .post('/en-GB/dataset') .attach('csv', csvfile) .field('filename', 'test-upload-data-1') .field('description', 'Test Data File 1'); @@ -86,7 +80,7 @@ describe('API Endpoints', () => { }); test('Get a filelist list returns 200 with a file list', async () => { - const res = await request(app).get('/en-GB/api/csv'); + const res = await request(app).get('/en-GB/dataset'); expect(res.status).toBe(200); expect(res.body).toEqual({ filelist: [ @@ -108,7 +102,7 @@ describe('API Endpoints', () => { const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); const testFile2Buffer = fs.readFileSync(testFile2); DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer); - const res = await request(app).get('/en-GB/api/csv/test-data-2.csv/view').query({ page_number: 20 }); + const res = await request(app).get('/en-GB/dataset/test-data-2.csv/view').query({ page_number: 20 }); expect(res.status).toBe(400); expect(res.body).toEqual({ success: false, @@ -134,7 +128,7 @@ describe('API Endpoints', () => { const testFile2Buffer = fs.readFileSync(testFile2); DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer); - const res = await request(app).get('/en-GB/api/csv/test-data-2.csv/view').query({ page_size: 1000 }); + const res = await request(app).get('/en-GB/dataset/test-data-2.csv/view').query({ page_size: 1000 }); expect(res.status).toBe(400); expect(res.body).toEqual({ success: false, @@ -160,7 +154,7 @@ describe('API Endpoints', () => { const testFile2Buffer = fs.readFileSync(testFile2); DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer); - const res = await request(app).get('/en-GB/api/csv/test-data-2.csv/view').query({ page_size: 1 }); + const res = await request(app).get('/en-GB/dataset/test-data-2.csv/view').query({ page_size: 1 }); expect(res.status).toBe(400); expect(res.body).toEqual({ success: false, @@ -185,19 +179,47 @@ describe('API Endpoints', () => { const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); const testFile2Buffer = fs.readFileSync(testFile2); DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer.toString()); + const datafile = await Datafile.findOneBy({ id: 'fa07be9d-3495-432d-8c1f-d0fc6daae359' }); + const res = await request(app).get('/en-GB/dataset/fa07be9d-3495-432d-8c1f-d0fc6daae359/'); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + id: datafile?.id, + name: datafile?.name, + description: datafile?.description, + creation_date: datafile?.creationDate.toISOString(), + csv_link: `/datafile/fa07be9d-3495-432d-8c1f-d0fc6daae359/csv`, + xslx_link: `/datafile/fa07be9d-3495-432d-8c1f-d0fc6daae359/xlsx`, + view_link: `/datafile/fa07be9d-3495-432d-8c1f-d0fc6daae359/view` + }); + }); + + test('Get csv file rertunrs 200 and complete file data', async () => { + const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile2); + DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer.toString()); - const res = await request(app).get('/en-GB/api/csv/test-data-2.csv'); + const res = await request(app).get('/en-GB/dataset/test-data-2.csv/csv'); expect(res.status).toBe(200); expect(res.text).toEqual(testFile2Buffer.toString()); }); + test('Get xlsx file rertunrs 200 and complete file data', async () => { + const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile2); + DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer.toString()); + + const res = await request(app).get('/en-GB/dataset/test-data-2.csv/xlsx'); + expect(res.status).toBe(200); + expect(res.body).toEqual({ message: 'Not implmented yet' }); + }); + test('Get file view returns 200 and correct page data', async () => { const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); const testFile1Buffer = fs.readFileSync(testFile2); DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile1Buffer.toString()); const res = await request(app) - .get('/en-GB/api/csv/fa07be9d-3495-432d-8c1f-d0fc6daae359/view') + .get('/en-GB/dataset/fa07be9d-3495-432d-8c1f-d0fc6daae359/view') .query({ page_number: 2, page_size: 100 }); expect(res.status).toBe(200); expect(res.body.current_page).toBe(2); @@ -211,7 +233,7 @@ describe('API Endpoints', () => { test('Get file view returns 404 when a non-existant file is requested', async () => { DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(null); - const res = await request(app).get('/en-GB/api/csv/test-data-4.csv'); + const res = await request(app).get('/en-GB/dataset/test-data-4.csv/csv'); expect(res.status).toBe(404); expect(res.body).toEqual({ message: 'File not found... file is null or undefined' }); }); @@ -219,7 +241,7 @@ describe('API Endpoints', () => { test('Get file view returns 404 when a non-existant file view is requested', async () => { DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(null); - const res = await request(app).get('/en-GB/api/csv/test-data-4.csv/view'); + const res = await request(app).get('/en-GB/dataset/test-data-4.csv/view'); expect(res.status).toBe(404); expect(res.body).toEqual({ message: 'File not found... file is null or undefined' }); });