Skip to content

Commit

Permalink
Changed the API route to Dataset, improved the API
Browse files Browse the repository at this point in the history 
As part of improving the backend api layer it was time to rename CSV to
dataset and remove the redundant API route as the backend service no
longer provides any views just API endpoints.
  • Loading branch information
@j-maynard
j-maynard committed May 17, 2024
1 parent aa8f6ec commit 5e1940c
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 28 deletions.
5 changes: 2 additions & 3 deletions src/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import Backend from 'i18next-fs-backend';
import i18nextMiddleware from 'i18next-http-middleware';
import { DataSourceOptions } from 'typeorm';

import { apiRoute } from './route/api';
import { apiRoute } from './route/dataset';
import { healthcheck } from './route/healthcheck';
import DatabaseManager from './database-manager';

Expand Down Expand Up @@ -45,10 +45,9 @@ export const logger = pino({
});

app.use(i18nextMiddleware.handle(i18next));
app.use('/:lang/api', apiRoute);
app.use('/:lang/dataset', apiRoute);
app.use('/:lang/healthcheck', healthcheck);
app.use('/healthcheck', healthcheck);
app.use('/public', express.static(`${__dirname}/public`));

app.get('/', (req: Request, res: Response) => {
const lang = req.headers['accept-language'] || req.headers['Accept-Language'] || req.i18n.language || 'en-GB';
Expand Down
9 changes: 9 additions & 0 deletions src/models/datafile-dto.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
export interface DatafileDTO {
id: string;
name: string;
description: string;
creation_date: Date;
csv_link: string;
xslx_link: string;
view_link: string;
}
52 changes: 44 additions & 8 deletions src/route/api.ts → src/route/dataset.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { processCSV, uploadCSV, DEFAULT_PAGE_SIZE } from '../controllers/csv-pro
import { DataLakeService } from '../controllers/datalake';
import { Datafile } from '../entity/Datafile';
import { FileDescription } from '../models/filelist';
import { DatafileDTO } from '../models/datafile-dto';

const storage = multer.memoryStorage();
const upload = multer({ storage });
Expand All @@ -17,11 +18,7 @@ export const logger = pino({

export const apiRoute = Router();

apiRoute.get('/', (req, res) => {
res.json({ message: req.t('api.available') });
});

apiRoute.post('/csv', upload.single('csv'), async (req: Request, res: Response) => {
apiRoute.post('/', upload.single('csv'), async (req: Request, res: Response) => {
if (!req.file) {
res.status(400);
res.json({
Expand Down Expand Up @@ -78,7 +75,7 @@ apiRoute.post('/csv', upload.single('csv'), async (req: Request, res: Response)
res.json(processedCSV);
});

apiRoute.get('/csv/', async (req, res) => {
apiRoute.get('/', async (req, res) => {
const datafiles = await Datafile.find();
const fileList: FileDescription[] = [];
for (const datafile of datafiles) {
Expand All @@ -91,7 +88,32 @@ apiRoute.get('/csv/', async (req, res) => {
res.json({ filelist: fileList });
});

apiRoute.get('/csv/:file', async (req, res) => {
apiRoute.get('/:file', async (req, res) => {
const fileId = req.params.file;
if (fileId === undefined || fileId === null) {
res.status(404);
res.json({ message: 'File not found... file is null or undefined' });
return;
}
const datafile = await Datafile.findOneBy({ id: fileId });
if (datafile === undefined || datafile === null) {
res.status(404);
res.json({ message: 'File not found... file ID not found in Database' });
return;
}
const datafileDTO: DatafileDTO = {
id: datafile.id,
name: datafile.name,
description: datafile.description,
creation_date: datafile.creationDate,
csv_link: `/datafile/${fileId}/csv`,
xslx_link: `/datafile/${fileId}/xlsx`,
view_link: `/datafile/${fileId}/view`
};
res.json(datafileDTO);
});

apiRoute.get('/:file/csv', async (req, res) => {
const dataLakeService = new DataLakeService();
const filename = req.params.file;
const file = await dataLakeService.downloadFile(`${filename}.csv`);
Expand All @@ -107,7 +129,21 @@ apiRoute.get('/csv/:file', async (req, res) => {
res.end();
});

apiRoute.get('/csv/:file/view', async (req, res) => {
apiRoute.get('/:file/xlsx', async (req, res) => {
const dataLakeService = new DataLakeService();
const filename = req.params.file;
const file = await dataLakeService.downloadFile(`${filename}.csv`);
if (file === undefined || file === null) {
res.status(404);
res.json({ message: 'File not found... file is null or undefined' });
return;
}
res.json({
message: 'Not implmented yet'
});
});

apiRoute.get('/:file/view', async (req, res) => {
const dataLakeService = new DataLakeService();
const filename = req.params.file;
const file = await dataLakeService.downloadFile(`${filename}.csv`);
Expand Down
56 changes: 39 additions & 17 deletions test/api.test.ts → test/dataset.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,8 @@ beforeAll(async () => {
});

describe('API Endpoints', () => {
test('Inital API endpoint works', async () => {
const res = await request(app).get('/en-GB/api/');
expect(res.status).toBe(200);
expect(res.body).toEqual({ message: 'api.available' });
});

test('Upload returns 400 if no file attached', async () => {
const res = await request(app).post('/en-GB/api/csv').query({ filename: 'test-data-1.csv' });
const res = await request(app).post('/en-GB/dataset').query({ filename: 'test-data-1.csv' });
expect(res.status).toBe(400);
expect(res.body).toEqual({
success: false,
Expand All @@ -53,7 +47,7 @@ describe('API Endpoints', () => {

test('Upload returns 400 if no filename is given', async () => {
const csvfile = path.resolve(__dirname, `./test-data-1.csv`);
const res = await request(app).post('/en-GB/api/csv').attach('csv', csvfile);
const res = await request(app).post('/en-GB/dataset').attach('csv', csvfile);
expect(res.status).toBe(400);
expect(res.body).toEqual({
success: false,
Expand All @@ -70,7 +64,7 @@ describe('API Endpoints', () => {
const csvfile = path.resolve(__dirname, `./test-data-1.csv`);

const res = await request(app)
.post('/en-GB/api/csv')
.post('/en-GB/dataset')
.attach('csv', csvfile)
.field('filename', 'test-upload-data-1')
.field('description', 'Test Data File 1');
Expand All @@ -86,7 +80,7 @@ describe('API Endpoints', () => {
});

test('Get a filelist list returns 200 with a file list', async () => {
const res = await request(app).get('/en-GB/api/csv');
const res = await request(app).get('/en-GB/dataset');
expect(res.status).toBe(200);
expect(res.body).toEqual({
filelist: [
Expand All @@ -108,7 +102,7 @@ describe('API Endpoints', () => {
const testFile2 = path.resolve(__dirname, `./test-data-2.csv`);
const testFile2Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer);
const res = await request(app).get('/en-GB/api/csv/test-data-2.csv/view').query({ page_number: 20 });
const res = await request(app).get('/en-GB/dataset/test-data-2.csv/view').query({ page_number: 20 });
expect(res.status).toBe(400);
expect(res.body).toEqual({
success: false,
Expand All @@ -134,7 +128,7 @@ describe('API Endpoints', () => {
const testFile2Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer);

const res = await request(app).get('/en-GB/api/csv/test-data-2.csv/view').query({ page_size: 1000 });
const res = await request(app).get('/en-GB/dataset/test-data-2.csv/view').query({ page_size: 1000 });
expect(res.status).toBe(400);
expect(res.body).toEqual({
success: false,
Expand All @@ -160,7 +154,7 @@ describe('API Endpoints', () => {
const testFile2Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer);

const res = await request(app).get('/en-GB/api/csv/test-data-2.csv/view').query({ page_size: 1 });
const res = await request(app).get('/en-GB/dataset/test-data-2.csv/view').query({ page_size: 1 });
expect(res.status).toBe(400);
expect(res.body).toEqual({
success: false,
Expand All @@ -185,19 +179,47 @@ describe('API Endpoints', () => {
const testFile2 = path.resolve(__dirname, `./test-data-2.csv`);
const testFile2Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer.toString());
const datafile = await Datafile.findOneBy({ id: 'fa07be9d-3495-432d-8c1f-d0fc6daae359' });
const res = await request(app).get('/en-GB/dataset/fa07be9d-3495-432d-8c1f-d0fc6daae359/');
expect(res.status).toBe(200);
expect(res.body).toEqual({
id: datafile?.id,
name: datafile?.name,
description: datafile?.description,
creation_date: datafile?.creationDate.toISOString(),
csv_link: `/datafile/fa07be9d-3495-432d-8c1f-d0fc6daae359/csv`,
xslx_link: `/datafile/fa07be9d-3495-432d-8c1f-d0fc6daae359/xlsx`,
view_link: `/datafile/fa07be9d-3495-432d-8c1f-d0fc6daae359/view`
});
});

test('Get csv file rertunrs 200 and complete file data', async () => {
const testFile2 = path.resolve(__dirname, `./test-data-2.csv`);
const testFile2Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer.toString());

const res = await request(app).get('/en-GB/api/csv/test-data-2.csv');
const res = await request(app).get('/en-GB/dataset/test-data-2.csv/csv');
expect(res.status).toBe(200);
expect(res.text).toEqual(testFile2Buffer.toString());
});

test('Get xlsx file rertunrs 200 and complete file data', async () => {
const testFile2 = path.resolve(__dirname, `./test-data-2.csv`);
const testFile2Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile2Buffer.toString());

const res = await request(app).get('/en-GB/dataset/test-data-2.csv/xlsx');
expect(res.status).toBe(200);
expect(res.body).toEqual({ message: 'Not implmented yet' });
});

test('Get file view returns 200 and correct page data', async () => {
const testFile2 = path.resolve(__dirname, `./test-data-2.csv`);
const testFile1Buffer = fs.readFileSync(testFile2);
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile1Buffer.toString());

const res = await request(app)
.get('/en-GB/api/csv/fa07be9d-3495-432d-8c1f-d0fc6daae359/view')
.get('/en-GB/dataset/fa07be9d-3495-432d-8c1f-d0fc6daae359/view')
.query({ page_number: 2, page_size: 100 });
expect(res.status).toBe(200);
expect(res.body.current_page).toBe(2);
Expand All @@ -211,15 +233,15 @@ describe('API Endpoints', () => {
test('Get file view returns 404 when a non-existant file is requested', async () => {
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(null);

const res = await request(app).get('/en-GB/api/csv/test-data-4.csv');
const res = await request(app).get('/en-GB/dataset/test-data-4.csv/csv');
expect(res.status).toBe(404);
expect(res.body).toEqual({ message: 'File not found... file is null or undefined' });
});

test('Get file view returns 404 when a non-existant file view is requested', async () => {
DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(null);

const res = await request(app).get('/en-GB/api/csv/test-data-4.csv/view');
const res = await request(app).get('/en-GB/dataset/test-data-4.csv/view');
expect(res.status).toBe(404);
expect(res.body).toEqual({ message: 'File not found... file is null or undefined' });
});
Expand Down

0 comments on commit 5e1940c

Please sign in to comment.