diff --git a/src/controllers/csv-processor.ts b/src/controllers/csv-processor.ts index 6c3369b..f96bb4c 100644 --- a/src/controllers/csv-processor.ts +++ b/src/controllers/csv-processor.ts @@ -11,7 +11,7 @@ import { CSVHeader, ViewStream, ViewDTO, ViewErrDTO } from '../dtos/view-dto'; import { Dataset } from '../entities/dataset'; import { Revision } from '../entities/revision'; import { Source } from '../entities/source'; -import { Import } from '../entities/import'; +import { FileImport } from '../entities/import_file'; import { BlobStorageService } from './blob-storage'; import { DataLakeService } from './datalake'; @@ -119,13 +119,13 @@ function validateParams(page_number: number, max_page_number: number, page_size: return errors; } -export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: string): Promise => { +export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: string): Promise => { const blobStorageService = new BlobStorageService(); if (!fileStream) { logger.error('No buffer to upload to blob storage'); throw new Error('No buffer to upload to blob storage'); } - const importRecord = new Import(); + const importRecord = new FileImport(); importRecord.id = randomUUID(); importRecord.mime_type = filetype; const extension = filetype === 'text/csv' ? 'csv' : 'zip'; @@ -141,7 +141,7 @@ export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: str await blobStorageService.uploadFile(`${importRecord.filename}`, fileStream); const resolvedHash = await promisedHash; if (resolvedHash) importRecord.hash = resolvedHash; - importRecord.uploaded_at = new Date(Date.now()); + importRecord.uploadedAt = new Date(Date.now()); importRecord.type = 'Draft'; importRecord.location = 'BlobStorage'; return importRecord; @@ -151,9 +151,9 @@ export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: str } }; -export const uploadCSVBufferToBlobStorage = async (fileBuffer: Buffer, filetype: string): Promise => { +export const uploadCSVBufferToBlobStorage = async (fileBuffer: Buffer, filetype: string): Promise => { const fileStream = Readable.from(fileBuffer); - const importRecord: Import = await uploadCSVToBlobStorage(fileStream, filetype); + const importRecord: FileImport = await uploadCSVToBlobStorage(fileStream, filetype); return importRecord; }; @@ -172,7 +172,7 @@ async function processCSVData( page: number, size: number, dataset: Dataset, - importObj: Import + importObj: FileImport ): Promise { const dataArray: Array> = (await parse(buffer, { delimiter: ',' @@ -241,7 +241,10 @@ async function processCSVData( }; } -export const getFileFromDataLake = async (dataset: Dataset, importObj: Import): Promise => { +export const getFileFromDataLake = async ( + dataset: Dataset, + importObj: FileImport +): Promise => { const datalakeService = new DataLakeService(); let stream: Readable; try { @@ -268,7 +271,7 @@ export const getFileFromDataLake = async (dataset: Dataset, importObj: Import): export const processCSVFromDatalake = async ( dataset: Dataset, - importObj: Import, + importObj: FileImport, page: number, size: number ): Promise => { @@ -296,7 +299,10 @@ export const processCSVFromDatalake = async ( return processCSVData(buff, page, size, dataset, importObj); }; -export const getFileFromBlobStorage = async (dataset: Dataset, importObj: Import): Promise => { +export const getFileFromBlobStorage = async ( + dataset: Dataset, + importObj: FileImport +): Promise => { const blobStoageService = new BlobStorageService(); let stream: Readable; try { @@ -312,7 +318,7 @@ export const getFileFromBlobStorage = async (dataset: Dataset, importObj: Import { lang: ENGLISH, message: t('errors.download_from_blobstorage', { lng: ENGLISH }) }, { lang: WELSH, message: t('errors.download_from_blobstorage', { lng: WELSH }) } ], - tag: { name: 'errors.download_from_datalake', params: {} } + tag: { name: 'errors.download_from_blobstorage', params: {} } } ], dataset_id: dataset.id @@ -326,7 +332,7 @@ export const getFileFromBlobStorage = async (dataset: Dataset, importObj: Import export const processCSVFromBlobStorage = async ( dataset: Dataset, - importObj: Import, + importObj: FileImport, page: number, size: number ): Promise => { @@ -345,7 +351,7 @@ export const processCSVFromBlobStorage = async ( { lang: ENGLISH, message: t('errors.download_from_blobstorage', { lng: ENGLISH }) }, { lang: WELSH, message: t('errors.download_from_blobstorage', { lng: WELSH }) } ], - tag: { name: 'errors.download_from_datalake', params: {} } + tag: { name: 'errors.download_from_blobstorage', params: {} } } ], dataset_id: dataset.id @@ -354,7 +360,7 @@ export const processCSVFromBlobStorage = async ( return processCSVData(buff, page, size, dataset, importObj); }; -export const moveFileToDataLake = async (importObj: Import) => { +export const moveFileToDataLake = async (importObj: FileImport) => { const blobStorageService = new BlobStorageService(); const datalakeService = new DataLakeService(); try { @@ -367,7 +373,7 @@ export const moveFileToDataLake = async (importObj: Import) => { } }; -export const createSources = async (importObj: Import): Promise => { +export const createSources = async (importObj: FileImport): Promise => { const revision: Revision = await importObj.revision; const dataset: Dataset = await revision.dataset; let fileView: ViewDTO | ViewErrDTO; @@ -395,7 +401,7 @@ export const createSources = async (importObj: Import): Promise => sources.push(source); source.save(); }); - const saveImport = await Import.findOneBy({ id: importObj.id }); + const saveImport = await FileImport.findOneBy({ id: importObj.id }); if (!saveImport) { throw new Error('Import not found'); } diff --git a/src/database-manager.ts b/src/database-manager.ts index 27d3908..4f8d17f 100644 --- a/src/database-manager.ts +++ b/src/database-manager.ts @@ -5,7 +5,7 @@ import { Logger } from 'pino'; import { Dataset } from './entities/dataset'; import { DatasetInfo } from './entities/dataset_info'; import { Revision } from './entities/revision'; -import { Import } from './entities/import'; +import { FileImport } from './entities/import_file'; import { CsvInfo } from './entities/csv_info'; import { Source } from './entities/source'; import { Dimension } from './entities/dimension'; @@ -39,7 +39,7 @@ class DatabaseManager { async initializeDataSource() { this.dataSource = new DataSource({ ...this.datasourceOptions, - entities: [Dataset, DatasetInfo, Revision, Import, CsvInfo, Source, Dimension, DimensionInfo, User] + entities: [Dataset, DatasetInfo, Revision, FileImport, CsvInfo, Source, Dimension, DimensionInfo, User] }); await this.dataSource diff --git a/src/dtos/dataset-dto.ts b/src/dtos/dataset-dto.ts index a26adae..3df8186 100644 --- a/src/dtos/dataset-dto.ts +++ b/src/dtos/dataset-dto.ts @@ -2,7 +2,7 @@ import { Dataset } from '../entities/dataset'; import { Dimension } from '../entities/dimension'; import { DimensionInfo } from '../entities/dimension_info'; import { Source } from '../entities/source'; -import { Import } from '../entities/import'; +import { FileImport } from '../entities/import_file'; import { Revision } from '../entities/revision'; import { DatasetInfo } from '../entities/dataset_info'; @@ -88,12 +88,12 @@ export class ImportDTO { mime_type: string; filename: string; hash: string; - uploaded_at: string; + uploadedAt: string; type: string; location: string; sources?: SourceDTO[]; - static async fromImport(importEntity: Import): Promise { + static async fromImport(importEntity: FileImport): Promise { const dto = new ImportDTO(); dto.id = importEntity.id; const revision = await importEntity.revision; @@ -101,7 +101,7 @@ export class ImportDTO { dto.mime_type = importEntity.mime_type; dto.filename = importEntity.filename; dto.hash = importEntity.hash; - dto.uploaded_at = importEntity.uploaded_at?.toISOString() || ''; + dto.uploadedAt = importEntity.uploadedAt?.toISOString() || ''; dto.type = importEntity.type; dto.location = importEntity.location; dto.sources = await Promise.all( @@ -146,14 +146,14 @@ export class RevisionDTO { revDto.approved_by = (await revision.approvedBy)?.name || undefined; revDto.created_by = (await revision.createdBy).name; revDto.imports = await Promise.all( - (await revision.imports).map(async (imp: Import) => { + (await revision.imports).map(async (imp: FileImport) => { const impDto = new ImportDTO(); impDto.id = imp.id; impDto.revision_id = (await imp.revision).id; impDto.mime_type = imp.mime_type; impDto.filename = imp.filename; impDto.hash = imp.hash; - impDto.uploaded_at = imp.uploaded_at.toISOString(); + impDto.uploadedAt = imp.uploadedAt.toISOString(); impDto.type = imp.type; impDto.location = imp.location; impDto.sources = await Promise.all( @@ -262,14 +262,14 @@ export class DatasetDTO { revDto.approved_by = (await revision.approvedBy)?.name || undefined; revDto.created_by = (await revision.createdBy)?.name; revDto.imports = await Promise.all( - (await revision.imports).map(async (imp: Import) => { + (await revision.imports).map(async (imp: FileImport) => { const impDto = new ImportDTO(); impDto.id = imp.id; impDto.revision_id = (await imp.revision).id; impDto.mime_type = imp.mime_type; impDto.filename = imp.filename; impDto.hash = imp.hash; - impDto.uploaded_at = imp.uploaded_at.toISOString(); + impDto.uploadedAt = imp.uploadedAt.toISOString(); impDto.type = imp.type; impDto.location = imp.location; impDto.sources = await Promise.all( @@ -356,13 +356,13 @@ export class DatasetDTO { revDto.approved_by = (await revision.approvedBy)?.name || undefined; revDto.created_by = (await revision.createdBy)?.name; revDto.imports = await Promise.all( - (await revision.imports).map((imp: Import) => { + (await revision.imports).map((imp: FileImport) => { const impDto = new ImportDTO(); impDto.id = imp.id; impDto.mime_type = imp.mime_type; impDto.filename = imp.filename; impDto.hash = imp.hash; - impDto.uploaded_at = imp.uploaded_at.toISOString(); + impDto.uploadedAt = imp.uploadedAt.toISOString(); impDto.type = imp.type; impDto.location = imp.location; return impDto; diff --git a/src/entities/csv_info.ts b/src/entities/csv_info.ts index 53b0d4a..330b0cf 100644 --- a/src/entities/csv_info.ts +++ b/src/entities/csv_info.ts @@ -1,7 +1,7 @@ import { Entity, PrimaryColumn, Column, BaseEntity, ManyToOne, JoinColumn } from 'typeorm'; // eslint-disable-next-line import/no-cycle -import { Import } from './import'; +import { FileImport } from './import_file'; @Entity() export class CsvInfo extends BaseEntity { @@ -17,10 +17,10 @@ export class CsvInfo extends BaseEntity { @Column({ type: 'varchar', length: 2 }) linebreak: string; - @ManyToOne(() => Import, (importEntity) => importEntity.csvInfo, { + @ManyToOne(() => FileImport, (importEntity) => importEntity.csvInfo, { onDelete: 'CASCADE', orphanedRowAction: 'delete' }) @JoinColumn({ name: 'import_id' }) - import: Promise; + import: Promise; } diff --git a/src/entities/import.ts b/src/entities/import_file.ts similarity index 86% rename from src/entities/import.ts rename to src/entities/import_file.ts index 7353cb2..874c6e1 100644 --- a/src/entities/import.ts +++ b/src/entities/import_file.ts @@ -16,8 +16,8 @@ import { CsvInfo } from './csv_info'; // eslint-disable-next-line import/no-cycle import { Source } from './source'; -@Entity() -export class Import extends BaseEntity { +@Entity({ name: 'file_import', orderBy: { uploadedAt: 'ASC' } }) +export class FileImport extends BaseEntity { @PrimaryGeneratedColumn('uuid') id: string; @@ -43,8 +43,8 @@ export class Import extends BaseEntity { @Column({ type: 'varchar', length: 255 }) hash: string; - @CreateDateColumn() - uploaded_at: Date; + @CreateDateColumn({ name: 'uploaded_at' }) + uploadedAt: Date; @Column({ type: process.env.NODE_ENV === 'test' ? 'text' : 'enum', @@ -55,7 +55,7 @@ export class Import extends BaseEntity { @Column({ type: process.env.NODE_ENV === 'test' ? 'text' : 'enum', - enum: ['BlobStorage', 'Datalake'], + enum: ['BlobStorage', 'Datalake', 'Unknown'], nullable: false }) location: string; diff --git a/src/entities/revision.ts b/src/entities/revision.ts index 6f06509..c4e70e3 100644 --- a/src/entities/revision.ts +++ b/src/entities/revision.ts @@ -15,7 +15,7 @@ import { User } from './user'; // eslint-disable-next-line import/no-cycle import { Source } from './source'; // eslint-disable-next-line import/no-cycle -import { Import } from './import'; +import { FileImport } from './import_file'; interface RevisionInterface { id: string; @@ -28,10 +28,10 @@ interface RevisionInterface { approvalDate: Date; approvedBy: Promise; createdBy: Promise; - imports: Promise; + imports: Promise; } -@Entity() +@Entity({ name: 'revision', orderBy: { creationDate: 'ASC' } }) export class Revision extends BaseEntity implements RevisionInterface { @PrimaryGeneratedColumn('uuid') id: string; @@ -79,10 +79,10 @@ export class Revision extends BaseEntity implements RevisionInterface { }) sources: Promise; - @OneToMany(() => Import, (importEntity) => importEntity.revision, { + @OneToMany(() => FileImport, (importEntity) => importEntity.revision, { cascade: true }) - imports: Promise; + imports: Promise; @ManyToOne(() => User, { nullable: true }) @JoinColumn({ name: 'approved_by' }) diff --git a/src/entities/source.ts b/src/entities/source.ts index 3732ff1..67a24c2 100644 --- a/src/entities/source.ts +++ b/src/entities/source.ts @@ -3,7 +3,7 @@ import { Entity, PrimaryGeneratedColumn, Column, BaseEntity, ManyToOne, JoinColu // eslint-disable-next-line import/no-cycle import { Dimension } from './dimension'; // eslint-disable-next-line import/no-cycle -import { Import } from './import'; +import { FileImport } from './import_file'; // eslint-disable-next-line import/no-cycle import { Revision } from './revision'; import { SourceType } from './source_type'; @@ -20,13 +20,13 @@ export class Source extends BaseEntity { @JoinColumn({ name: 'dimension_id' }) dimension: Promise; - @ManyToOne(() => Import, (importEntity) => importEntity.sources, { + @ManyToOne(() => FileImport, (importEntity) => importEntity.sources, { nullable: false, onDelete: 'CASCADE', orphanedRowAction: 'delete' }) @JoinColumn({ name: 'import_id' }) - import: Promise; + import: Promise; @ManyToOne(() => Revision, { onDelete: 'CASCADE', diff --git a/src/resources/locales/cy-GB.json b/src/resources/locales/cy-GB.json index 85e213b..5a11fcf 100644 --- a/src/resources/locales/cy-GB.json +++ b/src/resources/locales/cy-GB.json @@ -43,7 +43,8 @@ "page_number_to_high": "Rhaid i rif y dudalen fod yn llai na neu'n hafal i {{page_number}} (GPT)", "page_number_to_low": "Rhaid i rif y dudalen fod yn fwy na 0 (GPT)", "no_datafile": "Dim ffeil data ynghlwm wrth Set Ddata (GPT)", - "download_from_datalake": "Gwall wrth lawrlwytho ffeil o'r llyn data (GPT)" + "download_from_datalake": "Gwall wrth lawrlwytho ffeil o'r llyn data (GPT)", + "download_from_blobstorage": "Gwall wrth lawrlwytho ffeil o’r storfa blob (GPT)" }, "dimensionInfo": { "footnotes": { diff --git a/src/resources/locales/en-GB.json b/src/resources/locales/en-GB.json index 09949f6..defea5f 100644 --- a/src/resources/locales/en-GB.json +++ b/src/resources/locales/en-GB.json @@ -38,6 +38,7 @@ "page_number_to_low": "Page number must be greater than 0", "no_datafile": "No datafile attached to Dataset", "download_from_datalake": "Error downloading file from datalake", + "download_from_blobstorage": "Error downloading file from blob storage", "upload": { "no_title": "No title for the dataset has been provided", "no-internal-name": "No internal name for the dataset has been provided", diff --git a/src/route/dataset-route.ts b/src/route/dataset-route.ts index 41918a9..95e302c 100644 --- a/src/route/dataset-route.ts +++ b/src/route/dataset-route.ts @@ -27,9 +27,9 @@ import { Dataset } from '../entities/dataset'; import { DatasetInfo } from '../entities/dataset_info'; import { Dimension } from '../entities/dimension'; import { Revision } from '../entities/revision'; -import { Import } from '../entities/import'; +import { FileImport } from '../entities/import_file'; import { DatasetTitle, FileDescription } from '../dtos/filelist'; -import { DatasetDTO, DimensionDTO, RevisionDTO, ImportDTO, SourceDTO } from '../dtos/dataset-dto'; +import { DatasetDTO, DimensionDTO, RevisionDTO, ImportDTO } from '../dtos/dataset-dto'; export const logger = pino({ name: 'StatsWales-Alpha-App: DatasetRoute', @@ -52,7 +52,7 @@ function isValidUUID(uuid: string): boolean { } function validateIds(id: string, idType: string, res: Response): boolean { - if (id === undefined || id === null) { + if (id === undefined) { res.status(400); res.json({ message: `${idType} ID is null or undefined` }); return false; @@ -98,9 +98,9 @@ async function validateRevision(revisionID: string, res: Response): Promise { +async function validateImport(importID: string, res: Response): Promise { if (!validateIds(importID, IMPORT, res)) return null; - const importObj = await Import.findOneBy({ id: importID }); + const importObj = await FileImport.findOneBy({ id: importID }); if (!importObj) { res.status(404); res.json({ message: 'Import not found.' }); @@ -156,7 +156,7 @@ apiRoute.post('/', upload.single('csv'), async (req: Request, res: Response) => res.json(errorDtoGenerator('title', 'errors.no_title')); return; } - let importRecord: Import; + let importRecord: FileImport; try { importRecord = await uploadCSVBufferToBlobStorage(req.file.buffer, req.file?.mimetype); } catch (err) { @@ -250,24 +250,18 @@ apiRoute.get('/:dataset_id/view', async (req: Request, res: Response) => { const datasetID: string = req.params.dataset_id; const dataset = await validateDataset(datasetID, res); if (!dataset) return; - const latestRevision = await Revision.find({ - where: { dataset }, - order: { creationDate: 'DESC' }, - take: 1 - }); + const latestRevision = (await dataset.revisions).pop(); if (!latestRevision) { + console.log('latestRevision:', JSON.stringify(latestRevision)); logger.error('Unable to find the last revision'); - res.status(404); + res.status(500); res.json({ message: 'No revision found for dataset' }); return; } - const latestImport = await Import.findOne({ - where: [{ revision: latestRevision[0] }], - order: { uploaded_at: 'DESC' } - }); + const latestImport = (await latestRevision.imports).pop(); if (!latestImport) { logger.error('Unable to find the last import record'); - res.status(404); + res.status(500); res.json({ message: 'No import record found for dataset' }); return; } @@ -281,12 +275,12 @@ apiRoute.get('/:dataset_id/view', async (req: Request, res: Response) => { } else if (latestImport.location === 'Datalake') { processedCSV = await processCSVFromDatalake(dataset, latestImport, page_number, page_size); } else { - res.status(400); + res.status(500); res.json({ message: 'Import location not supported.' }); return; } if (!processedCSV.success) { - res.status(400); + res.status(500); } res.json(processedCSV); }); @@ -336,29 +330,6 @@ apiRoute.get( } ); -// GET /api/dataset/:dataset_id/revision/id/:revision_id/import/id/:import_id/sources -// Returns details of an import with its sources -apiRoute.get( - '/:dataset_id/revision/by-id/:revision_id/import/by-id/:import_id/sources', - async (req: Request, res: Response) => { - const datasetID: string = req.params.dataset_id; - const dataset = await validateDataset(datasetID, res); - if (!dataset) return; - const revisionID: string = req.params.revision_id; - const revision = await validateRevision(revisionID, res); - if (!revision) return; - const importID: string = req.params.import_id; - const importRecord = await validateImport(importID, res); - if (!importRecord) return; - const sources = await importRecord.sources; - const dtos: SourceDTO[] = []; - for (const source of sources) { - dtos.push(await SourceDTO.fromSource(source)); - } - res.json(dtos); - } -); - // GET /api/dataset/:dataset_id/revision/id/:revision_id/import/id/:import_id/preview // Returns a view of the data file attached to the import apiRoute.get( diff --git a/test/dataset.test.ts b/test/dataset.test.ts deleted file mode 100644 index 6631656..0000000 --- a/test/dataset.test.ts +++ /dev/null @@ -1,504 +0,0 @@ -import path from 'path'; -import * as fs from 'fs'; -import { createHash } from 'crypto'; - -import request from 'supertest'; - -import { DataLakeService } from '../src/controllers/datalake'; -import { BlobStorageService } from '../src/controllers/blob-storage'; -import app, { ENGLISH, WELSH, t, dbManager, databaseManager } from '../src/app'; -import { Dataset } from '../src/entities/dataset'; -import { DatasetInfo } from '../src/entities/dataset_info'; -import { Revision } from '../src/entities/revision'; -import { Import } from '../src/entities/import'; -import { CsvInfo } from '../src/entities/csv_info'; -import { Source } from '../src/entities/source'; -import { Dimension } from '../src/entities/dimension'; -import { DimensionType } from '../src/entities/dimension_type'; -import { DimensionInfo } from '../src/entities/dimension_info'; -import { User } from '../src/entities/user'; -import { DatasetDTO, DimensionDTO, RevisionDTO } from '../src/dtos/dataset-dto'; -import { ViewErrDTO } from '../src/dtos/view-dto'; -import { MAX_PAGE_SIZE, MIN_PAGE_SIZE } from '../src/controllers/csv-processor'; - -import { datasourceOptions } from './test-data-source'; - -DataLakeService.prototype.listFiles = jest - .fn() - .mockReturnValue([{ name: 'test-data-1.csv', path: 'test/test-data-1.csv', isDirectory: false }]); - -BlobStorageService.prototype.uploadFile = jest.fn(); - -DataLakeService.prototype.uploadFile = jest.fn(); - -const dataset1Id = 'BDC40218-AF89-424B-B86E-D21710BC92F1'; -const revision1Id = '85F0E416-8BD1-4946-9E2C-1C958897C6EF'; -const import1Id = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359'; -const dimension1Id = '2D7ACD0B-A46A-43F7-8A88-224CE97FC8B9'; - -describe('API Endpoints', () => { - beforeAll(async () => { - await databaseManager(datasourceOptions); - await dbManager.initializeDataSource(); - const user = User.getTestUser(); - await user.save(); - // First create a dataset - const dataset1 = new Dataset(); - dataset1.id = dataset1Id; - dataset1.createdBy = Promise.resolve(user); - dataset1.live = new Date(Date.now()); - // Give it some info - const datasetInfo1 = new DatasetInfo(); - datasetInfo1.dataset = Promise.resolve(dataset1); - datasetInfo1.title = 'Test Dataset 1'; - datasetInfo1.description = 'I am the first test dataset'; - datasetInfo1.language = 'en-GB'; - dataset1.datasetInfo = Promise.resolve([datasetInfo1]); - // At the sametime we also always create a first revision - const revision1 = new Revision(); - revision1.id = revision1Id; - revision1.dataset = Promise.resolve(dataset1); - revision1.createdBy = Promise.resolve(user); - revision1.revisionIndex = 1; - dataset1.revisions = Promise.resolve([revision1]); - // Attach an import e.g. a file to the revision - const import1 = new Import(); - import1.revision = Promise.resolve(revision1); - import1.id = import1Id; - import1.filename = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359.csv'; - const testFile1 = path.resolve(__dirname, `./test-data-2.csv`); - const testFile2Buffer = fs.readFileSync(testFile1); - import1.hash = createHash('sha256').update(testFile2Buffer).digest('hex'); - // First is a draft import and a first upload so everything is in blob storage - import1.location = 'BlobStorage'; - import1.type = 'Draft'; - import1.mime_type = 'text/csv'; - // Its a CSV file so we need to know how to parse it - const csvInfo1 = new CsvInfo(); - csvInfo1.import = Promise.resolve(import1); - csvInfo1.delimiter = ','; - csvInfo1.quote = '"'; - csvInfo1.linebreak = '\n'; - import1.csvInfo = Promise.resolve([csvInfo1]); - revision1.imports = Promise.resolve([import1]); - await dataset1.save(); - // Create some sources for each of the columns in the CSV - const sources: Source[] = []; - const source1 = new Source(); - source1.id = '304574E6-8DD0-4654-BE67-FA055C9F7C81'; - source1.import = Promise.resolve(import1); - source1.revision = Promise.resolve(revision1); - source1.csvField = 'ID'; - source1.columnIndex = 0; - source1.action = 'ignore'; - sources.push(source1); - const source2 = new Source(); - source2.id = 'D3D3D3D3-8DD0-4654-BE67-FA055C9F7C81'; - source2.import = Promise.resolve(import1); - source2.revision = Promise.resolve(revision1); - source2.csvField = 'Text'; - source2.columnIndex = 1; - source2.action = 'create'; - sources.push(source2); - const source3 = new Source(); - source3.id = 'D62FA390-9AB2-496E-A6CA-0C0E2FCF206E'; - source3.import = Promise.resolve(import1); - source3.revision = Promise.resolve(revision1); - source3.csvField = 'Number'; - source3.columnIndex = 2; - source3.action = 'create'; - sources.push(source3); - const source4 = new Source(); - source4.id = 'FB25D668-54F2-44EF-99FE-B4EDC4AF2911'; - source4.import = Promise.resolve(import1); - source4.revision = Promise.resolve(revision1); - source4.csvField = 'Date'; - source4.columnIndex = 3; - source4.action = 'create'; - sources.push(source4); - import1.sources = Promise.resolve(sources); - await import1.save(); - // Next create some dimensions - const dimensions: Dimension[] = []; - const dimension1 = new Dimension(); - dimension1.id = dimension1Id; - dimension1.dataset = Promise.resolve(dataset1); - dimension1.startRevision = Promise.resolve(revision1); - dimension1.type = DimensionType.RAW; - const dimension1Info = new DimensionInfo(); - dimension1Info.dimension = Promise.resolve(dimension1); - dimension1Info.name = 'ID'; - dimension1Info.description = 'Unique identifier'; - dimension1Info.language = 'en-GB'; - dimension1.dimensionInfo = Promise.resolve([dimension1Info]); - dimension1.sources = Promise.resolve([source1]); - source1.dimension = Promise.resolve(dimension1); - dimensions.push(dimension1); - // Dimension 2 - const dimension2 = new Dimension(); - dimension2.id = '61D51F82-0771-4C90-849E-55FFA7A4D802'; - dimension2.dataset = Promise.resolve(dataset1); - dimension2.startRevision = Promise.resolve(revision1); - dimension2.type = DimensionType.TEXT; - const dimension2Info = new DimensionInfo(); - dimension2Info.dimension = Promise.resolve(dimension2); - dimension2Info.name = 'Text'; - dimension2Info.description = 'Sample text strings'; - dimension2Info.language = 'en-GB'; - dimension2.dimensionInfo = Promise.resolve([dimension2Info]); - dimension2.sources = Promise.resolve([source2]); - source2.dimension = Promise.resolve(dimension2); - dimensions.push(dimension2); - // Dimension 3 - const dimension3 = new Dimension(); - dimension3.id = 'F4D5B0F4-180E-4020-AAD5-9300B673D92B'; - dimension3.dataset = Promise.resolve(dataset1); - dimension3.startRevision = Promise.resolve(revision1); - dimension3.type = DimensionType.NUMERIC; - const dimension3Info = new DimensionInfo(); - dimension3Info.dimension = Promise.resolve(dimension3); - dimension3Info.name = 'Value'; - dimension3Info.description = 'Sample numeric values'; - dimension3Info.language = 'en-GB'; - dimension3.dimensionInfo = Promise.resolve([dimension3Info]); - dimension3.sources = Promise.resolve([source3]); - source3.dimension = Promise.resolve(dimension3); - dimensions.push(dimension3); - // Dimension 4 - const dimension4 = new Dimension(); - dimension4.id = 'C24962F4-F395-40EF-B4DD-270E90E10972'; - dimension4.dataset = Promise.resolve(dataset1); - dimension4.startRevision = Promise.resolve(revision1); - dimension4.type = DimensionType.TIME_POINT; - const dimension4Info = new DimensionInfo(); - dimension4Info.dimension = Promise.resolve(dimension4); - dimension4Info.name = 'Date'; - dimension4Info.description = 'Sample date values'; - dimension4Info.language = 'en-GB'; - dimension4.dimensionInfo = Promise.resolve([dimension4Info]); - dimension4.sources = Promise.resolve([source4]); - source4.dimension = Promise.resolve(dimension4); - dimensions.push(dimension4); - dataset1.dimensions = Promise.resolve(dimensions); - await dataset1.save(); - }); - - test('Return true test', async () => { - const dataset1 = await Dataset.findOneBy({ id: dataset1Id }); - if (!dataset1) { - throw new Error('Dataset not found'); - } - const dto = await DatasetDTO.fromDatasetComplete(dataset1); - expect(dto).toBe(dto); - }); - - test('Upload returns 400 if no file attached', async () => { - const err: ViewErrDTO = { - success: false, - dataset_id: undefined, - errors: [ - { - field: 'csv', - message: [ - { - lang: ENGLISH, - message: t('errors.no_csv_data', { lng: ENGLISH }) - }, - { - lang: WELSH, - message: t('errors.no_csv_data', { lng: WELSH }) - } - ], - tag: { - name: 'errors.no_csv_data', - params: {} - } - } - ] - }; - const res = await request(app).post('/en-GB/dataset').query({ filename: 'test-data-1.csv' }); - expect(res.status).toBe(400); - expect(res.body).toEqual(err); - }); - - test('Upload returns 400 if no title is given', async () => { - const err: ViewErrDTO = { - success: false, - dataset_id: undefined, - errors: [ - { - field: 'title', - message: [ - { - lang: ENGLISH, - message: t('errors.no_title', { lng: ENGLISH }) - }, - { - lang: WELSH, - message: t('errors.no_title', { lng: WELSH }) - } - ], - tag: { - name: 'errors.no_title', - params: {} - } - } - ] - }; - const csvfile = path.resolve(__dirname, `./test-data-1.csv`); - const res = await request(app).post('/en-GB/dataset').attach('csv', csvfile); - expect(res.status).toBe(400); - expect(res.body).toEqual(err); - }); - - test('Upload returns 201 if a file is attached', async () => { - const csvfile = path.resolve(__dirname, `./test-data-1.csv`); - - const res = await request(app) - .post('/en-GB/dataset') - .attach('csv', csvfile) - .field('title', 'Test Dataset 3') - .field('lang', 'en-GB'); - const datasetInfo = await DatasetInfo.findOneBy({ title: 'Test Dataset 3' }); - if (!datasetInfo) { - expect(datasetInfo).not.toBeNull(); - return; - } - const dataset = await datasetInfo.dataset; - const datasetDTO = await DatasetDTO.fromDatasetWithRevisionsAndImports(dataset); - expect(res.status).toBe(201); - expect(res.body).toEqual(datasetDTO); - await Dataset.remove(dataset); - }); - - test('Get a filelist list returns 200 with a file list', async () => { - const res = await request(app).get('/en-GB/dataset'); - expect(res.status).toBe(200); - expect(res.body).toEqual({ - filelist: [ - { - titles: [{ language: 'en-GB', title: 'Test Dataset 1' }], - dataset_id: 'BDC40218-AF89-424B-B86E-D21710BC92F1' - } - ] - }); - }); - - test('Get a dataset returns 200 with a shallow object', async () => { - const dataset1 = await Dataset.findOneBy({ id: dataset1Id }); - if (!dataset1) { - throw new Error('Dataset not found'); - } - const dto = await DatasetDTO.fromDatasetComplete(dataset1); - const res = await request(app).get(`/en-GB/dataset/${dataset1Id}`); - expect(res.status).toBe(200); - expect(res.body).toEqual(dto); - }); - - test('Get a dimension returns 200 with a shallow object', async () => { - const dimension = await Dimension.findOneBy({ id: dimension1Id }); - if (!dimension) { - throw new Error('Dataset not found'); - } - const dto = await DimensionDTO.fromDimension(dimension); - const res = await request(app).get(`/en-GB/dataset/${dataset1Id}/dimension/by-id/${dimension1Id}`); - expect(res.status).toBe(200); - expect(res.body).toEqual(dto); - }); - - test('Get a revision returns 200 with a shallow object', async () => { - const revision = await Revision.findOneBy({ id: revision1Id }); - if (!revision) { - throw new Error('Dataset not found'); - } - const dto = await RevisionDTO.fromRevision(revision); - const res = await request(app).get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}`); - expect(res.status).toBe(200); - expect(res.body).toEqual(dto); - }); - - test('Get file view returns 400 if page_number is too high', async () => { - const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); - const testFile2Buffer = fs.readFileSync(testFile2); - BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile2Buffer); - const res = await request(app) - .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) - .query({ page_number: 20 }); - expect(res.status).toBe(400); - expect(res.body).toEqual({ - success: false, - dataset_id: dataset1Id, - errors: [ - { - field: 'page_number', - message: [ - { lang: ENGLISH, message: t('errors.page_number_to_high', { lng: ENGLISH, page_number: 6 }) }, - { lang: WELSH, message: t('errors.page_number_to_high', { lng: WELSH, page_number: 6 }) } - ], - tag: { - name: 'errors.page_number_to_high', - params: { page_number: 6 } - } - } - ] - }); - }); - - test('Get file view returns 400 if page_size is too high', async () => { - const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); - const testFile2Buffer = fs.readFileSync(testFile2); - BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile2Buffer); - - const res = await request(app) - .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) - .query({ page_size: 1000 }); - expect(res.status).toBe(400); - expect(res.body).toEqual({ - success: false, - dataset_id: dataset1Id, - errors: [ - { - field: 'page_size', - message: [ - { - lang: ENGLISH, - message: t('errors.page_size', { - lng: ENGLISH, - max_page_size: MAX_PAGE_SIZE, - min_page_size: MIN_PAGE_SIZE - }) - }, - { - lang: WELSH, - message: t('errors.page_size', { - lng: WELSH, - max_page_size: MAX_PAGE_SIZE, - min_page_size: MIN_PAGE_SIZE - }) - } - ], - tag: { - name: 'errors.page_size', - params: { max_page_size: MAX_PAGE_SIZE, min_page_size: MIN_PAGE_SIZE } - } - } - ] - }); - }); - - test('Get file view returns 400 if page_size is too low', async () => { - const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); - const testFile2Buffer = fs.readFileSync(testFile2); - BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile2Buffer); - - const res = await request(app) - .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) - .query({ page_size: 1 }); - expect(res.status).toBe(400); - expect(res.body).toEqual({ - success: false, - dataset_id: dataset1Id, - errors: [ - { - field: 'page_size', - message: [ - { - lang: ENGLISH, - message: t('errors.page_size', { - lng: ENGLISH, - max_page_size: MAX_PAGE_SIZE, - min_page_size: MIN_PAGE_SIZE - }) - }, - { - lang: WELSH, - message: t('errors.page_size', { - lng: WELSH, - max_page_size: MAX_PAGE_SIZE, - min_page_size: MIN_PAGE_SIZE - }) - } - ], - tag: { - name: 'errors.page_size', - params: { max_page_size: MAX_PAGE_SIZE, min_page_size: MIN_PAGE_SIZE } - } - } - ] - }); - }); - - test('Get file from a dataset rertunrs 200 and complete file data', async () => { - const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); - const testFile1Buffer = fs.readFileSync(testFile2); - BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile1Buffer.toString()); - - const res = await request(app) - .get(`/en-GB/dataset/${dataset1Id}/view`) - .query({ page_number: 2, page_size: 100 }); - expect(res.status).toBe(200); - expect(res.body.current_page).toBe(2); - expect(res.body.total_pages).toBe(6); - expect(res.body.page_size).toBe(100); - expect(res.body.headers).toEqual([ - { index: 0, name: 'ID' }, - { index: 1, name: 'Text' }, - { index: 2, name: 'Number' }, - { index: 3, name: 'Date' } - ]); - expect(res.body.data[0]).toEqual(['101', 'GEYiRzLIFM', '774477', '2002-03-13']); - expect(res.body.data[99]).toEqual(['200', 'QhBxdmrUPb', '3256099', '2026-12-17']); - }); - - test('Get file from a revision and import rertunrs 200 and complete file data', async () => { - const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); - const testFileStream = fs.createReadStream(testFile2); - const testFile2Buffer = fs.readFileSync(testFile2); - BlobStorageService.prototype.getReadableStream = jest.fn().mockReturnValue(testFileStream); - const res = await request(app).get( - `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/raw` - ); - expect(res.status).toBe(200); - expect(res.text).toEqual(testFile2Buffer.toString()); - }); - - test('Get preview of an import returns 200 and correct page data', async () => { - const testFile2 = path.resolve(__dirname, `./test-data-2.csv`); - const testFile1Buffer = fs.readFileSync(testFile2); - BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile1Buffer.toString()); - - const res = await request(app) - .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) - .query({ page_number: 2, page_size: 100 }); - expect(res.status).toBe(200); - expect(res.body.current_page).toBe(2); - expect(res.body.total_pages).toBe(6); - expect(res.body.page_size).toBe(100); - expect(res.body.headers).toEqual([ - { index: 0, name: 'ID' }, - { index: 1, name: 'Text' }, - { index: 2, name: 'Number' }, - { index: 3, name: 'Date' } - ]); - expect(res.body.data[0]).toEqual(['101', 'GEYiRzLIFM', '774477', '2002-03-13']); - expect(res.body.data[99]).toEqual(['200', 'QhBxdmrUPb', '3256099', '2026-12-17']); - }); - - test('Get preview of an import returns 404 when a non-existant import is requested', async () => { - const res = await request(app).get( - `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/97C3F48F-127C-4317-B39C-87350F222310/preview` - ); - expect(res.status).toBe(404); - expect(res.body).toEqual({ message: 'Import not found.' }); - }); - - test('Get file view returns 400 when a not valid UUID is supplied', async () => { - const res = await request(app).get(`/en-GB/dataset/NOT-VALID-ID`); - expect(res.status).toBe(400); - expect(res.body).toEqual({ message: 'Dataset ID is not valid' }); - }); - - afterAll(async () => { - await dbManager.getDataSource().dropDatabase(); - }); -}); diff --git a/test/helpers/test-data-source.ts b/test/helpers/test-data-source.ts new file mode 100644 index 0000000..68962dd --- /dev/null +++ b/test/helpers/test-data-source.ts @@ -0,0 +1,25 @@ +import 'reflect-metadata'; +import { DataSourceOptions } from 'typeorm'; +import * as dotenv from 'dotenv'; + +import { Dataset } from '../../src/entities/dataset'; +import { DatasetInfo } from '../../src/entities/dataset_info'; +import { Revision } from '../../src/entities/revision'; +import { FileImport } from '../../src/entities/import_file'; +import { CsvInfo } from '../../src/entities/csv_info'; +import { Source } from '../../src/entities/source'; +import { Dimension } from '../../src/entities/dimension'; +import { DimensionInfo } from '../../src/entities/dimension_info'; +import { User } from '../../src/entities/user'; + +dotenv.config(); + +export const datasourceOptions: DataSourceOptions = { + name: 'default', + type: 'better-sqlite3', + database: ':memory:', + synchronize: true, + logging: false, + entities: [Dataset, DatasetInfo, Revision, FileImport, CsvInfo, Source, Dimension, DimensionInfo, User], + subscribers: [] +}; diff --git a/test/helpers/test-helper.ts b/test/helpers/test-helper.ts new file mode 100644 index 0000000..24756ed --- /dev/null +++ b/test/helpers/test-helper.ts @@ -0,0 +1,204 @@ +import path from 'path'; +import * as fs from 'fs'; +import { createHash } from 'crypto'; + +import { Dataset } from '../../src/entities/dataset'; +import { DatasetInfo } from '../../src/entities/dataset_info'; +import { Revision } from '../../src/entities/revision'; +import { FileImport } from '../../src/entities/import_file'; +import { CsvInfo } from '../../src/entities/csv_info'; +import { Source } from '../../src/entities/source'; +import { Dimension } from '../../src/entities/dimension'; +import { DimensionType } from '../../src/entities/dimension_type'; +import { DimensionInfo } from '../../src/entities/dimension_info'; +import { User } from '../../src/entities/user'; + +export async function createFullDataset(datasetId: string, revisionId: string, importId: string, dimensionId: string) { + const user = User.getTestUser(); + await user.save(); + // First create a dataset + const dataset = new Dataset(); + dataset.id = datasetId; + dataset.createdBy = Promise.resolve(user); + dataset.live = new Date(Date.now()); + // Give it some info + const datasetInfo = new DatasetInfo(); + datasetInfo.dataset = Promise.resolve(dataset); + datasetInfo.title = 'Test Dataset 1'; + datasetInfo.description = 'I am the first test dataset'; + datasetInfo.language = 'en-GB'; + dataset.datasetInfo = Promise.resolve([datasetInfo]); + // At the sametime we also always create a first revision + const revision = new Revision(); + revision.id = revisionId; + revision.dataset = Promise.resolve(dataset); + revision.createdBy = Promise.resolve(user); + revision.revisionIndex = 1; + dataset.revisions = Promise.resolve([revision]); + // Attach an import e.g. a file to the revision + const imp = new FileImport(); + imp.revision = Promise.resolve(revision); + imp.id = importId; + imp.filename = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359.csv'; + const testFile1 = path.resolve(__dirname, `../sample-csvs/test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile1); + imp.hash = createHash('sha256').update(testFile2Buffer).digest('hex'); + // First is a draft import and a first upload so everything is in blob storage + imp.location = 'BlobStorage'; + imp.type = 'Draft'; + imp.mime_type = 'text/csv'; + // Its a CSV file so we need to know how to parse it + const csvInfo = new CsvInfo(); + csvInfo.import = Promise.resolve(imp); + csvInfo.delimiter = ','; + csvInfo.quote = '"'; + csvInfo.linebreak = '\n'; + imp.csvInfo = Promise.resolve([csvInfo]); + revision.imports = Promise.resolve([imp]); + await dataset.save(); + // Create some sources for each of the columns in the CSV + const sources: Source[] = []; + const source1 = new Source(); + source1.id = '304574E6-8DD0-4654-BE67-FA055C9F7C81'; + source1.import = Promise.resolve(imp); + source1.revision = Promise.resolve(revision); + source1.csvField = 'ID'; + source1.columnIndex = 0; + source1.action = 'ignore'; + sources.push(source1); + const source2 = new Source(); + source2.id = 'D3D3D3D3-8DD0-4654-BE67-FA055C9F7C81'; + source2.import = Promise.resolve(imp); + source2.revision = Promise.resolve(revision); + source2.csvField = 'Text'; + source2.columnIndex = 1; + source2.action = 'create'; + sources.push(source2); + const source3 = new Source(); + source3.id = 'D62FA390-9AB2-496E-A6CA-0C0E2FCF206E'; + source3.import = Promise.resolve(imp); + source3.revision = Promise.resolve(revision); + source3.csvField = 'Number'; + source3.columnIndex = 2; + source3.action = 'create'; + sources.push(source3); + const source4 = new Source(); + source4.id = 'FB25D668-54F2-44EF-99FE-B4EDC4AF2911'; + source4.import = Promise.resolve(imp); + source4.revision = Promise.resolve(revision); + source4.csvField = 'Date'; + source4.columnIndex = 3; + source4.action = 'create'; + sources.push(source4); + imp.sources = Promise.resolve(sources); + await imp.save(); + // Next create some dimensions + const dimensions: Dimension[] = []; + const dimension1 = new Dimension(); + dimension1.id = dimensionId; + dimension1.dataset = Promise.resolve(dataset); + dimension1.startRevision = Promise.resolve(revision); + dimension1.type = DimensionType.RAW; + const dimension1Info = new DimensionInfo(); + dimension1Info.dimension = Promise.resolve(dimension1); + dimension1Info.name = 'ID'; + dimension1Info.description = 'Unique identifier'; + dimension1Info.language = 'en-GB'; + dimension1.dimensionInfo = Promise.resolve([dimension1Info]); + dimension1.sources = Promise.resolve([source1]); + source1.dimension = Promise.resolve(dimension1); + dimensions.push(dimension1); + // Dimension 2 + const dimension2 = new Dimension(); + dimension2.id = '61D51F82-0771-4C90-849E-55FFA7A4D802'; + dimension2.dataset = Promise.resolve(dataset); + dimension2.startRevision = Promise.resolve(revision); + dimension2.type = DimensionType.TEXT; + const dimension2Info = new DimensionInfo(); + dimension2Info.dimension = Promise.resolve(dimension2); + dimension2Info.name = 'Text'; + dimension2Info.description = 'Sample text strings'; + dimension2Info.language = 'en-GB'; + dimension2.dimensionInfo = Promise.resolve([dimension2Info]); + dimension2.sources = Promise.resolve([source2]); + source2.dimension = Promise.resolve(dimension2); + dimensions.push(dimension2); + // Dimension 3 + const dimension3 = new Dimension(); + dimension3.id = 'F4D5B0F4-180E-4020-AAD5-9300B673D92B'; + dimension3.dataset = Promise.resolve(dataset); + dimension3.startRevision = Promise.resolve(revision); + dimension3.type = DimensionType.NUMERIC; + const dimension3Info = new DimensionInfo(); + dimension3Info.dimension = Promise.resolve(dimension3); + dimension3Info.name = 'Value'; + dimension3Info.description = 'Sample numeric values'; + dimension3Info.language = 'en-GB'; + dimension3.dimensionInfo = Promise.resolve([dimension3Info]); + dimension3.sources = Promise.resolve([source3]); + source3.dimension = Promise.resolve(dimension3); + dimensions.push(dimension3); + // Dimension 4 + const dimension4 = new Dimension(); + dimension4.id = 'C24962F4-F395-40EF-B4DD-270E90E10972'; + dimension4.dataset = Promise.resolve(dataset); + dimension4.startRevision = Promise.resolve(revision); + dimension4.type = DimensionType.TIME_POINT; + const dimension4Info = new DimensionInfo(); + dimension4Info.dimension = Promise.resolve(dimension4); + dimension4Info.name = 'Date'; + dimension4Info.description = 'Sample date values'; + dimension4Info.language = 'en-GB'; + dimension4.dimensionInfo = Promise.resolve([dimension4Info]); + dimension4.sources = Promise.resolve([source4]); + source4.dimension = Promise.resolve(dimension4); + dimensions.push(dimension4); + dataset.dimensions = Promise.resolve(dimensions); + await dataset.save(); +} + +export async function createSmallDataset(datasetId: string, revisionId: string, importId: string) { + const user = User.getTestUser(); + await user.save(); + // First create a dataset + const dataset = new Dataset(); + dataset.id = datasetId; + dataset.createdBy = Promise.resolve(user); + dataset.live = new Date(Date.now()); + // Give it some info + const datasetInfo = new DatasetInfo(); + datasetInfo.dataset = Promise.resolve(dataset); + datasetInfo.title = 'Test Dataset 1'; + datasetInfo.description = 'I am a small incomplete test dataset'; + datasetInfo.language = 'en-GB'; + dataset.datasetInfo = Promise.resolve([datasetInfo]); + // At the sametime we also always create a first revision + const revision = new Revision(); + revision.id = revisionId; + revision.dataset = Promise.resolve(dataset); + revision.createdBy = Promise.resolve(user); + revision.revisionIndex = 1; + dataset.revisions = Promise.resolve([revision]); + // Attach an import e.g. a file to the revision + const imp = new FileImport(); + imp.revision = Promise.resolve(revision); + imp.id = importId; + imp.filename = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359.csv'; + const testFile1 = path.resolve(__dirname, `../sample-csvs/test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile1); + imp.hash = createHash('sha256').update(testFile2Buffer).digest('hex'); + // First is a draft import and a first upload so everything is in blob storage + imp.location = 'BlobStorage'; + imp.type = 'Draft'; + imp.mime_type = 'text/csv'; + // Its a CSV file so we need to know how to parse it + const csvInfo = new CsvInfo(); + csvInfo.import = Promise.resolve(imp); + csvInfo.delimiter = ','; + csvInfo.quote = '"'; + csvInfo.linebreak = '\n'; + imp.csvInfo = Promise.resolve([csvInfo]); + revision.imports = Promise.resolve([imp]); + await dataset.save(); + return dataset; +} diff --git a/test/metadata-routes.test.ts b/test/metadata-routes.test.ts new file mode 100644 index 0000000..a4fb1c8 --- /dev/null +++ b/test/metadata-routes.test.ts @@ -0,0 +1,159 @@ +import request from 'supertest'; + +import { DataLakeService } from '../src/controllers/datalake'; +import { BlobStorageService } from '../src/controllers/blob-storage'; +import app, { dbManager, databaseManager } from '../src/app'; +import { Dataset } from '../src/entities/dataset'; +import { Revision } from '../src/entities/revision'; +import { Dimension } from '../src/entities/dimension'; +import { FileImport } from '../src/entities/import_file'; +import { DatasetDTO, DimensionDTO, ImportDTO, RevisionDTO } from '../src/dtos/dataset-dto'; + +import { createFullDataset } from './helpers/test-helper'; +import { datasourceOptions } from './helpers/test-data-source'; + +DataLakeService.prototype.listFiles = jest + .fn() + .mockReturnValue([{ name: 'test-data-1.csv', path: 'test/test-data-1.csv', isDirectory: false }]); + +BlobStorageService.prototype.uploadFile = jest.fn(); + +DataLakeService.prototype.uploadFile = jest.fn(); + +const dataset1Id = 'BDC40218-AF89-424B-B86E-D21710BC92F1'; +const revision1Id = '85F0E416-8BD1-4946-9E2C-1C958897C6EF'; +const import1Id = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359'; +const dimension1Id = '2D7ACD0B-A46A-43F7-8A88-224CE97FC8B9'; + +describe('API Endpoints for viewing dataset objects', () => { + beforeAll(async () => { + await databaseManager(datasourceOptions); + await dbManager.initializeDataSource(); + await createFullDataset(dataset1Id, revision1Id, import1Id, dimension1Id); + }); + + describe('List all datasets', () => { + test('Get a list of all datasets returns 200 with a file list', async () => { + const res = await request(app).get('/en-GB/dataset'); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + filelist: [ + { + titles: [{ language: 'en-GB', title: 'Test Dataset 1' }], + dataset_id: 'BDC40218-AF89-424B-B86E-D21710BC92F1' + } + ] + }); + }); + }); + + describe('Display dataset object endpoints', () => { + test('Get a dataset returns 200 with a shallow object', async () => { + const dataset1 = await Dataset.findOneBy({ id: dataset1Id }); + if (!dataset1) { + throw new Error('Dataset not found'); + } + const dto = await DatasetDTO.fromDatasetComplete(dataset1); + const res = await request(app).get(`/en-GB/dataset/${dataset1Id}`); + expect(res.status).toBe(200); + expect(res.body).toEqual(dto); + }); + + test('Get a dataset returns 400 if an invalid ID is given', async () => { + const res = await request(app).get(`/en-GB/dataset/INVALID-ID`); + expect(res.status).toBe(400); + expect(res.body).toEqual({ message: 'Dataset ID is not valid' }); + }); + + test('Get a dataset returns 404 if a non-existant ID is given', async () => { + const res = await request(app).get(`/en-GB/dataset/8B9434D1-4807-41CD-8E81-228769671A07`); + expect(res.status).toBe(404); + }); + }); + + describe('Display dimension metadata endpoints', () => { + test('Get a dimension returns 200 with a shallow object', async () => { + const dimension = await Dimension.findOneBy({ id: dimension1Id }); + if (!dimension) { + throw new Error('Dataset not found'); + } + const dto = await DimensionDTO.fromDimension(dimension); + const res = await request(app).get(`/en-GB/dataset/${dataset1Id}/dimension/by-id/${dimension1Id}`); + expect(res.status).toBe(200); + expect(res.body).toEqual(dto); + }); + + test('Get a dimension returns 400 if an invalid ID is given', async () => { + const res = await request(app).get(`/en-GB/dataset/${dataset1Id}/dimension/by-id/INVALID-ID`); + expect(res.status).toBe(400); + expect(res.body).toEqual({ message: 'Dimension ID is not valid' }); + }); + + test('Get a dimension returns 404 if a non-existant ID is given', async () => { + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/dimension/by-id/8B9434D1-4807-41CD-8E81-228769671A07` + ); + expect(res.status).toBe(404); + }); + }); + + describe('Get revision metadata endpoints', () => { + test('Get a revision returns 200 with a shallow object', async () => { + const revision = await Revision.findOneBy({ id: revision1Id }); + if (!revision) { + throw new Error('Dataset not found'); + } + const dto = await RevisionDTO.fromRevision(revision); + const res = await request(app).get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}`); + expect(res.status).toBe(200); + expect(res.body).toEqual(dto); + }); + + test('Get revision returns 400 if an invalid ID is given', async () => { + const res = await request(app).get(`/en-GB/dataset/${dataset1Id}/revision/by-id/INVALID-ID`); + expect(res.status).toBe(400); + expect(res.body).toEqual({ message: 'Revision ID is not valid' }); + }); + + test('Get revision returns 404 if a ID is given', async () => { + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/revision/by-id/8B9434D1-4807-41CD-8E81-228769671A07` + ); + expect(res.status).toBe(404); + }); + }); + + describe('Get FileImport metadata endpoints', () => { + test('Get import returns 200 with object', async () => { + const imp = await FileImport.findOneBy({ id: import1Id }); + if (!imp) { + throw new Error('Import not found'); + } + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}` + ); + const expectedDTO = await ImportDTO.fromImport(imp); + expect(res.status).toBe(200); + expect(res.body).toEqual(expectedDTO); + }); + + test('Get import returns 400 if given an invalid ID', async () => { + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/IN-VALID-ID` + ); + expect(res.status).toBe(400); + expect(res.body).toEqual({ message: 'Import ID is not valid' }); + }); + + test('Get import returns 404 if given a missing ID', async () => { + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/8B9434D1-4807-41CD-8E81-228769671A07` + ); + expect(res.status).toBe(404); + }); + }); + + afterAll(async () => { + await dbManager.getDataSource().dropDatabase(); + }); +}); diff --git a/test/publisher-journey.test.ts b/test/publisher-journey.test.ts new file mode 100644 index 0000000..f8a2cb3 --- /dev/null +++ b/test/publisher-journey.test.ts @@ -0,0 +1,311 @@ +import path from 'path'; +import * as fs from 'fs'; + +import request from 'supertest'; + +import { DataLakeService } from '../src/controllers/datalake'; +import { BlobStorageService } from '../src/controllers/blob-storage'; +import app, { ENGLISH, WELSH, t, dbManager, databaseManager } from '../src/app'; +import { Dataset } from '../src/entities/dataset'; +import { DatasetInfo } from '../src/entities/dataset_info'; +import { DatasetDTO } from '../src/dtos/dataset-dto'; +import { ViewErrDTO } from '../src/dtos/view-dto'; +import { MAX_PAGE_SIZE, MIN_PAGE_SIZE } from '../src/controllers/csv-processor'; + +import { createFullDataset, createSmallDataset } from './helpers/test-helper'; +import { datasourceOptions } from './helpers/test-data-source'; + +DataLakeService.prototype.listFiles = jest + .fn() + .mockReturnValue([{ name: 'test-data-1.csv', path: 'test/test-data-1.csv', isDirectory: false }]); + +BlobStorageService.prototype.uploadFile = jest.fn(); + +DataLakeService.prototype.uploadFile = jest.fn(); + +const dataset1Id = 'BDC40218-AF89-424B-B86E-D21710BC92F1'; +const revision1Id = '85F0E416-8BD1-4946-9E2C-1C958897C6EF'; +const import1Id = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359'; +const dimension1Id = '2D7ACD0B-A46A-43F7-8A88-224CE97FC8B9'; + +describe('API Endpoints', () => { + beforeAll(async () => { + await databaseManager(datasourceOptions); + await dbManager.initializeDataSource(); + await createFullDataset(dataset1Id, revision1Id, import1Id, dimension1Id); + }); + + test('Return true test', async () => { + const dataset1 = await Dataset.findOneBy({ id: dataset1Id }); + if (!dataset1) { + throw new Error('Dataset not found'); + } + const dto = await DatasetDTO.fromDatasetComplete(dataset1); + expect(dto).toBe(dto); + }); + + test('Upload returns 400 if no file attached', async () => { + const err: ViewErrDTO = { + success: false, + dataset_id: undefined, + errors: [ + { + field: 'csv', + message: [ + { + lang: ENGLISH, + message: t('errors.no_csv_data', { lng: ENGLISH }) + }, + { + lang: WELSH, + message: t('errors.no_csv_data', { lng: WELSH }) + } + ], + tag: { + name: 'errors.no_csv_data', + params: {} + } + } + ] + }; + const res = await request(app).post('/en-GB/dataset').query({ filename: 'test-data-1.csv' }); + expect(res.status).toBe(400); + expect(res.body).toEqual(err); + }); + + test('Upload returns 400 if no title is given', async () => { + const err: ViewErrDTO = { + success: false, + dataset_id: undefined, + errors: [ + { + field: 'title', + message: [ + { + lang: ENGLISH, + message: t('errors.no_title', { lng: ENGLISH }) + }, + { + lang: WELSH, + message: t('errors.no_title', { lng: WELSH }) + } + ], + tag: { + name: 'errors.no_title', + params: {} + } + } + ] + }; + const csvfile = path.resolve(__dirname, `sample-csvs/test-data-1.csv`); + const res = await request(app).post('/en-GB/dataset').attach('csv', csvfile); + expect(res.status).toBe(400); + expect(res.body).toEqual(err); + }); + + test('Upload returns 201 if a file is attached', async () => { + const csvfile = path.resolve(__dirname, `sample-csvs/test-data-1.csv`); + + const res = await request(app) + .post('/en-GB/dataset') + .attach('csv', csvfile) + .field('title', 'Test Dataset 3') + .field('lang', 'en-GB'); + const datasetInfo = await DatasetInfo.findOneBy({ title: 'Test Dataset 3' }); + if (!datasetInfo) { + expect(datasetInfo).not.toBeNull(); + return; + } + const dataset = await datasetInfo.dataset; + const datasetDTO = await DatasetDTO.fromDatasetWithRevisionsAndImports(dataset); + expect(res.status).toBe(201); + expect(res.body).toEqual(datasetDTO); + await Dataset.remove(dataset); + }); + + test('Upload returns 500 if an error occurs with blob storage', async () => { + BlobStorageService.prototype.uploadFile = jest.fn().mockImplementation(() => { + throw new Error('Test error'); + }); + const csvfile = path.resolve(__dirname, `sample-csvs/test-data-1.csv`); + const res = await request(app) + .post('/en-GB/dataset') + .attach('csv', csvfile) + .field('title', 'Test Dataset 3') + .field('lang', 'en-GB'); + expect(res.status).toBe(500); + expect(res.body).toEqual({ message: 'Error uploading file' }); + }); + + test('Get file view returns 400 if page_number is too high', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile2); + BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile2Buffer); + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) + .query({ page_number: 20 }); + expect(res.status).toBe(400); + expect(res.body).toEqual({ + success: false, + dataset_id: dataset1Id, + errors: [ + { + field: 'page_number', + message: [ + { lang: ENGLISH, message: t('errors.page_number_to_high', { lng: ENGLISH, page_number: 6 }) }, + { lang: WELSH, message: t('errors.page_number_to_high', { lng: WELSH, page_number: 6 }) } + ], + tag: { + name: 'errors.page_number_to_high', + params: { page_number: 6 } + } + } + ] + }); + }); + + test('Get file view returns 400 if page_size is too high', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile2); + BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile2Buffer); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) + .query({ page_size: 1000 }); + expect(res.status).toBe(400); + expect(res.body).toEqual({ + success: false, + dataset_id: dataset1Id, + errors: [ + { + field: 'page_size', + message: [ + { + lang: ENGLISH, + message: t('errors.page_size', { + lng: ENGLISH, + max_page_size: MAX_PAGE_SIZE, + min_page_size: MIN_PAGE_SIZE + }) + }, + { + lang: WELSH, + message: t('errors.page_size', { + lng: WELSH, + max_page_size: MAX_PAGE_SIZE, + min_page_size: MIN_PAGE_SIZE + }) + } + ], + tag: { + name: 'errors.page_size', + params: { max_page_size: MAX_PAGE_SIZE, min_page_size: MIN_PAGE_SIZE } + } + } + ] + }); + }); + + test('Get file view returns 400 if page_size is too low', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFile2Buffer = fs.readFileSync(testFile2); + BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile2Buffer); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) + .query({ page_size: 1 }); + expect(res.status).toBe(400); + expect(res.body).toEqual({ + success: false, + dataset_id: dataset1Id, + errors: [ + { + field: 'page_size', + message: [ + { + lang: ENGLISH, + message: t('errors.page_size', { + lng: ENGLISH, + max_page_size: MAX_PAGE_SIZE, + min_page_size: MIN_PAGE_SIZE + }) + }, + { + lang: WELSH, + message: t('errors.page_size', { + lng: WELSH, + max_page_size: MAX_PAGE_SIZE, + min_page_size: MIN_PAGE_SIZE + }) + } + ], + tag: { + name: 'errors.page_size', + params: { max_page_size: MAX_PAGE_SIZE, min_page_size: MIN_PAGE_SIZE } + } + } + ] + }); + }); + + test('Get file from a revision and import rertunrs 200 and complete file data', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFileStream = fs.createReadStream(testFile2); + const testFile2Buffer = fs.readFileSync(testFile2); + BlobStorageService.prototype.getReadableStream = jest.fn().mockReturnValue(testFileStream); + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/raw` + ); + expect(res.status).toBe(200); + expect(res.text).toEqual(testFile2Buffer.toString()); + }); + + test('Get preview of an import returns 200 and correct page data', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFile1Buffer = fs.readFileSync(testFile2); + BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile1Buffer.toString()); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/${import1Id}/preview`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(200); + expect(res.body.current_page).toBe(2); + expect(res.body.total_pages).toBe(6); + expect(res.body.page_size).toBe(100); + expect(res.body.headers).toEqual([ + { index: 0, name: 'ID' }, + { index: 1, name: 'Text' }, + { index: 2, name: 'Number' }, + { index: 3, name: 'Date' } + ]); + expect(res.body.data[0]).toEqual(['101', 'GEYiRzLIFM', '774477', '2002-03-13']); + expect(res.body.data[99]).toEqual(['200', 'QhBxdmrUPb', '3256099', '2026-12-17']); + }); + + test('Get preview of an import returns 404 when a non-existant import is requested', async () => { + const res = await request(app).get( + `/en-GB/dataset/${dataset1Id}/revision/by-id/${revision1Id}/import/by-id/97C3F48F-127C-4317-B39C-87350F222310/preview` + ); + expect(res.status).toBe(404); + expect(res.body).toEqual({ message: 'Import not found.' }); + }); + + test('Delete a dataset actaully deletes the dataset', async () => { + const datasetID = crypto.randomUUID(); + const testDataset = await createSmallDataset(datasetID, crypto.randomUUID(), crypto.randomUUID()); + expect(testDataset).not.toBeNull(); + expect(testDataset.id).toBe(datasetID); + const datesetFromDb = await Dataset.findOneBy({ id: datasetID }); + expect(datesetFromDb).not.toBeNull(); + expect(datesetFromDb?.id).toBe(datasetID); + + const res = await request(app).delete(`/en-GB/dataset/${datasetID}`); + expect(res.status).toBe(204); + const dataset = await Dataset.findOneBy({ id: datasetID }); + expect(dataset).toBeNull(); + }); + + afterAll(async () => { + await dbManager.getDataSource().dropDatabase(); + }); +}); diff --git a/test/test-data-1.csv b/test/sample-csvs/test-data-1.csv similarity index 100% rename from test/test-data-1.csv rename to test/sample-csvs/test-data-1.csv diff --git a/test/test-data-2.csv b/test/sample-csvs/test-data-2.csv similarity index 100% rename from test/test-data-2.csv rename to test/sample-csvs/test-data-2.csv diff --git a/test/test-data-source.ts b/test/test-data-source.ts deleted file mode 100644 index fcb1181..0000000 --- a/test/test-data-source.ts +++ /dev/null @@ -1,25 +0,0 @@ -import 'reflect-metadata'; -import { DataSourceOptions } from 'typeorm'; -import * as dotenv from 'dotenv'; - -import { Dataset } from '../src/entities/dataset'; -import { DatasetInfo } from '../src/entities/dataset_info'; -import { Revision } from '../src/entities/revision'; -import { Import } from '../src/entities/import'; -import { CsvInfo } from '../src/entities/csv_info'; -import { Source } from '../src/entities/source'; -import { Dimension } from '../src/entities/dimension'; -import { DimensionInfo } from '../src/entities/dimension_info'; -import { User } from '../src/entities/user'; - -dotenv.config(); - -export const datasourceOptions: DataSourceOptions = { - name: 'default', - type: 'better-sqlite3', - database: ':memory:', - synchronize: true, - logging: false, - entities: [Dataset, DatasetInfo, Revision, Import, CsvInfo, Source, Dimension, DimensionInfo, User], - subscribers: [] -}; diff --git a/test/test-helper.ts b/test/test-helper.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test/view-dataset-contents.test.ts b/test/view-dataset-contents.test.ts new file mode 100644 index 0000000..1700602 --- /dev/null +++ b/test/view-dataset-contents.test.ts @@ -0,0 +1,170 @@ +import path from 'path'; +import * as fs from 'fs'; + +import request from 'supertest'; + +import { DataLakeService } from '../src/controllers/datalake'; +import { BlobStorageService } from '../src/controllers/blob-storage'; +import app, { ENGLISH, WELSH, i18n, dbManager, databaseManager } from '../src/app'; +import { Revision } from '../src/entities/revision'; +import { FileImport } from '../src/entities/import_file'; + +import { createFullDataset, createSmallDataset } from './helpers/test-helper'; +import { datasourceOptions } from './helpers/test-data-source'; + +DataLakeService.prototype.listFiles = jest + .fn() + .mockReturnValue([{ name: 'test-data-1.csv', path: 'test/test-data-1.csv', isDirectory: false }]); + +BlobStorageService.prototype.uploadFile = jest.fn(); + +DataLakeService.prototype.uploadFile = jest.fn(); + +const dataset1Id = 'BDC40218-AF89-424B-B86E-D21710BC92F1'; +const revision1Id = '85F0E416-8BD1-4946-9E2C-1C958897C6EF'; +const import1Id = 'FA07BE9D-3495-432D-8C1F-D0FC6DAAE359'; +const dimension1Id = '2D7ACD0B-A46A-43F7-8A88-224CE97FC8B9'; + +describe('API Endpoints for viewing the contents of a dataset', () => { + beforeAll(async () => { + await databaseManager(datasourceOptions); + await dbManager.initializeDataSource(); + await createFullDataset(dataset1Id, revision1Id, import1Id, dimension1Id); + }); + + test('Get file from a dataset, stored in blobStorage, returns 200 and complete file data', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFile1Buffer = fs.readFileSync(testFile2); + BlobStorageService.prototype.readFile = jest.fn().mockReturnValue(testFile1Buffer.toString()); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/view`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(200); + expect(res.body.current_page).toBe(2); + expect(res.body.total_pages).toBe(6); + expect(res.body.page_size).toBe(100); + expect(res.body.headers).toEqual([ + { index: 0, name: 'ID' }, + { index: 1, name: 'Text' }, + { index: 2, name: 'Number' }, + { index: 3, name: 'Date' } + ]); + expect(res.body.data[0]).toEqual(['101', 'GEYiRzLIFM', '774477', '2002-03-13']); + expect(res.body.data[99]).toEqual(['200', 'QhBxdmrUPb', '3256099', '2026-12-17']); + }); + + test('Get file from a dataset, stored in datalake, returns 200 and complete file data', async () => { + const testFile2 = path.resolve(__dirname, `sample-csvs/test-data-2.csv`); + const testFile1Buffer = fs.readFileSync(testFile2); + DataLakeService.prototype.downloadFile = jest.fn().mockReturnValue(testFile1Buffer.toString()); + const fileImport = await FileImport.findOneBy({ id: import1Id }); + if (!fileImport) { + throw new Error('Import not found'); + } + fileImport.location = 'Datalake'; + await FileImport.save(fileImport); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/view`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(200); + expect(res.body.current_page).toBe(2); + expect(res.body.total_pages).toBe(6); + expect(res.body.page_size).toBe(100); + expect(res.body.headers).toEqual([ + { index: 0, name: 'ID' }, + { index: 1, name: 'Text' }, + { index: 2, name: 'Number' }, + { index: 3, name: 'Date' } + ]); + expect(res.body.data[0]).toEqual(['101', 'GEYiRzLIFM', '774477', '2002-03-13']); + expect(res.body.data[99]).toEqual(['200', 'QhBxdmrUPb', '3256099', '2026-12-17']); + }); + + test('Get file from a dataset, stored in an unknown location, returns 500 and an error message', async () => { + const fileImport = await FileImport.findOneBy({ id: import1Id }); + if (!fileImport) { + throw new Error('Import not found'); + } + fileImport.location = 'Unknown'; + await FileImport.save(fileImport); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/view`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(500); + expect(res.body).toEqual({ message: 'Import location not supported.' }); + }); + + test('Get file from a dataset, stored in blob storage, returns 500 if the file is empty and an error message', async () => { + const fileImport = await FileImport.findOneBy({ id: import1Id }); + if (!fileImport) { + throw new Error('Import not found'); + } + fileImport.location = 'BlobStorage'; + await FileImport.save(fileImport); + BlobStorageService.prototype.readFile = jest.fn().mockRejectedValue(new Error('File is empty')); + + const res = await request(app) + .get(`/en-GB/dataset/${dataset1Id}/view`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(500); + expect(res.body).toEqual({ + success: false, + errors: [ + { + field: 'csv', + message: [ + { lang: ENGLISH, message: i18n.t('errors.download_from_blobstorage', { lng: ENGLISH }) }, + { lang: WELSH, message: i18n.t('errors.download_from_blobstorage', { lng: WELSH }) } + ], + tag: { name: 'errors.download_from_blobstorage', params: {} } + } + ], + dataset_id: dataset1Id + }); + }); + + test('Get a dataset view returns 500 if there is no revision on the dataset', async () => { + const removeRevisionDatasetID = crypto.randomUUID(); + const revisionID = crypto.randomUUID(); + await createSmallDataset(removeRevisionDatasetID, revisionID, crypto.randomUUID()); + const revision = await Revision.findOneBy({ id: revisionID }); + if (!revision) { + throw new Error('Revision not found... Either it was not created or the test is broken'); + } + await Revision.remove(revision); + const res = await request(app) + .get(`/en-GB/dataset/${removeRevisionDatasetID}/view`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(500); + expect(res.body).toEqual({ message: 'No revision found for dataset' }); + }); + + test('Get a dataset view returns 500 if there is no import on the dataset', async () => { + const removeRevisionDatasetID = crypto.randomUUID(); + const importId = crypto.randomUUID(); + await createSmallDataset(removeRevisionDatasetID, crypto.randomUUID(), importId); + const fileImport = await FileImport.findOneBy({ id: importId }); + if (!fileImport) { + throw new Error('Revision not found... Either it was not created or the test is broken'); + } + await FileImport.remove(fileImport); + const res = await request(app) + .get(`/en-GB/dataset/${removeRevisionDatasetID}/view`) + .query({ page_number: 2, page_size: 100 }); + expect(res.status).toBe(500); + expect(res.body).toEqual({ message: 'No import record found for dataset' }); + }); + + test('Get file view returns 400 when a not valid UUID is supplied', async () => { + const res = await request(app).get(`/en-GB/dataset/NOT-VALID-ID`); + expect(res.status).toBe(400); + expect(res.body).toEqual({ message: 'Dataset ID is not valid' }); + }); + + afterAll(async () => { + await dbManager.getDataSource().dropDatabase(); + }); +});