Updated test code with a few bug fixes thrown in

src/controllers/csv-processor.ts

@@ -11,7 +11,7 @@ import { CSVHeader, ViewStream, ViewDTO, ViewErrDTO } from '../dtos/view-dto';
 import { Dataset } from '../entities/dataset';
 import { Revision } from '../entities/revision';
 import { Source } from '../entities/source';
-import { Import } from '../entities/import';
+import { FileImport } from '../entities/import_file';
 
 import { BlobStorageService } from './blob-storage';
 import { DataLakeService } from './datalake';
@@ -119,13 +119,13 @@ function validateParams(page_number: number, max_page_number: number, page_size:
     return errors;
 }
 
-export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: string): Promise<Import> => {
+export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: string): Promise<FileImport> => {
     const blobStorageService = new BlobStorageService();
     if (!fileStream) {
         logger.error('No buffer to upload to blob storage');
         throw new Error('No buffer to upload to blob storage');
     }
-    const importRecord = new Import();
+    const importRecord = new FileImport();
     importRecord.id = randomUUID();
     importRecord.mime_type = filetype;
     const extension = filetype === 'text/csv' ? 'csv' : 'zip';
@@ -141,7 +141,7 @@ export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: str
         await blobStorageService.uploadFile(`${importRecord.filename}`, fileStream);
         const resolvedHash = await promisedHash;
         if (resolvedHash) importRecord.hash = resolvedHash;
-        importRecord.uploaded_at = new Date(Date.now());
+        importRecord.uploadedAt = new Date(Date.now());
         importRecord.type = 'Draft';
         importRecord.location = 'BlobStorage';
         return importRecord;
@@ -151,9 +151,9 @@ export const uploadCSVToBlobStorage = async (fileStream: Readable, filetype: str
     }
 };
 
-export const uploadCSVBufferToBlobStorage = async (fileBuffer: Buffer, filetype: string): Promise<Import> => {
+export const uploadCSVBufferToBlobStorage = async (fileBuffer: Buffer, filetype: string): Promise<FileImport> => {
     const fileStream = Readable.from(fileBuffer);
-    const importRecord: Import = await uploadCSVToBlobStorage(fileStream, filetype);
+    const importRecord: FileImport = await uploadCSVToBlobStorage(fileStream, filetype);
     return importRecord;
 };
 
@@ -172,7 +172,7 @@ async function processCSVData(
     page: number,
     size: number,
     dataset: Dataset,
-    importObj: Import
+    importObj: FileImport
 ): Promise<ViewDTO | ViewErrDTO> {
     const dataArray: Array<Array<string>> = (await parse(buffer, {
         delimiter: ','
@@ -241,7 +241,10 @@ async function processCSVData(
     };
 }
 
-export const getFileFromDataLake = async (dataset: Dataset, importObj: Import): Promise<ViewStream | ViewErrDTO> => {
+export const getFileFromDataLake = async (
+    dataset: Dataset,
+    importObj: FileImport
+): Promise<ViewStream | ViewErrDTO> => {
     const datalakeService = new DataLakeService();
     let stream: Readable;
     try {
@@ -268,7 +271,7 @@ export const getFileFromDataLake = async (dataset: Dataset, importObj: Import):
 
 export const processCSVFromDatalake = async (
     dataset: Dataset,
-    importObj: Import,
+    importObj: FileImport,
     page: number,
     size: number
 ): Promise<ViewErrDTO | ViewDTO> => {
@@ -296,7 +299,10 @@ export const processCSVFromDatalake = async (
     return processCSVData(buff, page, size, dataset, importObj);
 };
 
-export const getFileFromBlobStorage = async (dataset: Dataset, importObj: Import): Promise<ViewStream | ViewErrDTO> => {
+export const getFileFromBlobStorage = async (
+    dataset: Dataset,
+    importObj: FileImport
+): Promise<ViewStream | ViewErrDTO> => {
     const blobStoageService = new BlobStorageService();
     let stream: Readable;
     try {
@@ -312,7 +318,7 @@ export const getFileFromBlobStorage = async (dataset: Dataset, importObj: Import
                         { lang: ENGLISH, message: t('errors.download_from_blobstorage', { lng: ENGLISH }) },
                         { lang: WELSH, message: t('errors.download_from_blobstorage', { lng: WELSH }) }
                     ],
-                    tag: { name: 'errors.download_from_datalake', params: {} }
+                    tag: { name: 'errors.download_from_blobstorage', params: {} }
                 }
             ],
             dataset_id: dataset.id
@@ -326,7 +332,7 @@ export const getFileFromBlobStorage = async (dataset: Dataset, importObj: Import
 
 export const processCSVFromBlobStorage = async (
     dataset: Dataset,
-    importObj: Import,
+    importObj: FileImport,
     page: number,
     size: number
 ): Promise<ViewErrDTO | ViewDTO> => {
@@ -345,7 +351,7 @@ export const processCSVFromBlobStorage = async (
                         { lang: ENGLISH, message: t('errors.download_from_blobstorage', { lng: ENGLISH }) },
                         { lang: WELSH, message: t('errors.download_from_blobstorage', { lng: WELSH }) }
                     ],
-                    tag: { name: 'errors.download_from_datalake', params: {} }
+                    tag: { name: 'errors.download_from_blobstorage', params: {} }
                 }
             ],
             dataset_id: dataset.id
@@ -354,7 +360,7 @@ export const processCSVFromBlobStorage = async (
     return processCSVData(buff, page, size, dataset, importObj);
 };
 
-export const moveFileToDataLake = async (importObj: Import) => {
+export const moveFileToDataLake = async (importObj: FileImport) => {
     const blobStorageService = new BlobStorageService();
     const datalakeService = new DataLakeService();
     try {
@@ -367,7 +373,7 @@ export const moveFileToDataLake = async (importObj: Import) => {
     }
 };
 
-export const createSources = async (importObj: Import): Promise<RevisionDTO> => {
+export const createSources = async (importObj: FileImport): Promise<RevisionDTO> => {
     const revision: Revision = await importObj.revision;
     const dataset: Dataset = await revision.dataset;
     let fileView: ViewDTO | ViewErrDTO;
@@ -395,7 +401,7 @@ export const createSources = async (importObj: Import): Promise<RevisionDTO> =>
         sources.push(source);
         source.save();
     });
-    const saveImport = await Import.findOneBy({ id: importObj.id });
+    const saveImport = await FileImport.findOneBy({ id: importObj.id });
     if (!saveImport) {
         throw new Error('Import not found');
     }

src/database-manager.ts

@@ -5,7 +5,7 @@ import { Logger } from 'pino';
 import { Dataset } from './entities/dataset';
 import { DatasetInfo } from './entities/dataset_info';
 import { Revision } from './entities/revision';
-import { Import } from './entities/import';
+import { FileImport } from './entities/import_file';
 import { CsvInfo } from './entities/csv_info';
 import { Source } from './entities/source';
 import { Dimension } from './entities/dimension';
@@ -39,7 +39,7 @@ class DatabaseManager {
     async initializeDataSource() {
         this.dataSource = new DataSource({
             ...this.datasourceOptions,
-            entities: [Dataset, DatasetInfo, Revision, Import, CsvInfo, Source, Dimension, DimensionInfo, User]
+            entities: [Dataset, DatasetInfo, Revision, FileImport, CsvInfo, Source, Dimension, DimensionInfo, User]
         });
 
         await this.dataSource

src/dtos/dataset-dto.ts

@@ -2,7 +2,7 @@ import { Dataset } from '../entities/dataset';
 import { Dimension } from '../entities/dimension';
 import { DimensionInfo } from '../entities/dimension_info';
 import { Source } from '../entities/source';
-import { Import } from '../entities/import';
+import { FileImport } from '../entities/import_file';
 import { Revision } from '../entities/revision';
 import { DatasetInfo } from '../entities/dataset_info';
 
@@ -88,20 +88,20 @@ export class ImportDTO {
     mime_type: string;
     filename: string;
     hash: string;
-    uploaded_at: string;
+    uploadedAt: string;
     type: string;
     location: string;
     sources?: SourceDTO[];
 
-    static async fromImport(importEntity: Import): Promise<ImportDTO> {
+    static async fromImport(importEntity: FileImport): Promise<ImportDTO> {
         const dto = new ImportDTO();
         dto.id = importEntity.id;
         const revision = await importEntity.revision;
         dto.revision_id = revision.id;
         dto.mime_type = importEntity.mime_type;
         dto.filename = importEntity.filename;
         dto.hash = importEntity.hash;
-        dto.uploaded_at = importEntity.uploaded_at?.toISOString() || '';
+        dto.uploadedAt = importEntity.uploadedAt?.toISOString() || '';
         dto.type = importEntity.type;
         dto.location = importEntity.location;
         dto.sources = await Promise.all(
@@ -146,14 +146,14 @@ export class RevisionDTO {
         revDto.approved_by = (await revision.approvedBy)?.name || undefined;
         revDto.created_by = (await revision.createdBy).name;
         revDto.imports = await Promise.all(
-            (await revision.imports).map(async (imp: Import) => {
+            (await revision.imports).map(async (imp: FileImport) => {
                 const impDto = new ImportDTO();
                 impDto.id = imp.id;
                 impDto.revision_id = (await imp.revision).id;
                 impDto.mime_type = imp.mime_type;
                 impDto.filename = imp.filename;
                 impDto.hash = imp.hash;
-                impDto.uploaded_at = imp.uploaded_at.toISOString();
+                impDto.uploadedAt = imp.uploadedAt.toISOString();
                 impDto.type = imp.type;
                 impDto.location = imp.location;
                 impDto.sources = await Promise.all(
@@ -262,14 +262,14 @@ export class DatasetDTO {
                 revDto.approved_by = (await revision.approvedBy)?.name || undefined;
                 revDto.created_by = (await revision.createdBy)?.name;
                 revDto.imports = await Promise.all(
-                    (await revision.imports).map(async (imp: Import) => {
+                    (await revision.imports).map(async (imp: FileImport) => {
                         const impDto = new ImportDTO();
                         impDto.id = imp.id;
                         impDto.revision_id = (await imp.revision).id;
                         impDto.mime_type = imp.mime_type;
                         impDto.filename = imp.filename;
                         impDto.hash = imp.hash;
-                        impDto.uploaded_at = imp.uploaded_at.toISOString();
+                        impDto.uploadedAt = imp.uploadedAt.toISOString();
                         impDto.type = imp.type;
                         impDto.location = imp.location;
                         impDto.sources = await Promise.all(
@@ -356,13 +356,13 @@ export class DatasetDTO {
                 revDto.approved_by = (await revision.approvedBy)?.name || undefined;
                 revDto.created_by = (await revision.createdBy)?.name;
                 revDto.imports = await Promise.all(
-                    (await revision.imports).map((imp: Import) => {
+                    (await revision.imports).map((imp: FileImport) => {
                         const impDto = new ImportDTO();
                         impDto.id = imp.id;
                         impDto.mime_type = imp.mime_type;
                         impDto.filename = imp.filename;
                         impDto.hash = imp.hash;
-                        impDto.uploaded_at = imp.uploaded_at.toISOString();
+                        impDto.uploadedAt = imp.uploadedAt.toISOString();
                         impDto.type = imp.type;
                         impDto.location = imp.location;
                         return impDto;

src/entities/csv_info.ts

@@ -1,7 +1,7 @@
 import { Entity, PrimaryColumn, Column, BaseEntity, ManyToOne, JoinColumn } from 'typeorm';
 
 // eslint-disable-next-line import/no-cycle
-import { Import } from './import';
+import { FileImport } from './import_file';
 
 @Entity()
 export class CsvInfo extends BaseEntity {
@@ -17,10 +17,10 @@ export class CsvInfo extends BaseEntity {
     @Column({ type: 'varchar', length: 2 })
     linebreak: string;
 
-    @ManyToOne(() => Import, (importEntity) => importEntity.csvInfo, {
+    @ManyToOne(() => FileImport, (importEntity) => importEntity.csvInfo, {
         onDelete: 'CASCADE',
         orphanedRowAction: 'delete'
     })
     @JoinColumn({ name: 'import_id' })
-    import: Promise<Import>;
+    import: Promise<FileImport>;
 }

src/entities/import.ts → src/entities/import_file.ts

@@ -16,8 +16,8 @@ import { CsvInfo } from './csv_info';
 // eslint-disable-next-line import/no-cycle
 import { Source } from './source';
 
-@Entity()
-export class Import extends BaseEntity {
+@Entity({ name: 'file_import', orderBy: { uploadedAt: 'ASC' } })
+export class FileImport extends BaseEntity {
     @PrimaryGeneratedColumn('uuid')
     id: string;
 
@@ -43,8 +43,8 @@ export class Import extends BaseEntity {
     @Column({ type: 'varchar', length: 255 })
     hash: string;
 
-    @CreateDateColumn()
-    uploaded_at: Date;
+    @CreateDateColumn({ name: 'uploaded_at' })
+    uploadedAt: Date;
 
     @Column({
         type: process.env.NODE_ENV === 'test' ? 'text' : 'enum',
@@ -55,7 +55,7 @@ export class Import extends BaseEntity {
 
     @Column({
         type: process.env.NODE_ENV === 'test' ? 'text' : 'enum',
-        enum: ['BlobStorage', 'Datalake'],
+        enum: ['BlobStorage', 'Datalake', 'Unknown'],
         nullable: false
     })
     location: string;

src/entities/revision.ts

@@ -15,7 +15,7 @@ import { User } from './user';
 // eslint-disable-next-line import/no-cycle
 import { Source } from './source';
 // eslint-disable-next-line import/no-cycle
-import { Import } from './import';
+import { FileImport } from './import_file';
 
 interface RevisionInterface {
     id: string;
@@ -28,10 +28,10 @@ interface RevisionInterface {
     approvalDate: Date;
     approvedBy: Promise<User>;
     createdBy: Promise<User>;
-    imports: Promise<Import[]>;
+    imports: Promise<FileImport[]>;
 }
 
-@Entity()
+@Entity({ name: 'revision', orderBy: { creationDate: 'ASC' } })
 export class Revision extends BaseEntity implements RevisionInterface {
     @PrimaryGeneratedColumn('uuid')
     id: string;
@@ -79,10 +79,10 @@ export class Revision extends BaseEntity implements RevisionInterface {
     })
     sources: Promise<Source[]>;
 
-    @OneToMany(() => Import, (importEntity) => importEntity.revision, {
+    @OneToMany(() => FileImport, (importEntity) => importEntity.revision, {
         cascade: true
     })
-    imports: Promise<Import[]>;
+    imports: Promise<FileImport[]>;
 
     @ManyToOne(() => User, { nullable: true })
     @JoinColumn({ name: 'approved_by' })

src/entities/source.ts

@@ -3,7 +3,7 @@ import { Entity, PrimaryGeneratedColumn, Column, BaseEntity, ManyToOne, JoinColu
 // eslint-disable-next-line import/no-cycle
 import { Dimension } from './dimension';
 // eslint-disable-next-line import/no-cycle
-import { Import } from './import';
+import { FileImport } from './import_file';
 // eslint-disable-next-line import/no-cycle
 import { Revision } from './revision';
 import { SourceType } from './source_type';
@@ -20,13 +20,13 @@ export class Source extends BaseEntity {
     @JoinColumn({ name: 'dimension_id' })
     dimension: Promise<Dimension>;
 
-    @ManyToOne(() => Import, (importEntity) => importEntity.sources, {
+    @ManyToOne(() => FileImport, (importEntity) => importEntity.sources, {
         nullable: false,
         onDelete: 'CASCADE',
         orphanedRowAction: 'delete'
     })
     @JoinColumn({ name: 'import_id' })
-    import: Promise<Import>;
+    import: Promise<FileImport>;
 
     @ManyToOne(() => Revision, {
         onDelete: 'CASCADE',

src/resources/locales/cy-GB.json

@@ -43,7 +43,8 @@
         "page_number_to_high": "Rhaid i rif y dudalen fod yn llai na neu'n hafal i {{page_number}} (GPT)",
         "page_number_to_low": "Rhaid i rif y dudalen fod yn fwy na 0 (GPT)",
         "no_datafile": "Dim ffeil data ynghlwm wrth Set Ddata (GPT)",
-        "download_from_datalake": "Gwall wrth lawrlwytho ffeil o'r llyn data (GPT)"
+        "download_from_datalake": "Gwall wrth lawrlwytho ffeil o'r llyn data (GPT)",
+        "download_from_blobstorage": "Gwall wrth lawrlwytho ffeil o’r storfa blob (GPT)"
 	},
     "dimensionInfo": {
         "footnotes": {

src/resources/locales/en-GB.json

@@ -38,6 +38,7 @@
         "page_number_to_low": "Page number must be greater than 0",
         "no_datafile": "No datafile attached to Dataset",
         "download_from_datalake": "Error downloading file from datalake",
+        "download_from_blobstorage": "Error downloading file from blob storage",
         "upload": {
             "no_title": "No title for the dataset has been provided",
             "no-internal-name": "No internal name for the dataset has been provided",