diff --git a/src/@types/express-session/index.d.ts b/src/@types/express-session/index.d.ts index 50943ef..21e0fdb 100644 --- a/src/@types/express-session/index.d.ts +++ b/src/@types/express-session/index.d.ts @@ -1,8 +1,10 @@ import 'express-session'; +import { DimensionPatchDto } from '../../dtos/dimension-patch-dto'; import { ViewError } from '../../dtos/view-error'; declare module 'express-session' { interface SessionData { errors: ViewError[] | undefined; + dimensionPatch: DimensionPatchDto | undefined; } } diff --git a/src/controllers/publish.ts b/src/controllers/publish.ts index 2dcf565..3115a9e 100644 --- a/src/controllers/publish.ts +++ b/src/controllers/publish.ts @@ -32,7 +32,7 @@ import { } from '../validators'; import { ViewError } from '../dtos/view-error'; import { logger } from '../utils/logger'; -import { ViewDTO } from '../dtos/view-dto'; +import { ViewDTO, ViewErrDTO } from '../dtos/view-dto'; import { SourceType } from '../enums/source-type'; import { FactTableInfoDto } from '../dtos/fact-table-info'; import { SourceAssignmentDTO } from '../dtos/source-assignment-dto'; @@ -53,9 +53,13 @@ import { fileMimeTypeHandler } from '../utils/file-mimetype-handler'; import { TopicDTO } from '../dtos/topic'; import { DatasetTopicDTO } from '../dtos/dataset-topic'; import { nestTopics } from '../utils/nested-topics'; -import { ApiException } from '../exceptions/api.exception'; import { OrganisationDTO } from '../dtos/organisation'; import { TeamDTO } from '../dtos/team'; +import { DimensionType } from '../enums/dimension-type'; +import { DimensionPatchDto } from '../dtos/dimension-patch-dto'; +import { ApiException } from '../exceptions/api.exception'; +import { DimensionInfoDTO } from '../dtos/dimension-info'; +import { YearType } from '../enums/year-type'; export const start = (req: Request, res: Response, next: NextFunction) => { res.render('publish/start'); @@ -256,10 +260,11 @@ export const sources = async (req: Request, res: Response, next: NextFunction) = export const taskList = async (req: Request, res: Response, next: NextFunction) => { try { const datasetTitle = singleLangDataset(res.locals.dataset, req.language).datasetInfo?.title; + const dimensions = singleLangDataset(res.locals.dataset, req.language).dimensions; const taskList: TaskListState = await req.swapi.getTaskList(res.locals.datasetId); - res.render('publish/tasklist', { datasetTitle, taskList, statusToColour }); + res.render('publish/tasklist', { datasetTitle, taskList, dimensions, statusToColour }); } catch (err) { - logger.error('Failed to get tasklist', err); + logger.error(`Failed to get tasklist with the following error: ${err}`); next(new NotFoundException()); } }; @@ -268,6 +273,591 @@ export const redirectToTasklist = (req: Request, res: Response) => { res.redirect(req.buildUrl(`/publish/${req.params.datasetId}/tasklist`, req.language)); }; +export const fetchDimensionPreview = async (req: Request, res: Response, next: NextFunction) => { + try { + const dimension = singleLangDataset(res.locals.dataset, req.language).dimensions?.find( + (dim) => dim.id === req.params.dimensionId + ); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + const dataPreview = await req.swapi.getDimensionPreview(res.locals.dataset.id, dimension.id); + res.render('publish/dimension-chooser', { ...dataPreview }); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +// I know this is the same as the `fetchDimensionPreview` however longer term +// I'd like to introduce a sniffer to sniff the time time dimension which could +// reduce the questions we need to ask date identification. +export const fetchTimeDimensionPreview = async (req: Request, res: Response, next: NextFunction) => { + try { + const dimension = singleLangDataset(res.locals.dataset, req.language).dimensions?.find( + (dim) => dim.id === req.params.dimensionId + ); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + if (req.method === 'POST') { + switch (req.body.dimensionType) { + case 'time_period': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time`, + req.language + ) + ); + break; + case 'time_point': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/point-in-time`, + req.language + ) + ); + break; + } + return; + } + + const dataPreview = await req.swapi.getDimensionPreview(res.locals.dataset.id, dimension.id); + res.render('publish/time-chooser', { ...dataPreview, dimension }); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const yearTypeChooser = async (req: Request, res: Response, next: NextFunction) => { + try { + const dimension = singleLangDataset(res.locals.dataset, req.language).dimensions?.find( + (dim) => dim.id === req.params.dimensionId + ); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + if (req.method === 'POST') { + if (req.body.yearType === 'calendar') { + req.session.dimensionPatch = { + dimension_type: DimensionType.TimePeriod, + date_type: req.body.yearType, + year_format: 'YYYY' + }; + req.session.save(); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/period-type`, + req.language + ) + ); + return; + } + req.session.dimensionPatch = { + dimension_type: DimensionType.TimePeriod, + date_type: req.body.yearType + }; + req.session.save(); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/year-format`, + req.language + ) + ); + return; + } + + res.render('publish/year-type'); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const yearFormat = async (req: Request, res: Response, next: NextFunction) => { + try { + const dimension = singleLangDataset(res.locals.dataset, req.language).dimensions?.find( + (dim) => dim.id === req.params.dimensionId + ); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + if (req.method === 'POST') { + if (!req.session.dimensionPatch) { + logger.error('Failed to find patch information in the session.'); + throw new Error('Year type not set in previous step'); + } + req.session.dimensionPatch.year_format = req.body.yearType; + req.session.save(); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/period-type`, + req.language + ) + ); + return; + } + + res.render('publish/year-format'); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const periodType = async (req: Request, res: Response, next: NextFunction) => { + try { + const dataset = singleLangDataset(res.locals.dataset, req.language); + const dimension = dataset.dimensions?.find((dim) => dim.id === req.params.dimensionId); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + const patchRequest = req.session.dimensionPatch; + if (!patchRequest) { + logger.error('Failed to find patch information in the session'); + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/`, + req.language + ); + return; + } + + if (req.method === 'POST') { + switch (req.body.periodType) { + case 'years': + try { + const previewData = await req.swapi.patchDimension(dataset.id, dimension.id, patchRequest); + logger.debug('Matching complete for year... Redirecting to review.'); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/review`, + req.language + ) + ); + return; + } catch (err) { + const error = err as ApiException; + logger.debug(`Error is: ${JSON.stringify(error, null, 2)}`); + if (error.status === 400) { + res.status(400); + logger.error('Date dimension had inconsistent formats than supplied by the user.', err); + const failurePreview = JSON.parse(error.body as string) as ViewErrDTO; + res.render('publish/period-match-failure', { ...failurePreview, patchRequest, dimension }); + return; + } + logger.error('Something went wrong other than not matching'); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/`, + req.language + ) + ); + return; + } + case 'quarters': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/quarters`, + req.language + ) + ); + return; + case 'months': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/months`, + req.language + ) + ); + return; + } + } + res.render('publish/period-type'); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const quarterChooser = async (req: Request, res: Response, next: NextFunction) => { + try { + const dataset = singleLangDataset(res.locals.dataset, req.language); + const dimension = dataset.dimensions?.find((dim) => dim.id === req.params.dimensionId); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + let quarterTotals = false; + if (req.session.dimensionPatch?.month_format) { + quarterTotals = true; + } + + if (req.method === 'POST') { + const patchRequest = req.session.dimensionPatch; + if (!patchRequest) { + res.redirect(`/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}`); + return; + } + patchRequest.quarter_format = req.body.quarterType; + if (req.body.fifthQuarter) { + patchRequest.fifth_quarter = true; + } + try { + const previewData = await req.swapi.patchDimension(dataset.id, dimension.id, patchRequest); + // eslint-disable-next-line require-atomic-updates + req.session.dimensionPatch = undefined; + req.session.save(); + res.redirect(`/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/review`); + return; + } catch (err) { + const error = err as ApiException; + logger.debug(`Error is: ${JSON.stringify(error, null, 2)}`); + if (error.status === 400) { + res.status(400); + logger.error('Date dimension had inconsistent formats than supplied by the user.', err); + const failurePreview = JSON.parse(error.body as string) as ViewErrDTO; + res.render('publish/period-match-failure', { ...failurePreview, patchRequest, dimension }); + return; + } + logger.error('Something went wrong other than not matching'); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/`, + req.language + ) + ); + return; + } + } + logger.debug(`Session dimensionPatch = ${JSON.stringify(req.session.dimensionPatch, null, 2)}`); + res.render('publish/quarter-format', { quarterTotals }); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const monthChooser = async (req: Request, res: Response, next: NextFunction) => { + try { + const dataset = singleLangDataset(res.locals.dataset, req.language); + const dimension = dataset.dimensions?.find((dim) => dim.id === req.params.dimensionId); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + if (req.method === 'POST') { + const patchRequest = req.session.dimensionPatch; + if (!patchRequest) { + logger.error('Failed to find dimension in dataset in session'); + res.redirect(`/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}`); + return; + } + patchRequest.month_format = req.body.monthFormat; + req.session.dimensionPatch = patchRequest; + logger.debug( + `Saving Dimension Patch to session with the following: ${JSON.stringify(patchRequest, null, 2)}` + ); + req.session.save(); + try { + const previewData = await req.swapi.patchDimension(dataset.id, dimension.id, patchRequest); + // eslint-disable-next-line require-atomic-updates + req.session.dimensionPatch = undefined; + req.session.save(); + res.redirect(`/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/review`); + return; + } catch (err) { + logger.debug(`There were rows which didn't match. Lets ask the user about quarterly totals.`); + res.redirect( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/period-of-time/quarters` + ); + return; + } + } + const dataPreview = await req.swapi.getDimensionPreview(res.locals.dataset.id, dimension.id); + res.render('publish/month-format', { ...dataPreview }); + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const periodReview = async (req: Request, res: Response, next: NextFunction) => { + try { + const dataset = singleLangDataset(res.locals.dataset, req.language); + const dimension = dataset.dimensions?.find((dim) => dim.id === req.params.dimensionId); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + let errors: ViewErrDTO | undefined; + + if (req.method === 'POST') { + switch (req.body.confirm) { + case 'true': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/dimension/${req.params.dimensionId}/name`, + req.language + ) + ); + break; + case 'goback': + try { + await req.swapi.resetDimension(dataset.id, dimension.id); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/`, + req.language + ) + ); + } catch (err) { + const error = err as ApiException; + logger.error( + `Something went wrong trying to reset the dimension with the following error: ${err}` + ); + errors = { + status: error.status || 500, + errors: [ + { + field: '', + message: { + key: 'errors.dimension_reset' + } + } + ], + dataset_id: req.params.datasetId + } as ViewErrDTO; + } + break; + } + return; + } + + const dataPreview = await req.swapi.getDimensionPreview(res.locals.dataset.id, dimension.id); + if (errors) { + res.status(errors.status || 500); + res.render('publish/time-chooser', { ...dataPreview, review: true, dimension, errors }); + } else { + res.render('publish/time-chooser', { ...dataPreview, review: true, dimension }); + } + } catch (err) { + logger.error('Failed to get dimension preview', err); + next(new NotFoundException()); + } +}; + +export const dimensionName = async (req: Request, res: Response, next: NextFunction) => { + try { + const dataset = singleLangDataset(res.locals.dataset, req.language); + const dimension = dataset.dimensions?.find((dim) => dim.id === req.params.dimensionId); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + let errors: ViewErrDTO | undefined; + const dimensionName = dimension.dimensionInfo?.name || ''; + if (req.method === 'POST') { + const updatedName = req.body.name; + if (!updatedName) { + logger.error('User failed to submit a name'); + res.status(400); + errors = { + status: 400, + errors: [ + { + field: 'name', + message: { + key: 'errors.no_name' + } + } + ], + dataset_id: req.params.datasetId + }; + res.status(400); + res.render('publish/dimension-name', { ...{ updatedName }, errors }); + return; + } + if (updatedName.length > 1024) { + logger.error(`Dimension name is to long... Dimension name is ${req.body.name.length} characters long.`); + errors = { + status: 400, + errors: [ + { + field: 'name', + message: { + key: 'errors.dimension.name_to_long' + } + } + ], + dataset_id: req.params.datasetId + }; + res.status(400); + res.render('publish/dimension-name', { ...{ updatedName }, errors }); + return; + } else if (updatedName.length < 1) { + logger.error(`Dimension name is too short.`); + errors = { + status: 400, + errors: [ + { + field: 'name', + message: { + key: 'errors.dimension.name_to_short' + } + } + ], + dataset_id: req.params.datasetId + }; + res.status(400); + res.render('publish/dimension-name', { ...{ updatedName }, errors }); + return; + } + const info: DimensionInfoDTO = { + name: updatedName, + language: req.language + }; + try { + await req.swapi.updateDimensionInfo(dataset.id, dimension.id, info); + res.redirect(req.buildUrl(`/publish/${req.params.datasetId}/tasklist`, req.language)); + return; + } catch (err) { + const error = err as ApiException; + logger.error(`Something went wrong trying to name the dimension with the following error: ${err}`); + errors = { + status: error.status || 500, + errors: [ + { + field: '', + message: { + key: 'errors.dimension.naming_failed' + } + } + ], + dataset_id: req.params.datasetId + }; + res.status(500); + res.render('publish/dimension-name', { ...{ dimensionName }, errors }); + return; + } + } + + res.render('publish/dimension-name', { ...{ dimensionName } }); + } catch (err) { + logger.error(`Failed to get dimension name with the following error: ${err}`); + next(new NotFoundException()); + } +}; + +export const pointInTimeChooser = async (req: Request, res: Response, next: NextFunction) => { + const dataset = singleLangDataset(res.locals.dataset, req.language); + const dimension = dataset.dimensions?.find((dim) => dim.id === req.params.dimensionId); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + if (req.method === 'POST') { + const patchRequest: DimensionPatchDto = { + date_format: req.body.dateFormat, + dimension_type: DimensionType.TimePoint, + date_type: YearType.PointInTime + }; + try { + const previewData = await req.swapi.patchDimension(dataset.id, dimension.id, patchRequest); + logger.debug('Matching complete for specific point in time... Redirecting to review.'); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/review`, + req.language + ) + ); + return; + } catch (err) { + const error = err as ApiException; + logger.debug(`Error is: ${JSON.stringify(error, null, 2)}`); + if (error.status === 400) { + res.status(400); + logger.error('Date dimension had inconsistent formats than supplied by the user.', err); + const failurePreview = JSON.parse(error.body as string) as ViewErrDTO; + res.render('publish/period-match-failure', { ...failurePreview, patchRequest, dimension }); + return; + } + logger.error('Something went wrong other than not matching'); + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/point-in-time/`, + req.language + ) + ); + return; + } + } + res.render('publish/point-in-time-chooser'); +}; + +export const periodOfTimeChooser = async (req: Request, res: Response, next: NextFunction) => { + const dimension = singleLangDataset(res.locals.dataset, req.language).dimensions?.find( + (dim) => dim.id === req.params.dimensionId + ); + if (!dimension) { + logger.error('Failed to find dimension in dataset'); + next(new NotFoundException()); + return; + } + + if (req.method === 'POST') { + switch (req.body.dimensionType) { + case 'years': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/point-in-time/years`, + req.language + ) + ); + return; + case 'quarters': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/point-in-time/quarters`, + req.language + ) + ); + return; + case 'months': + res.redirect( + req.buildUrl( + `/publish/${req.params.datasetId}/time-period/${req.params.dimensionId}/point-in-time/months`, + req.language + ) + ); + } + } + + res.render('publish/period-of-time-type'); +}; + export const changeData = async (req: Request, res: Response, next: NextFunction) => { if (req.method === 'POST') { if (req.body.change === 'table') { diff --git a/src/dtos/dimension-patch-dto.ts b/src/dtos/dimension-patch-dto.ts new file mode 100644 index 0000000..cbe3810 --- /dev/null +++ b/src/dtos/dimension-patch-dto.ts @@ -0,0 +1,15 @@ +import { DimensionType } from '../enums/dimension-type'; +import { YearType } from '../enums/year-type'; + +export interface DimensionPatchDto { + dimension_type: DimensionType; + dimension_title?: string; + lookup_join_column?: string; + reference_type?: string; + date_type?: YearType; + year_format?: string; + quarter_format?: string; + month_format?: string; + date_format?: string; + fifth_quarter?: boolean; +} diff --git a/src/dtos/dimension-state.ts b/src/dtos/dimension-state.ts index fdc4672..28e2b59 100644 --- a/src/dtos/dimension-state.ts +++ b/src/dtos/dimension-state.ts @@ -3,4 +3,5 @@ import { TaskStatus } from '../enums/task-status'; export interface DimensionState { name: string; status: TaskStatus; + type: string; } diff --git a/src/dtos/task-list-state.ts b/src/dtos/task-list-state.ts index a7f1e17..8e220d6 100644 --- a/src/dtos/task-list-state.ts +++ b/src/dtos/task-list-state.ts @@ -4,7 +4,7 @@ import { DimensionState } from './dimension-state'; export interface TaskListState { datatable: TaskStatus; - + measure?: DimensionState; dimensions: DimensionState[]; metadata: { diff --git a/src/dtos/view-dto.ts b/src/dtos/view-dto.ts index af7719c..429325e 100644 --- a/src/dtos/view-dto.ts +++ b/src/dtos/view-dto.ts @@ -23,6 +23,7 @@ export interface ViewErrDTO { } export interface ViewDTO { + status: number; dataset: DatasetDTO; fact_table: FactTableDto; current_page: number; diff --git a/src/enums/year-type.ts b/src/enums/year-type.ts new file mode 100644 index 0000000..f0e4212 --- /dev/null +++ b/src/enums/year-type.ts @@ -0,0 +1,8 @@ +export enum YearType { + Financial = 'financial', + Tax = 'tax', + Academic = 'academic', + Calendar = 'calendar', + Meteorological = 'meteorological', + PointInTime = 'pointInTime' +} diff --git a/src/exceptions/api.exception.ts b/src/exceptions/api.exception.ts index 6c2abe8..0a4cd7d 100644 --- a/src/exceptions/api.exception.ts +++ b/src/exceptions/api.exception.ts @@ -1,10 +1,12 @@ export class ApiException extends Error { constructor( public message: string, - public status?: number + public status?: number, + public body?: string | FormData ) { super(message); this.name = 'ApiException'; this.status = status; + this.body = body; } } diff --git a/src/i18n/en.json b/src/i18n/en.json index 35bb02c..41cb362 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -6,7 +6,8 @@ "continue": "Continue", "back": "Back", "cancel": "Cancel", - "upload_csv": "Upload CSV File" + "upload_csv": "Upload CSV File", + "preview": "Preview (opens in new tab)" }, "pagination": { "previous": "Previous", @@ -68,6 +69,122 @@ "lookup_table": "lookup tables for all relevant dimensions - using either common tables within this service or uploading them yourself", "metadata": "details about this dataset including description, quality and how the data was collected" }, + "dimension_name": { + "heading": "What should this dimension be called on the StatsWales website?", + "hint": "This should be:", + "concise": "concise and clearly explain what the dimension contains", + "language": "entered in the language in which you're viewing this service" + }, + "dimension_type_chooser": { + "heading": "Add reference data", + "subheading": "Dimension", + "question": "What kind of data does this dimension contain?", + "chooser": { + "age": "Age", + "ethnicity": "Ethnicity", + "geography": "Geography", + "religion": "Religion", + "sex_gender": "Sex and gender", + "lookup": "Something else" + } + }, + "time_dimension_chooser": { + "heading": "Set up dimension containing time", + "subheading": "Dates", + "showing": "A sample of {{rows}} of {{total}} rows.", + "question": "What kind of dates does the dimension contain?", + "chooser": { + "period": "Periods of time", + "period-hint": "For example, months or years for which data values apply to", + "point": "Specific points in time", + "point-hint": "For example, specific dates when data values were collected" + } + }, + "time_dimension_review": { + "heading": "Confirm the dates are correct", + "showing": "Showing a sample of {{rows}} of {{total}} rows.", + "confirm": "By continuing, you confirm the dates are correct.", + "go_back": "Change date format", + "column_headers": { + "date_code": "Date code", + "description": "Description", + "start_date": "Start", + "end_date": "End", + "date_type": "Date type" + }, + "year_type": { + "calendar_year": "Calendar Year", + "calendar_quarter": "Calendar Quarter", + "calendar_month": "Calendar Month", + "financial_year": "Financial Year", + "financial_quarter": "Financial Quarter", + "financial_month": "Financial Month", + "academic_year": "Academic Year", + "academic_quarter": "Academic Quarter", + "academic_month": "Academic Month", + "tax_year": "Tax Year", + "tax_quarter": "Tax Quarter", + "tax_month": "Tax Month", + "meteorological_year": "Meteorological Year", + "meteorological_quarter": "Meteorological Quarter", + "meteorological_month": "Meteorological Month" + } + }, + "period-type-chooser": { + "heading": "What are the periods of time?", + "subheading": "Dates", + "chooser": { + "years": "Years", + "quarters": "Quarters", + "months": "Months" + } + }, + "year_type": { + "heading": "What type of year does this dimension represent?", + "chooser": { + "calendar": "Calendar", + "calendar-hint": "1st January - 31st December", + "financial" : "Financial", + "financial-hint": "1st April - 31st March", + "tax": "Tax", + "tax-hint": "6th April - 5th April", + "academic": "Academic", + "academic-hint": "1st September - 31st August", + "meteorological": "Meteorological", + "meteorological-hint": "1st March - 28th/29th February" + } + }, + "year_format": { + "heading": "What format is used to represent years?", + "example": "For example, {{example}}" + }, + "quarter_format": { + "heading": "What format is used to represent quarters?", + "heading-alt": "What format is used for quarterly totals?", + "example": "For example, {{example}}", + "fifth_quarter": "Year totals are represented as a 5th quarter", + "no_quarterly_totals": "There are no quarterly totals" + }, + "month_format": { + "heading": "What format is used to represent months?", + "example": "For example, {{example}}" + }, + "period_match_failure": { + "heading": "Date formatting cannot be matched to the fact table", + "information": "{{failureCount}} of the dates in the fact table do not match the date formatting you have indicated. You should check:", + "formatting": "the date formatting in the fact table is correct", + "choices": "you indicated the correct date formats", + "supplied_format":"Indicated date format:", + "year_format": "Years: {{format}}", + "quarter_format": "Quarters: {{format}}", + "month_format": "Months: {{format}}", + "date_format": "Date: {{format}}", + "subheading": "Date formats that cannot be matched", + "upload_different_file": "Upload corrected or different data table", + "upload_different_file_warning": "(This will remove reference data from all dimensions)", + "try_different_format": "Indicate different date formatting", + "no_matches": "None of the dates matched the format supplied" + }, "title": { "heading": "What is the title of this dataset?", "appear": "This title will appear on the StatsWales website", @@ -400,6 +517,10 @@ "label": "Change what each column in the data table contains", "description": "This will remove reference data and notes for any columns you change that contain dimensions" } + }, + "point_in_time": { + "heading": "What date format is used?", + "example": "For example: {{example}}" } }, "view": { @@ -470,6 +591,15 @@ "try_later": "Please try later", "name_missing": "Dataset Name Missing", "dataset_missing": "No dataset found with this ID", + "dimension": { + "name_to_short": "The Dimension name is to short or missing", + "name_to_long": "Dimension name is limited to 1024 characters", + "naming_failed": "Something went wrong trying to set the name. Please try again" + }, + "title": { + "missing": "Either a title was not entered or there is a problem.", + "duplicate": "Title already exists" + }, "description": { "missing": "You need to provide a summary of this dataset" }, @@ -511,7 +641,8 @@ "no_csv_data": "No CSV data available" }, "not_found": "Page not found", - "server_error": "An unknown error occcured, please try again later" + "server_error": "An unknown error occcured, please try again later", + "dimension_reset": "Something went wrong trying to reset the dimension" }, "routes": { "healthcheck": "healthcheck", diff --git a/src/middleware/services.ts b/src/middleware/services.ts index 8e4ab84..d985353 100644 --- a/src/middleware/services.ts +++ b/src/middleware/services.ts @@ -1,4 +1,5 @@ import { NextFunction, Request, Response } from 'express'; +import { format, parseISO } from 'date-fns'; import { Locale } from '../enums/locale'; import { StatsWalesApi } from '../services/stats-wales-api'; @@ -14,6 +15,9 @@ export const initServices = (req: Request, res: Response, next: NextFunction): v req.buildUrl = localeUrl; // for controllers res.locals.buildUrl = localeUrl; // for templates res.locals.url = req.originalUrl; // Allows the passing through of the URL + res.locals.referrer = req.get('Referrer'); + res.locals.parseISO = parseISO; + res.locals.dateFormat = format; } next(); }; diff --git a/src/routes/publish.ts b/src/routes/publish.ts index aa7fb7f..3e18993 100644 --- a/src/routes/publish.ts +++ b/src/routes/publish.ts @@ -20,7 +20,18 @@ import { provideDesignation, provideTopics, providePublishDate, - provideOrganisation + provideOrganisation, + fetchDimensionPreview, + fetchTimeDimensionPreview, + pointInTimeChooser, + periodOfTimeChooser, + yearFormat, + quarterChooser, + monthChooser, + yearTypeChooser, + periodType, + periodReview, + dimensionName } from '../controllers/publish'; export const publish = Router(); @@ -48,6 +59,32 @@ publish.post('/:datasetId/sources', fetchDataset, upload.none(), sources); publish.get('/:datasetId/tasklist', fetchDataset, taskList); +/* Dimension creation routes */ +publish.get('/:datasetId/dimension-data-chooser/:dimensionId', fetchDataset, fetchDimensionPreview); +publish.post('/:datasetId/dimension-data-chooser/:dimensionId', fetchDataset, fetchDimensionPreview); + +publish.get('/:datasetId/time-period/:dimensionId', fetchDataset, fetchTimeDimensionPreview); +publish.post('/:datasetId/time-period/:dimensionId', fetchDataset, fetchTimeDimensionPreview); +publish.get('/:datasetId/time-period/:dimensionId/point-in-time', fetchDataset, pointInTimeChooser); +publish.post('/:datasetId/time-period/:dimensionId/point-in-time', fetchDataset, pointInTimeChooser); + +// Period of time flow) +publish.get('/:datasetId/time-period/:dimensionId/period-of-time', fetchDataset, yearTypeChooser); +publish.post('/:datasetId/time-period/:dimensionId/period-of-time', fetchDataset, yearTypeChooser); +publish.get('/:datasetId/time-period/:dimensionId/period-of-time/year-format', fetchDataset, yearFormat); +publish.post('/:datasetId/time-period/:dimensionId/period-of-time/year-format', fetchDataset, yearFormat); +publish.get('/:datasetId/time-period/:dimensionId/period-of-time/period-type', fetchDataset, periodType); +publish.post('/:datasetId/time-period/:dimensionId/period-of-time/period-type', fetchDataset, periodType); +publish.get('/:datasetId/time-period/:dimensionId/period-of-time/quarters', fetchDataset, quarterChooser); +publish.post('/:datasetId/time-period/:dimensionId/period-of-time/quarters', fetchDataset, quarterChooser); +publish.get('/:datasetId/time-period/:dimensionId/period-of-time/months', fetchDataset, monthChooser); +publish.post('/:datasetId/time-period/:dimensionId/period-of-time/months', fetchDataset, monthChooser); +publish.get('/:datasetId/time-period/:dimensionId/review', fetchDataset, periodReview); +publish.post('/:datasetId/time-period/:dimensionId/review', fetchDataset, periodReview); +publish.get('/:datasetId/dimension/:dimensionId/name', upload.none(), fetchDataset, dimensionName); +publish.post('/:datasetId/dimension/:dimensionId/name', upload.none(), fetchDataset, dimensionName); + +/* Meta Data related Routes */ publish.get('/:datasetId/change', fetchDataset, changeData); publish.post('/:datasetId/change', fetchDataset, upload.none(), changeData); diff --git a/src/services/stats-wales-api.ts b/src/services/stats-wales-api.ts index cc8d2ad..79a2754 100644 --- a/src/services/stats-wales-api.ts +++ b/src/services/stats-wales-api.ts @@ -1,6 +1,6 @@ import { ReadableStream } from 'node:stream/web'; -import { ViewDTO } from '../dtos/view-dto'; +import { ViewDTO, ViewErrDTO } from '../dtos/view-dto'; import { DatasetDTO } from '../dtos/dataset'; import { DatasetInfoDTO } from '../dtos/dataset-info'; import { FactTableDto } from '../dtos/fact-table'; @@ -19,6 +19,9 @@ import { ProviderSourceDTO } from '../dtos/provider-source'; import { TopicDTO } from '../dtos/topic'; import { OrganisationDTO } from '../dtos/organisation'; import { TeamDTO } from '../dtos/team'; +import { DimensionPatchDto } from '../dtos/dimension-patch-dto'; +import { DimensionDTO } from '../dtos/dimension'; +import { DimensionInfoDTO } from '../dtos/dimension-info'; const config = appConfig(); @@ -57,15 +60,19 @@ export class StatsWalesApi { const data = json ? JSON.stringify(json) : body; return fetch(`${this.backendUrl}/${url}`, { method, headers: head, body: data }) - .then((response: Response) => { + .then(async (response: Response) => { if (!response.ok) { + const body = await new Response(response.body).text(); + if (body) { + throw new ApiException(response.statusText, response.status, body); + } throw new ApiException(response.statusText, response.status); } return response; }) .catch((error) => { logger.error(`An api error occurred with status '${error.status}' and message '${error.message}'`); - throw new ApiException(error.message, error.status); + throw new ApiException(error.message, error.status, error.body); }); } @@ -159,6 +166,15 @@ export class StatsWalesApi { }).then((response) => response.json() as unknown as DatasetDTO); } + public async resetDimension(datasetId: string, dimensionId: string): Promise { + logger.debug(`Resetting dimension: ${dimensionId}`); + + return this.fetch({ + url: `dataset/${datasetId}/dimension/by-id/${dimensionId}/reset`, + method: HttpMethod.Delete + }).then((response) => response.json() as unknown as DimensionDTO); + } + public async getImportPreview( datasetId: string, revisionId: string, @@ -190,6 +206,32 @@ export class StatsWalesApi { }).then((response) => response.json() as unknown as DatasetDTO); } + public async patchDimension( + datasetId: string, + dimensionId: string, + dimensionPatch: DimensionPatchDto + ): Promise { + logger.debug(`sending patch request for dimension: ${dimensionId}`); + + return this.fetch({ + url: `dataset/${datasetId}/dimension/by-id/${dimensionId}`, + method: HttpMethod.Patch, + json: dimensionPatch + }).then((response) => response.json() as unknown as ViewDTO); + } + + public async updateDimensionInfo( + datasetId: string, + dimensionId: string, + dimensionInfo: DimensionInfoDTO + ): Promise { + return this.fetch({ + url: `dataset/${datasetId}/dimension/by-id/${dimensionId}/info`, + method: HttpMethod.Patch, + json: dimensionInfo + }).then((response) => response.json() as unknown as DimensionDTO); + } + public async updateDatasetInfo(datasetId: string, datasetInfo: DatasetInfoDTO): Promise { return this.fetch({ url: `dataset/${datasetId}/info`, method: HttpMethod.Patch, json: datasetInfo }).then( (response) => response.json() as unknown as DatasetDTO @@ -215,6 +257,20 @@ export class StatsWalesApi { ); } + public async getDimensionPreview(datasetId: string, dimensionId: string): Promise { + logger.debug(`Fetching dimension preview for dimension: ${datasetId}`); + return this.fetch({ url: `dataset/${datasetId}/dimension/by-id/${dimensionId}/preview` }).then( + (response) => response.json() as unknown as ViewDTO + ); + } + + public async getDimension(datasetId: string, dimensionId: string): Promise { + logger.debug(`Fetching dimension: ${dimensionId}`); + return this.fetch({ url: `dataset/${datasetId}/dimension/by-id/${dimensionId}/` }).then( + (response) => response.json() as unknown as DimensionDTO + ); + } + public async uploadCSVtoCreateDataset(file: Blob, filename: string, title: string): Promise { logger.debug(`Uploading CSV to create dataset with title '${title}'`); diff --git a/src/validators/index.ts b/src/validators/index.ts index a5ca248..b6c340d 100644 --- a/src/validators/index.ts +++ b/src/validators/index.ts @@ -1,5 +1,3 @@ -/* eslint-disable prettier/prettier */ - import { Request } from 'express'; import { body, FieldValidationError, param, ValidationChain } from 'express-validator'; import { ResultWithContext } from 'express-validator/lib/chain/context-runner'; @@ -42,10 +40,17 @@ export const datasetIdValidator = () => param('datasetId').trim().notEmpty().isU export const revisionIdValidator = () => param('revisionId').trim().notEmpty().isUUID(4); export const factTableIdValidator = () => param('factTableId').trim().notEmpty().isUUID(4); -export const titleValidator = () => body('title').trim() - .notEmpty().withMessage('missing').bail() - .isLength({ min: 3 }).withMessage('too_short').bail() - .isLength({ max: 1000 }).withMessage('too_long'); +export const titleValidator = () => + body('title') + .trim() + .notEmpty() + .withMessage('missing') + .bail() + .isLength({ min: 3 }) + .withMessage('too_short') + .bail() + .isLength({ max: 1000 }) + .withMessage('too_long'); export const descriptionValidator = () => body('description').trim().notEmpty(); export const collectionValidator = () => body('collection').trim().notEmpty(); diff --git a/src/views/publish/dimension-chooser.ejs b/src/views/publish/dimension-chooser.ejs new file mode 100644 index 0000000..a2cd4c6 --- /dev/null +++ b/src/views/publish/dimension-chooser.ejs @@ -0,0 +1,155 @@ +<%- include("../partials/header", t); %> + +
+ +
+ + +

<%= t('publish.dimension_type_chooser.heading') %>

+ +

<%= t('publish.dimension_type_chooser.subheading') %>

+ + <% if (locals?.errors) { %> +
+
+

+ <%= t('errors.problem') %> +

+
+ +
+
+
+ <% } %> + + <% if (locals?.data) { %> +
+
+ + + <% locals.headers.forEach(function(cell, idx) { %> + + <% }); %> + + + + <% locals.headers.forEach(function(cell, idx) { %> + + <% }); %> + + + + <% locals.data.forEach(function(row) { %> + + <% row.forEach(function(cell, index) { %> + <% if (locals.headers[index].source_type === 'line_number') { %> + + <% } else { %> + + <% } %> + <% }); %> + + <% }); %> + +
+ <% if (cell.source_type && cell.source_type !== 'unknown' && cell.source_type !== 'line_number') { %> + <%= t(`publish.preview.source_type.${cell.source_type}`) %>
+ <% } %> + <% if (cell.source_type !== 'line_number') { %> + <%= cell.name || t('publish.preview.unnamed_column', { colNum: idx + 1 }) %> + <% } else { %> + <%= t('publish.preview.row_number') %> + <% } %> +
<%= cell %><%= cell %>
+
+
+ +
+
+
+
+
+ +

+ <%= t('publish.dimension_type_chooser.question') %> +

+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
or
+
+ + +
+
+
+
+
+ + + <%= t('buttons.preview') %> + +
+
+
+
+ <% } %> +
+
+
<%= JSON.stringify(locals.dimensionPreview) %>
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/dimension-name.ejs b/src/views/publish/dimension-name.ejs new file mode 100644 index 0000000..3d28e8a --- /dev/null +++ b/src/views/publish/dimension-name.ejs @@ -0,0 +1,33 @@ +<%- include("../partials/header"); %> +
+ +
+ + +

<%= t('publish.dimension_name.heading') %>

+ + <%- include("../partials/error-handler"); %> + +

<%= t('publish.dimension_name.hint') %>

+
    +
  • <%= t('publish.dimension_name.concise') %>
  • +
  • <%= t('publish.dimension_name.language') %>
  • +
+ +
+
+ +
+ +
+
+
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/month-format.ejs b/src/views/publish/month-format.ejs new file mode 100644 index 0000000..bf80adb --- /dev/null +++ b/src/views/publish/month-format.ejs @@ -0,0 +1,85 @@ +<%- include("../partials/header", t); %> + +
+
+ + +

<%= t('publish.month_format.heading') %>

+ + <% if (locals?.errors) { %> +
+
+

+ <%= t('errors.problem') %> +

+
+ +
+
+
+ <% } %> + +
+
+
+
+
+
+
+ + +
+ <%= t('publish.month_format.example', { example: 'Jan' })%> +
+
+
+ + +
+ <%= t('publish.month_format.example', { example: 'm01' })%> +
+
+
+ + +
+ <%= t('publish.month_format.example', { example: '01' })%> +
+
+
+
+
+
+ + + <%= t('buttons.preview') %> + +
+
+
+
+
+
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/period-match-failure.ejs b/src/views/publish/period-match-failure.ejs new file mode 100644 index 0000000..36b0c80 --- /dev/null +++ b/src/views/publish/period-match-failure.ejs @@ -0,0 +1,59 @@ +<%- include("../partials/header", t); %> + +
+
+ + +

<%= t('publish.period_match_failure.heading') %>

+ +

<%= t('publish.period_match_failure.information', {failureCount: locals.extension.totalNonMatching}) %>

+
    +
  • <%= t('publish.period_match_failure.formatting') %>
  • +
  • <%= t('publish.period_match_failure.choices') %>
  • +
+ +

<%= t('publish.period_match_failure.supplied_format') %>

+ +
    + <% if (locals.patchRequest.year_format) { %> +
  • <%- t('publish.period_match_failure.year_format', {format: locals.patchRequest.year_format}) %>
  • + <% } %> + <% if (locals.patchRequest.quarter_format) { %> +
  • <%- t('publish.period_match_failure.quarter_format', {format: locals.patchRequest.quarter_format}) %>
  • + <% }%> + <% if (locals.patchRequest.month_format) { %> +
  • <%- t('publish.period_match_failure.month_format', {format: locals.patchRequest.month_format}) %>
  • + <% }%> + <% if (locals.patchRequest.date_format) { %> +
  • <%- t('publish.period_match_failure.date_format', {format: locals.patchRequest.date_format}) %>
  • + <% }%> +
+ +

<%= t('publish.period_match_failure.subheading') %>

+ +
    + <% if (locals.extension.nonMatchingValues.length === 0) { %> +
  • <%= t('publish.period_match_failure.no_matches') %>
  • + <% } else { %> + <% locals.extension.nonMatchingValues.forEach((value) => { %> +
  • <%= value %>
  • + <% });%> + <% } %> +
+ +

+ <%= t('publish.period_match_failure.upload_different_file')%> +
+ <%= t('publish.period_match_failure.upload_different_file_warning')%>

+ +

+ <%= t('publish.period_match_failure.try_different_format')%>

+
+
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/period-type.ejs b/src/views/publish/period-type.ejs new file mode 100644 index 0000000..6c01f7a --- /dev/null +++ b/src/views/publish/period-type.ejs @@ -0,0 +1,76 @@ +<%- include("../partials/header", t); %> + +
+
+ + +

<%= t('publish.period-type-chooser.heading') %>

+ + <% if (locals?.errors) { %> +
+
+

+ <%= t('errors.problem') %> +

+
+ +
+
+
+ <% } %> + +
+
+
+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+ + + <%= t('buttons.preview') %> + +
+
+
+
+
+
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/point-in-time-chooser.ejs b/src/views/publish/point-in-time-chooser.ejs new file mode 100644 index 0000000..0978f5d --- /dev/null +++ b/src/views/publish/point-in-time-chooser.ejs @@ -0,0 +1,76 @@ +<%- include("../partials/header", t); %> + +
+
+ + +

<%= t('publish.point_in_time.heading') %>

+ + <%- include("../partials/error-handler"); %> + + +
+
+
+
+
+
+
+ + +
+ <%- t('publish.point_in_time.example', { example: '14/10/2024' })%> +
+
+
+ + +
+ <%- t('publish.point_in_time.example', { example: '14-10-2024' })%> +
+
+
+ + +
+ <%- t('publish.point_in_time.example', { example: '2024-10-14' })%> +
+
+
+ + +
+ <%- t('publish.point_in_time.example', { example: '20241014' })%> +
+
+
+
+
+
+ + + <%= t('buttons.preview') %> + +
+
+
+
+
+
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/quarter-format.ejs b/src/views/publish/quarter-format.ejs new file mode 100644 index 0000000..969c6b9 --- /dev/null +++ b/src/views/publish/quarter-format.ejs @@ -0,0 +1,113 @@ +<%- include("../partials/header", t); %> + +
+
+ + + <% if (locals.quarterTotals) { %> +

<%= t('publish.quarter_format.heading-alt') %>

+ <% } else { %> +

<%= t('publish.quarter_format.heading') %>

+ <% } %> + + <%- include("../partials/error-handler"); %> + +
+
+
+
+
+
+
+ + +
+ <%= t('publish.quarter_format.example', { example: 'Q1' })%> +
+
+
+ + +
+ <%= t('publish.quarter_format.example', { example: '_Q1' })%> +
+
+
+ + +
+ <%= t('publish.quarter_format.example', { example: '-Q1' })%> +
+
+
+ + +
+ <%- t('publish.quarter_format.example', { example: '1' })%> +
+
+
+ + +
+ <%- t('publish.quarter_format.example', { example: '_1' })%> +
+
+
+ + +
+ <%- t('publish.quarter_format.example', { example: '-1' })%> +
+
+
+ <% if (locals.quarterTotals) { %> +
or
+
+ + +
+ <% } %> +
 
+
+ + +
+
+
+
+ + + <%= t('buttons.preview') %> + +
+
+
+
+
+
+ +<%- include("../partials/footer"); %> diff --git a/src/views/publish/tasklist.ejs b/src/views/publish/tasklist.ejs index 7fcbe59..46b7172 100644 --- a/src/views/publish/tasklist.ejs +++ b/src/views/publish/tasklist.ejs @@ -26,10 +26,27 @@ + <% if (taskList.measure) { %> + + <% } %> <% taskList.dimensions.forEach(function(dimension) { %> + <% const dim = locals.dimensions.find((d) => d.dimensionInfo.name === dimension.name) %>