Skip to content

Commit

Permalink
Fixes a couple of last minute bugs
Browse files Browse the repository at this point in the history
  • Loading branch information
@j-maynard
j-maynard committed Dec 11, 2024
1 parent 50e32eb commit e9e539b
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 61 deletions.
21 changes: 11 additions & 10 deletions src/controllers/dimension-processor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,9 +331,10 @@ export const validateDateTypeDimension = async (
dateFormat: dimensionPatchRequest.date_format
};
logger.debug(`Extractor created with the following JSON: ${JSON.stringify(extractor)}`);
const previewQuery = `SELECT DISTINCT ${dimension.factTableColumn} FROM ${tableName}`;
const previewQuery = `SELECT DISTINCT "${dimension.factTableColumn}" FROM ${tableName}`;
const preview = await quack.all(previewQuery);
try {
logger.debug(`Extractor created with ${JSON.stringify(extractor)}`);
dateDimensionTable = dateDimensionReferenceTableCreator(extractor, preview);
} catch (error) {
logger.error(
Expand Down Expand Up @@ -377,7 +378,7 @@ export const validateDateTypeDimension = async (

// Now validate everything matches
const nonMatchedRows = await quack.all(
`SELECT line_number, fact_table_date, date_dimension.date_code FROM (SELECT row_number() OVER () as line_number, "${dimension.factTableColumn}" as fact_table_date FROM ${tableName}) as fact_table LEFT JOIN date_dimension ON fact_table.fact_table_date=date_dimension.date_code where date_code IS NULL;`
`SELECT line_number, fact_table_date, date_dimension.date_code FROM (SELECT row_number() OVER () as line_number, "${dimension.factTableColumn}" as fact_table_date FROM ${tableName}) as fact_table LEFT JOIN date_dimension ON CAST(fact_table.fact_table_date AS VARCHAR)=CAST(date_dimension.date_code AS VARCHAR) where date_code IS NULL;`
);
if (nonMatchedRows.length > 0) {
if (nonMatchedRows.length === preview.length) {
Expand Down Expand Up @@ -410,7 +411,7 @@ export const validateDateTypeDimension = async (
`There were ${nonMatchedRows.length} row(s) which didn't match based on the information given to us by the user`
);
const nonMatchedRowSample = await quack.all(
`SELECT DISTINCT fact_table_date, FROM (SELECT row_number() OVER () as line_number, "${dimension.factTableColumn}" as fact_table_date FROM ${tableName}) as fact_table LEFT JOIN date_dimension ON fact_table.fact_table_date=date_dimension.date_code where date_code IS NULL;`
`SELECT DISTINCT fact_table_date, FROM (SELECT row_number() OVER () as line_number, "${dimension.factTableColumn}" as fact_table_date FROM ${tableName}) as fact_table LEFT JOIN date_dimension ON CAST(fact_table.fact_table_date AS VARCHAR)=CAST(date_dimension.date_code AS VARCHAR) where date_code IS NULL;`
);
const nonMatchingValues = nonMatchedRowSample
.map((item) => item.fact_table_date)
Expand Down Expand Up @@ -444,7 +445,7 @@ export const validateDateTypeDimension = async (
}
}
} catch (error) {
logger.error(error);
logger.error(`Something went wrong trying to validate the data with the following error: ${error}`);
await quack.close();
tempFile.removeCallback();
throw error;
Expand Down Expand Up @@ -527,11 +528,11 @@ async function getDatePreviewWithExtractor(
fact_table: FactTableDTO.fromFactTable(currentImport),
current_page: 1,
page_info: {
total_records: columnData.length,
total_records: dimensionTable.length,
start_record: 1,
end_record: sampleSize
end_record: dimensionTable.length < sampleSize ? dimensionTable.length : sampleSize
},
page_size: sampleSize,
page_size: dimensionTable.length < sampleSize ? dimensionTable.length : sampleSize,
total_pages: 1,
headers,
data: dataArray
Expand All @@ -546,7 +547,7 @@ async function getDatePreviewWithoutExtractor(
tableName: string
): Promise<ViewDTO> {
const preview = await quack.all(
`SELECT DISTINCT "${dimension.factTableColumn}" FROM ${tableName} USING SAMPLE ${sampleSize};`
`SELECT DISTINCT "${dimension.factTableColumn}" FROM ${tableName} ORDER BY "${dimension.factTableColumn}" ASC LIMIT ${sampleSize};`
);
const tableHeaders = Object.keys(preview[0]);
const dataArray = preview.map((row) => Object.values(row));
Expand All @@ -567,9 +568,9 @@ async function getDatePreviewWithoutExtractor(
page_info: {
total_records: preview.length,
start_record: 1,
end_record: sampleSize
end_record: preview.length
},
page_size: sampleSize,
page_size: preview.length < sampleSize ? preview.length : sampleSize,
total_pages: 1,
headers,
data: dataArray
Expand Down
41 changes: 25 additions & 16 deletions src/controllers/time-matching.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ function createAllTypesOfPeriod(dateFormat: DateExtractor, dataColumn: TableData
logger.debug(`date extractor = ${JSON.stringify(dateFormat)}`);
if (dateFormat.quarterFormat && dateFormat.quarterTotalIsFifthQuart) {
logger.debug('5th quarter used to represent whole year totals');
referenceTable = referenceTable.concat(periodTableCreator(dateFormat, dataColumn));
return periodTableCreator(dateFormat, dataColumn);
} else {
// We always need years generate those first
logger.debug('Creating table for year');
Expand Down Expand Up @@ -162,9 +162,8 @@ function periodTableCreator(dateFormat: DateExtractor, dataColumn: TableData) {
const endYear = Math.max(...dataYears);
const type = yearType(dateFormat.type);

// We want 1 year before our earliest date and 1 year after to cope with Financial, Tax and Meteorological years
let year = sub(parseISO(`${startYear}-${type.start}`), { years: 1 });
const end = add(parseISO(`${endYear}-${type.start}`), { years: 2 });
let year = parseISO(`${startYear}-${type.start}`);
const end = add(parseISO(`${endYear}-${type.start}`), { years: 1 });

// Quarters and month numbers are different depending on the type of year
let quarterIndex = 1;
Expand Down Expand Up @@ -202,26 +201,36 @@ function periodTableCreator(dateFormat: DateExtractor, dataColumn: TableData) {
dateFormat.type === YearType.Calendar
? format(displayYear, 'yyyy')
: `${format(displayYear, 'yyyy')}-${format(add(displayYear, { years: 1 }), 'yy')}`;
if (dateFormat.quarterTotalIsFifthQuart && quarterIndex === 5) {

referenceTable.push({
dateCode: dateStr,
description,
start: year,
end: sub(add(year, { months: formatObj.increment }), { seconds: 1 }),
type: `${dateFormat.type}_${subType}`
});

if (dateFormat.quarterTotalIsFifthQuart && quarterIndex === 4) {
const yearStr = formatObj.formatStr
.replace('[full-start]', format(displayYear, 'yyyy'))
.replace('[full-end]', format(add(displayYear, { years: 1 }), 'yyyy'))
.replace('[end-year]', format(add(displayYear, { years: 1 }), 'yy'))
.replace('[quarterNo]', (quarterIndex + 1).toString())
.replace('[monthStr]', format(year, 'MMM'))
.replace('[monthNo]', String(monthNo).padStart(2, '0'));
referenceTable.push({
dateCode: dateStr,
dateCode: yearStr,
description,
start: year,
end: sub(add(year, { months: 12 }), { seconds: 1 }),
type: `${dateFormat.type}_year`
});
} else {
referenceTable.push({
dateCode: dateStr,
description,
start: year,
end: sub(add(year, { months: formatObj.increment }), { seconds: 1 }),
type: `${dateFormat.type}_${subType}`
});
year = add(year, { months: formatObj.increment });
}

year = add(year, { months: formatObj.increment });

// This is needed because quarters aren't really things
if (quarterIndex < (dateFormat.quarterTotalIsFifthQuart ? 5 : 4)) {
if (quarterIndex < 4) {
quarterIndex++;
} else {
quarterIndex = 1;
Expand Down
1 change: 1 addition & 0 deletions src/route/dataset.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,7 @@ router.patch(
throw new Error('Not Implemented Yet!');
}
} catch (error) {
logger.error(`Something went wrong trying to validate the dimension with the following error: ${error}`);
res.status(500);
res.json({ message: 'Unable to validate or match dimension against patch' });
return;
Expand Down
63 changes: 28 additions & 35 deletions test/dimension-processor.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ describe('Date matching table generation', () => {
};
const dateColumn: any[] = [{ yearCode: 2023 }];
const refTable = dateDimensionReferenceTableCreator(extractor, dateColumn);
expect(refTable.length).toBe(3);
expect(refTable[1].dateCode).toBe('202324');
expect(refTable.length).toBe(1);
expect(refTable[0].dateCode).toBe('202324');
});

test('Given a correct year and quarter format it returns a valid table', async () => {
Expand All @@ -180,15 +180,12 @@ describe('Date matching table generation', () => {
};
const dateColumn: any[] = [{ yearCode: 2023 }];
const refTable = dateDimensionReferenceTableCreator(extractor, dateColumn);
expect(refTable.length).toBe(15);
expect(refTable[0].dateCode).toBe('202223');
expect(refTable[1].dateCode).toBe('202324');
expect(refTable[2].dateCode).toBe('202425');
expect(refTable[3].dateCode).toBe('202223Q1');
expect(refTable[4].dateCode).toBe('202223Q2');
expect(refTable[5].dateCode).toBe('202223Q3');
expect(refTable[6].dateCode).toBe('202223Q4');
expect(refTable[7].dateCode).toBe('202324Q1');
expect(refTable.length).toBe(5);
expect(refTable[0].dateCode).toBe('202324');
expect(refTable[1].dateCode).toBe('202324Q1');
expect(refTable[2].dateCode).toBe('202324Q2');
expect(refTable[3].dateCode).toBe('202324Q3');
expect(refTable[4].dateCode).toBe('202324Q4');
});

test('Given a the magic 5th setting and correct year and quarter format it returns a valid table', async () => {
Expand All @@ -200,13 +197,12 @@ describe('Date matching table generation', () => {
};
const dateColumn: any[] = [{ yearCode: 2023 }];
const refTable = dateDimensionReferenceTableCreator(extractor, dateColumn);
expect(refTable.length).toBe(14);
expect(refTable[0].dateCode).toBe('202223Q1');
expect(refTable[1].dateCode).toBe('202223Q2');
expect(refTable[2].dateCode).toBe('202223Q3');
expect(refTable[3].dateCode).toBe('202223Q4');
expect(refTable[4].dateCode).toBe('202223Q5');
expect(refTable[5].dateCode).toBe('202324Q1');
expect(refTable.length).toBe(5);
expect(refTable[0].dateCode).toBe('202324Q1');
expect(refTable[1].dateCode).toBe('202324Q2');
expect(refTable[2].dateCode).toBe('202324Q3');
expect(refTable[3].dateCode).toBe('202324Q4');
expect(refTable[4].dateCode).toBe('202324Q5');
});

test('Given a correct year and month format it returns a valid table', async () => {
Expand All @@ -217,23 +213,20 @@ describe('Date matching table generation', () => {
};
const dateColumn: any[] = [{ yearCode: 2023 }];
const refTable = dateDimensionReferenceTableCreator(extractor, dateColumn);
expect(refTable.length).toBe(39);
expect(refTable[0].dateCode).toBe('202223');
expect(refTable[1].dateCode).toBe('202324');
expect(refTable[2].dateCode).toBe('202425');
expect(refTable[3].dateCode).toBe('202223Apr');
expect(refTable[4].dateCode).toBe('202223May');
expect(refTable[5].dateCode).toBe('202223Jun');
expect(refTable[6].dateCode).toBe('202223Jul');
expect(refTable[7].dateCode).toBe('202223Aug');
expect(refTable[8].dateCode).toBe('202223Sep');
expect(refTable[9].dateCode).toBe('202223Oct');
expect(refTable[10].dateCode).toBe('202223Nov');
expect(refTable[11].dateCode).toBe('202223Dec');
expect(refTable[12].dateCode).toBe('202223Jan');
expect(refTable[13].dateCode).toBe('202223Feb');
expect(refTable[14].dateCode).toBe('202223Mar');
expect(refTable[15].dateCode).toBe('202324Apr');
expect(refTable.length).toBe(13);
expect(refTable[0].dateCode).toBe('202324');
expect(refTable[1].dateCode).toBe('202324Apr');
expect(refTable[2].dateCode).toBe('202324May');
expect(refTable[3].dateCode).toBe('202324Jun');
expect(refTable[4].dateCode).toBe('202324Jul');
expect(refTable[5].dateCode).toBe('202324Aug');
expect(refTable[6].dateCode).toBe('202324Sep');
expect(refTable[7].dateCode).toBe('202324Oct');
expect(refTable[8].dateCode).toBe('202324Nov');
expect(refTable[9].dateCode).toBe('202324Dec');
expect(refTable[10].dateCode).toBe('202324Jan');
expect(refTable[11].dateCode).toBe('202324Feb');
expect(refTable[12].dateCode).toBe('202324Mar');
});

test('If given an unknown year format it errors', async () => {
Expand Down

0 comments on commit e9e539b

Please sign in to comment.