diff --git a/src/configuration.ts b/src/configuration.ts index a1e2bed9..548c72d8 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -3,15 +3,7 @@ import * as tt from 'io-ts-types'; import * as E from 'fp-ts/Either'; import {pipe} from 'fp-ts/lib/function'; import {formatValidationErrors} from 'io-ts-reporters'; - -const withDefaultIfEmpty = (codec: C, ifEmpty: t.TypeOf) => - tt.withValidate(codec, (input, context) => - pipe( - tt.NonEmptyString.validate(input, context), - E.orElse(() => t.success(String(ifEmpty))), - E.chain(nonEmptyString => codec.validate(nonEmptyString, context)) - ) - ); +import {withDefaultIfEmpty} from './util'; const LogLevel = t.keyof({ trace: null, diff --git a/src/index.ts b/src/index.ts index f8297e0c..e8e5dae8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -73,7 +73,7 @@ const periodicExternalReadModelRefresh = setInterval(() => { 'Unexpected error when refreshing read model with external sources' ) ); -}, 60_000); +}, 30_000); server.on('close', () => { clearInterval(periodicReadModelRefresh); clearInterval(periodicExternalReadModelRefresh); diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 06e23003..e66961fb 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -1,17 +1,83 @@ import {Logger} from 'pino'; import * as TE from 'fp-ts/TaskEither'; -import {Failure} from '../../types'; +import * as t from 'io-ts'; +import * as tt from 'io-ts-types'; +import * as E from 'fp-ts/Either'; import {pipe} from 'fp-ts/lib/function'; -import {sheets, sheets_v4} from '@googleapis/sheets'; +import {sheets} from '@googleapis/sheets'; import {GoogleAuth} from 'google-auth-library'; +import {columnIndexToLetter} from '../../training-sheets/extract-metadata'; +import {formatValidationErrors} from 'io-ts-reporters'; +import {DateTime} from 'luxon'; -export const pullGoogleSheetData = +const DEFAULT_TIMEZONE = 'Europe/London'; + +// Not all the google form sheets are actually in Europe/London. +// Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). +export const GoogleTimezone = tt.withValidate(t.string, (input, context) => + pipe( + t.string.validate(input, context), + E.chain(timezoneRaw => + DateTime.local().setZone(timezoneRaw).isValid + ? E.right(timezoneRaw) + : E.left([]) + ), + E.orElse(() => t.success(DEFAULT_TIMEZONE)) + ) +); + +export const GoogleSpreadsheetInitialMetadata = t.strict({ + properties: t.strict({ + timeZone: GoogleTimezone, + }), + sheets: t.array( + t.strict({ + properties: t.strict({ + title: t.string, + gridProperties: t.strict({ + rowCount: t.number, + }), + }), + }) + ), +}); +export type GoogleSpreadsheetInitialMetadata = t.TypeOf< + typeof GoogleSpreadsheetInitialMetadata +>; + +// Contains only a single sheet. Structure is a little verbose to match the part of the +// google api it is taken from. +export const GoogleSpreadsheetDataForSheet = t.strict({ + sheets: tt.nonEmptyArray( + // Array always has length = 1 because this is data for a single sheet. + t.strict({ + data: tt.nonEmptyArray( + t.strict({ + rowData: tt.nonEmptyArray( + t.strict({ + values: tt.nonEmptyArray( + t.strict({ + formattedValue: tt.withFallback(t.string, ''), + }) + ), + }) + ), + }) + ), + }) + ), +}); +export type GoogleSpreadsheetDataForSheet = t.TypeOf< + typeof GoogleSpreadsheetDataForSheet +>; + +export const pullGoogleSheetDataMetadata = (auth: GoogleAuth) => ( logger: Logger, trainingSheetId: string - ): TE.TaskEither => + ): TE.TaskEither => pipe( TE.tryCatch( () => @@ -20,15 +86,87 @@ export const pullGoogleSheetData = auth, }).spreadsheets.get({ spreadsheetId: trainingSheetId, - includeGridData: true, + includeGridData: false, // Only the metadata. + fields: 'sheets(properties),properties(timeZone)', // Only the metadata about the sheets. }), reason => { - logger.error(reason, 'Failed to get spreadsheet'); - return { - // Expand failure reasons. - message: `Failed to get training spreadsheet ${trainingSheetId}`, - }; + logger.error(reason, 'Failed to get spreadsheet metadata'); + return `Failed to get training spreadsheet metadata ${trainingSheetId}`; + } + ), + TE.map(resp => resp.data), + TE.chain(data => + TE.fromEither( + pipe( + data, + GoogleSpreadsheetInitialMetadata.decode, + E.mapLeft( + e => + `Failed to get google spreadsheet metadata from API response: ${formatValidationErrors(e).join(',')}` + ) + ) + ) + ) + ); + +export const pullGoogleSheetData = + (auth: GoogleAuth) => + ( + logger: Logger, + trainingSheetId: string, + sheetName: string, + rowStart: number, // 1 indexed. + rowEnd: number, + columnStartIndex: number, // 0 indexed, converted to a letter. + columnEndIndex: number + ): TE.TaskEither => + pipe( + TE.tryCatch( + () => { + const ranges = [ + `${sheetName}!${columnIndexToLetter(columnStartIndex)}${rowStart}:${columnIndexToLetter(columnEndIndex)}${rowEnd}`, + ]; + const fields = 'sheets(data(rowData(values(formattedValue))))'; + logger.info( + 'Querying sheet %s for fields %s range %s', + trainingSheetId, + fields, + ranges + ); + return sheets({ + version: 'v4', + auth, + }).spreadsheets.get({ + spreadsheetId: trainingSheetId, + fields, + ranges, + }); + }, + reason => { + logger.error( + reason, + 'Failed to get training spreadsheet %s', + trainingSheetId + ); + return `Failed to get training spreadsheet ${trainingSheetId}`; } ), - TE.map(resp => resp.data) + TE.map(resp => resp.data), + TE.chain(data => + TE.fromEither( + pipe( + data, + GoogleSpreadsheetDataForSheet.decode, + E.mapLeft( + e => + `Failed to get all required google spreadsheet data from API response: ${formatValidationErrors(e).join(',')}` + ) + ) + ) + ) ); + +export interface GoogleHelpers { + pullGoogleSheetData: ReturnType; + pullGoogleSheetDataMetadata: ReturnType; +} diff --git a/src/init-dependencies/init-dependencies.ts b/src/init-dependencies/init-dependencies.ts index 34636063..b57c6799 100644 --- a/src/init-dependencies/init-dependencies.ts +++ b/src/init-dependencies/init-dependencies.ts @@ -10,7 +10,11 @@ import {commitEvent} from './event-store/commit-event'; import {getAllEvents, getAllEventsByType} from './event-store/get-all-events'; import {getResourceEvents} from './event-store/get-resource-events'; import {Client} from '@libsql/client'; -import {pullGoogleSheetData} from './google/pull_sheet_data'; +import { + GoogleHelpers, + pullGoogleSheetData, + pullGoogleSheetDataMetadata, +} from './google/pull_sheet_data'; import {initSharedReadModel} from '../read-models/shared-state'; import {GoogleAuth} from 'google-auth-library'; @@ -57,24 +61,26 @@ export const initDependencies = ( }) ); - const googleAuth = - conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON.toLowerCase().trim() === 'disabled' - ? O.none - : O.some( - pullGoogleSheetData( - new GoogleAuth({ - // Google issues the credentials file and validates it. - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - credentials: JSON.parse(conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON), - scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'], - }) - ) - ); + let googleHelpers: O.Option = O.none; + if ( + conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON.toLowerCase().trim() !== 'disabled' + ) { + const googleAuth = new GoogleAuth({ + // Google issues the credentials file and validates it. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + credentials: JSON.parse(conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON), + scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'], + }); + googleHelpers = O.some({ + pullGoogleSheetData: pullGoogleSheetData(googleAuth), + pullGoogleSheetDataMetadata: pullGoogleSheetDataMetadata(googleAuth), + }); + } const sharedReadModel = initSharedReadModel( dbClient, logger, - googleAuth, + googleHelpers, conf.GOOGLE_RATELIMIT_MS ); diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 6101a3ca..2bb6d0b8 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -1,43 +1,96 @@ import {Logger} from 'pino'; -import * as T from 'fp-ts/Task'; -import * as TE from 'fp-ts/TaskEither'; +import * as E from 'fp-ts/Either'; import * as O from 'fp-ts/Option'; import * as RA from 'fp-ts/ReadonlyArray'; -import {DomainEvent, Failure} from '../../types'; +import {DomainEvent} from '../../types'; import {BetterSQLite3Database} from 'drizzle-orm/better-sqlite3'; import {getAllEquipment} from './get-equipment'; import {pipe} from 'fp-ts/lib/function'; -import {sheets_v4} from '@googleapis/sheets'; import {EpochTimestampMilliseconds, Equipment} from './return-types'; -import {extractGoogleSheetData} from '../../training-sheets/google'; -import {constructEvent, EventOfType} from '../../types/domain-event'; +import { + columnBoundsRequired, + extractGoogleSheetData, + shouldPullFromSheet, +} from '../../training-sheets/google'; +import {constructEvent} from '../../types/domain-event'; +import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data'; +import { + extractGoogleSheetMetadata, + GoogleSheetMetadata, + MAX_COLUMN_INDEX, +} from '../../training-sheets/extract-metadata'; +import {getChunkIndexes} from '../../util'; -export type PullSheetData = ( +const ROW_BATCH_SIZE = 200; + +const pullNewEquipmentQuizResultsForSheet = async ( logger: Logger, - trainingSheetId: string -) => TE.TaskEither; + googleHelpers: GoogleHelpers, + equipment: Equipment, + trainingSheetId: string, + sheet: GoogleSheetMetadata, + timezone: string, + updateState: (event: DomainEvent) => void +) => { + logger.info('Processing sheet %s', sheet.name); + for (const [rowStart, rowEnd] of getChunkIndexes( + 2, // 1-indexed and first row is headers. + sheet.rowCount, + ROW_BATCH_SIZE + )) { + logger.debug( + 'Pulling data for sheet %s rows %s to %s', + sheet.name, + rowStart, + rowEnd + ); -export const pullNewEquipmentQuizResults = ( + const [minCol, maxCol] = columnBoundsRequired(sheet); + + const data = await googleHelpers.pullGoogleSheetData( + logger, + trainingSheetId, + sheet.name, + rowStart, + rowEnd, + minCol, + maxCol + )(); + if (E.isLeft(data)) { + logger.debug( + 'Failed to pull data for sheet %s rows %s to %s, skipping rest of sheet' + ); + return; + } + pipe( + data.right, + extractGoogleSheetData( + logger, + trainingSheetId, + equipment.id, + sheet, + timezone, + equipment.lastQuizResult + ), + RA.map(updateState) + ); + } +}; + +export const pullNewEquipmentQuizResults = async ( logger: Logger, - pullGoogleSheetData: PullSheetData, - equipment: Equipment -): T.Task< - ReadonlyArray< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - > -> => { + googleHelpers: GoogleHelpers, + equipment: Equipment, + updateState: (event: DomainEvent) => void +): Promise => { + // TODO - Refactor this into fp-ts style. if (O.isNone(equipment.trainingSheetId)) { logger.warn( 'No training sheet registered for equipment %s, skipping training data ingestion', equipment.name ); // eslint-disable-next-line @typescript-eslint/require-await - return async () => - [] as ReadonlyArray< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - >; + return; } const trainingSheetId = equipment.trainingSheetId.value; logger = logger.child({trainingSheetId}); @@ -45,38 +98,70 @@ export const pullNewEquipmentQuizResults = ( 'Scanning training sheet. Pulling google sheet data from %s...', equipment.lastQuizResult ); - return pipe( - pullGoogleSheetData(logger, trainingSheetId), - TE.map( - extractGoogleSheetData( - logger, - trainingSheetId, - equipment.id, - equipment.lastQuizResult - ) - ), - TE.map(RA.flatten), - TE.map( - RA.append< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - >( - constructEvent('EquipmentTrainingQuizSync')({ - equipmentId: equipment.id, - }) - ) - ), - // eslint-disable-next-line @typescript-eslint/require-await - TE.getOrElse(err => async () => { - logger.error( - 'Failed to receive data from google sheets for equipment %s: %s', - equipment.name, - err.message + + const initialMeta = await googleHelpers.pullGoogleSheetDataMetadata( + logger, + trainingSheetId + )(); + if (E.isLeft(initialMeta)) { + logger.warn(initialMeta.left); + return; + } + + const sheets: GoogleSheetMetadata[] = []; + for (const sheet of initialMeta.right.sheets) { + if (!shouldPullFromSheet(sheet)) { + logger.warn( + "Skipping sheet as doesn't match expected for form responses" ); - return [] as ReadonlyArray< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - >; + continue; + } + + const firstRowData = await googleHelpers.pullGoogleSheetData( + logger, + trainingSheetId, + sheet.properties.title, + 1, + 1, + 0, + MAX_COLUMN_INDEX + )(); + if (E.isLeft(firstRowData)) { + logger.warn( + 'Failed to get google sheet first row data for sheet %s, skipping', + sheet.properties.title + ); + continue; + } + + const meta = extractGoogleSheetMetadata(logger)(sheet, firstRowData.right); + if (O.isNone(meta)) { + continue; + } + + logger.info( + 'Got metadata for sheet: %s: %o', + sheet.properties.title, + meta.value + ); + sheets.push(meta.value); + } + + for (const sheet of sheets) { + await pullNewEquipmentQuizResultsForSheet( + logger, + googleHelpers, + equipment, + trainingSheetId, + sheet, + initialMeta.right.properties.timeZone, + updateState + ); + } + + updateState( + constructEvent('EquipmentTrainingQuizSync')({ + equipmentId: equipment.id, }) ); }; @@ -84,13 +169,13 @@ export const pullNewEquipmentQuizResults = ( export const asyncApplyExternalEventSources = ( logger: Logger, currentState: BetterSQLite3Database, - pullGoogleSheetData: O.Option, + googleHelpers: O.Option, updateState: (event: DomainEvent) => void, googleRateLimitMs: number ) => { return () => async () => { logger.info('Applying external event sources...'); - if (O.isNone(pullGoogleSheetData)) { + if (O.isNone(googleHelpers)) { logger.info('Google external event source disabled'); return; } @@ -106,13 +191,11 @@ export const asyncApplyExternalEventSources = ( 'Triggering event update from google training sheets for %s...', equipment.name ); - pipe( - await pullNewEquipmentQuizResults( - logger, - pullGoogleSheetData.value, - equipment - )(), - RA.map(updateState) + await pullNewEquipmentQuizResults( + logger, + googleHelpers.value, + equipment, + updateState ); logger.info( 'Finished pulling events from google training sheet for %s', diff --git a/src/read-models/shared-state/index.ts b/src/read-models/shared-state/index.ts index 0ec2931e..888abaeb 100644 --- a/src/read-models/shared-state/index.ts +++ b/src/read-models/shared-state/index.ts @@ -10,11 +10,9 @@ import {Client} from '@libsql/client/.'; import {asyncRefresh} from './async-refresh'; import {updateState} from './update-state'; import {Logger} from 'pino'; -import { - asyncApplyExternalEventSources, - PullSheetData, -} from './async-apply-external-event-sources'; +import {asyncApplyExternalEventSources} from './async-apply-external-event-sources'; import {UUID} from 'io-ts-types'; +import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data'; export {replayState} from './deprecated-replay'; @@ -35,7 +33,7 @@ export type SharedReadModel = { export const initSharedReadModel = ( eventStoreClient: Client, logger: Logger, - pullGoogleSheetData: O.Option, + googleHelpers: O.Option, googleRateLimitMs: number ): SharedReadModel => { const readModelDb = drizzle(new Database()); @@ -48,7 +46,7 @@ export const initSharedReadModel = ( asyncApplyExternalEventSources: asyncApplyExternalEventSources( logger, readModelDb, - pullGoogleSheetData, + googleHelpers, updateState_, googleRateLimitMs ), diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts new file mode 100644 index 00000000..300fe844 --- /dev/null +++ b/src/training-sheets/extract-metadata.ts @@ -0,0 +1,81 @@ +import * as RA from 'fp-ts/ReadonlyArray'; +import * as O from 'fp-ts/Option'; + +import {Logger} from 'pino'; +import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data'; + +const EMAIL_COLUMN_NAMES = ['email address', 'email']; + +export type GoogleSheetName = string; + +type ColumnLetter = string; +type ColumnIndex = number; // 0-indexed. +// Requires a subsequent call to get the column names. +export interface GoogleSheetMetadata { + name: GoogleSheetName; + rowCount: number; + mappedColumns: { + // Timestamp and score are required for every sheet, some other sheets only have email or member number. + timestamp: ColumnIndex; + score: ColumnIndex; + email: O.Option; + memberNumber: O.Option; + }; +} + +export const MAX_COLUMN_INDEX = 25; +// Doesn't support beyond 26 columns but actually thats fine for the current data. +export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter => + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index); + +export const extractGoogleSheetMetadata = + (logger: Logger) => + ( + initialMeta: { + properties: {title: string; gridProperties: {rowCount: number}}; + }, + firstRowData: GoogleSpreadsheetDataForSheet + ): O.Option => { + logger = logger.child({sheetName: initialMeta.properties.title}); + const columnNames = firstRowData.sheets[0].data[0].rowData[0].values.map( + col => col.formattedValue + ); + logger.trace('Found column names for sheet: %o', columnNames); + const timestamp = RA.findIndex( + val => val.toLowerCase() === 'timestamp' + )(columnNames); + if (O.isNone(timestamp)) { + logger.warn( + 'Failed to find timestamp column, skipping sheet: %s', + initialMeta.properties.title + ); + return O.none; + } + const score = RA.findIndex(val => val.toLowerCase() === 'score')( + columnNames + ); + if (O.isNone(score)) { + logger.warn( + 'Failed to find score column, skipping sheet: %s', + initialMeta.properties.title + ); + return O.none; + } + const memberNumber = RA.findIndex( + val => val.toLowerCase() === 'membership number' + )(columnNames); + const email = RA.findIndex(val => + EMAIL_COLUMN_NAMES.includes(val.toLowerCase()) + )(columnNames); + + return O.some({ + name: initialMeta.properties.title, + rowCount: initialMeta.properties.gridProperties.rowCount, + mappedColumns: { + timestamp: timestamp.value, + score: score.value, + email: email, + memberNumber: memberNumber, + }, + }); + }; diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index a442d6d4..3c8a4144 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -7,8 +7,9 @@ import {constructEvent, EventOfType} from '../types/domain-event'; import {v4} from 'uuid'; import {UUID} from 'io-ts-types'; import {DateTime} from 'luxon'; -import {sheets_v4} from '@googleapis/sheets'; import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-types'; +import {GoogleSheetMetadata} from './extract-metadata'; +import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data'; // Bounds to prevent clearly broken parsing. const MIN_RECOGNISED_MEMBER_NUMBER = 0; @@ -21,15 +22,6 @@ const MAX_VALID_TIMESTAMP_EPOCH_MS = const FORM_RESPONSES_SHEET_REGEX = /^Form Responses [0-9]*/i; -const extractRowFormattedValues = ( - row: sheets_v4.Schema$RowData -): O.Option => { - if (row.values) { - return O.some(row.values.map(cd => cd.formattedValue ?? '')); - } - return O.none; -}; - const extractScore = ( rowValue: string | undefined | null ): O.Option<{ @@ -130,100 +122,51 @@ const extractTimestamp = ( } }; -const EMAIL_COLUMN_NAMES = ['email address', 'email']; - -type SheetInfo = { - columnIndexes: { - timestamp: O.Option; - email: O.Option; - score: O.Option; - memberNumber: O.Option; - }; - columnNames: string[]; -}; - -const extractQuizSheetInformation = - (logger: Logger) => - (firstRow: sheets_v4.Schema$RowData): O.Option => { - const columnNames = extractRowFormattedValues(firstRow); - if (O.isNone(columnNames)) { - logger.debug('Failed to find column names'); - return O.none; - } - logger.trace('Found column names for sheet %o', columnNames.value); - - return O.some({ - columnIndexes: { - timestamp: RA.findIndex( - val => val.toLowerCase() === 'timestamp' - )(columnNames.value), - email: RA.findIndex(val => - EMAIL_COLUMN_NAMES.includes(val.toLowerCase()) - )(columnNames.value), - score: RA.findIndex(val => val.toLowerCase() === 'score')( - columnNames.value - ), - memberNumber: RA.findIndex( - val => val.toLowerCase() === 'membership number' - )(columnNames.value), - }, - columnNames: columnNames.value, - }); - }; - const extractFromRow = ( logger: Logger, - sheetInfo: SheetInfo, + metadata: GoogleSheetMetadata, equipmentId: UUID, trainingSheetId: string, timezone: string ) => - ( - row: sheets_v4.Schema$RowData - ): O.Option> => { - if (!row.values) { - return O.none; - } - - const email = O.isSome(sheetInfo.columnIndexes.email) + (row: { + values: {formattedValue: string}[]; + }): O.Option> => { + const email = O.isSome(metadata.mappedColumns.email) ? extractEmail( - row.values[sheetInfo.columnIndexes.email.value].formattedValue + row.values[metadata.mappedColumns.email.value].formattedValue ) : O.none; - const memberNumber = O.isSome(sheetInfo.columnIndexes.memberNumber) + const memberNumber = O.isSome(metadata.mappedColumns.memberNumber) ? extractMemberNumber( - row.values[sheetInfo.columnIndexes.memberNumber.value].formattedValue - ) - : O.none; - const score = O.isSome(sheetInfo.columnIndexes.score) - ? extractScore( - row.values[sheetInfo.columnIndexes.score.value].formattedValue - ) - : O.none; - const timestampEpochMS = O.isSome(sheetInfo.columnIndexes.timestamp) - ? extractTimestamp( - timezone, - row.values[sheetInfo.columnIndexes.timestamp.value].formattedValue + row.values[metadata.mappedColumns.memberNumber.value].formattedValue ) : O.none; + const score = extractScore( + row.values[metadata.mappedColumns.score].formattedValue + ); + const timestampEpochMS = extractTimestamp( + timezone, + row.values[metadata.mappedColumns.timestamp].formattedValue + ); if (O.isNone(email) && O.isNone(memberNumber)) { // Note that some quizes only require the member number. logger.warn( 'Failed to extract email or member number from row, skipping quiz result' ); - logger.trace('Skipped quiz row: %O', row.values); + logger.trace('Skipped quiz row: %O', row); return O.none; } if (O.isNone(score)) { logger.warn('Failed to extract score from row, skipped row'); - logger.trace('Skipped quiz row: %o', row.values); + logger.trace('Skipped quiz row: %o', row); return O.none; } if (O.isNone(timestampEpochMS)) { logger.warn('Failed to extract timestamp from row, skipped row'); - logger.trace('Skipped quiz row: %o', row.values); + logger.trace('Skipped quiz row: %o', row); return O.none; } return O.some( @@ -246,74 +189,40 @@ export const extractGoogleSheetData = logger: Logger, trainingSheetId: string, equipmentId: UUID, + metadata: GoogleSheetMetadata, + timezone: string, + // Note we filter events on timestamp rather than last row currently to handle + // blank rows but potentially we could switch if we added detection for blank rows. eventsFromExclusive: O.Option ) => ( - spreadsheet: sheets_v4.Schema$Spreadsheet - ): ReadonlyArray>> => - !spreadsheet.sheets || spreadsheet.sheets.length < 1 - ? [] - : spreadsheet.sheets.map(sheet => { - const title = sheet.properties?.title; - if (!title) { - logger.warn('Skipping sheet due to missing title'); - return []; - } - if (!FORM_RESPONSES_SHEET_REGEX.test(title)) { - logger.warn( - `Skipping sheet '${title}' as title doesn't match expected for form responses` - ); - } - - if ( - !sheet.data || - sheet.data.length < 1 || - !sheet.data[0].rowData || - sheet.data[0].rowData.length < 1 - ) { - logger.warn(`Skipping sheet '${title}' as missing data`); - return []; - } - - let timezone = spreadsheet.properties?.timeZone; - if (!timezone || !DateTime.local().setZone(timezone).isValid) { - // Not all the google form sheets are actually in Europe/London. - // Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). - logger.info( - `Unable to determine timezone for google sheet '${spreadsheet.properties?.title}', '${timezone}' - defaulting to Europe/London` - ); - timezone = 'Europe/London'; - } - - const [headers, ...data] = sheet.data[0].rowData; + spreadsheet: GoogleSpreadsheetDataForSheet + ): ReadonlyArray> => { + return pipe( + spreadsheet.sheets[0].data[0].rowData, + RA.map( + extractFromRow(logger, metadata, equipmentId, trainingSheetId, timezone) + ), + RA.filterMap(e => e), + RA.filter( + e => + O.isNone(eventsFromExclusive) || + e.timestampEpochMS > eventsFromExclusive.value + ) + ); + }; - return pipe( - headers, - extractQuizSheetInformation(logger), - O.match( - () => { - logger.warn('Failed to extract sheet info'); - return []; - }, - sheetInfo => - pipe( - data, - RA.map( - extractFromRow( - logger, - sheetInfo, - equipmentId, - trainingSheetId, - timezone - ) - ), - RA.filterMap(e => e), - RA.filter( - e => - O.isNone(eventsFromExclusive) || - e.timestampEpochMS > eventsFromExclusive.value - ) - ) - ) - ); - }); +export const shouldPullFromSheet = (sheet: { + properties: { + title: string; + }; +}): boolean => FORM_RESPONSES_SHEET_REGEX.test(sheet.properties.title); + +export const columnBoundsRequired = ( + sheet: GoogleSheetMetadata +): [number, number] => { + const colIndexes = Object.values(sheet.mappedColumns) + .filter(col => typeof col === 'number' || O.isSome(col)) + .map(col => (typeof col === 'number' ? col : col.value)); + return [Math.min(...colIndexes), Math.max(...colIndexes)]; +}; diff --git a/src/util.ts b/src/util.ts index a9754059..e06727b1 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,4 +1,8 @@ import {Logger} from 'pino'; +import * as t from 'io-ts'; +import * as tt from 'io-ts-types'; +import * as E from 'fp-ts/Either'; +import {pipe} from 'fp-ts/lib/function'; export const logPassThru = (logger: Logger, msg: string) => @@ -6,3 +10,31 @@ export const logPassThru = logger.info(msg); return input; }; + +export const getChunkIndexes = ( + startInclusive: number, + endInclusive: number, + chunkSize: number +): [number, number][] => { + const result: [number, number][] = []; + let start = startInclusive; + while (start < endInclusive) { + let end = start + chunkSize; + end = end > endInclusive ? endInclusive : end; + result.push([start, end]); + start = end + 1; + } + return result; +}; + +export const withDefaultIfEmpty = ( + codec: C, + ifEmpty: t.TypeOf +) => + tt.withValidate(codec, (input, context) => + pipe( + tt.NonEmptyString.validate(input, context), + E.orElse(() => t.success(String(ifEmpty))), + E.chain(nonEmptyString => codec.validate(nonEmptyString, context)) + ) + ); diff --git a/tests/data/google_sheet_data.ts b/tests/data/google_sheet_data.ts index 7c612a3f..a8c4965e 100644 --- a/tests/data/google_sheet_data.ts +++ b/tests/data/google_sheet_data.ts @@ -1,6 +1,12 @@ import {sheets_v4} from '@googleapis/sheets'; import {readFileSync} from 'node:fs'; import {EpochTimestampMilliseconds} from '../../src/read-models/shared-state/return-types'; +import { + GoogleSpreadsheetDataForSheet, + GoogleSpreadsheetInitialMetadata, +} from '../../src/init-dependencies/google/pull_sheet_data'; +import {GoogleSheetName} from '../../src/training-sheets/extract-metadata'; +import {getRightOrFail} from '../helpers'; type ManualParsedEntry = { emailProvided: string; @@ -13,22 +19,47 @@ type ManualParsedEntry = { }; export type ManualParsed = { - data: sheets_v4.Schema$Spreadsheet; + apiResp: sheets_v4.Schema$Spreadsheet; + metadata: GoogleSpreadsheetInitialMetadata; + sheets: Record; entries: ManualParsedEntry[]; }; -export const EMPTY: ManualParsed = { - data: JSON.parse( +const genManualParsed = ( + // Enumlate the google api which filters out other sheets using the range parameter. + apiResp: sheets_v4.Schema$Spreadsheet, + entries: ManualParsedEntry[] +): ManualParsed => { + const sheets: Record = {}; + for (const sheet of apiResp.sheets!) { + const apiRespCopy = JSON.parse(JSON.stringify(apiResp)) as typeof apiResp; + apiRespCopy.sheets = apiRespCopy.sheets!.filter( + s => s.properties!.title === sheet.properties!.title + ); + sheets[sheet.properties!.title as string] = getRightOrFail( + GoogleSpreadsheetDataForSheet.decode(apiRespCopy) + ); + } + return { + apiResp, + entries, + sheets, + metadata: getRightOrFail(GoogleSpreadsheetInitialMetadata.decode(apiResp)), + }; +}; + +export const EMPTY = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_empty.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - entries: [], -}; -export const METAL_LATHE: ManualParsed = { - data: JSON.parse( + [] +); + +export const METAL_LATHE: ManualParsed = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_metal_lathe.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - // Manually parsed data for testing: - entries: [ + [ { emailProvided: 'test@makespace.com', memberNumberProvided: 1234, @@ -38,14 +69,14 @@ export const METAL_LATHE: ManualParsed = { fullMarks: false, timestampEpochMS: 1705770960_000 as EpochTimestampMilliseconds, }, - ], -}; + ] +); -export const BAMBU: ManualParsed = { - data: JSON.parse( +export const BAMBU: ManualParsed = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_bambu.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - entries: [ + [ // Manually parsed data for testing: { emailProvided: 'flonn@example.com', @@ -83,13 +114,14 @@ export const BAMBU: ManualParsed = { fullMarks: true, timestampEpochMS: 1710249842_000 as EpochTimestampMilliseconds, }, - ], -}; -export const LASER_CUTTER: ManualParsed = { - data: JSON.parse( + ] +); + +export const LASER_CUTTER: ManualParsed = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_laser_cutter.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - entries: [ + [ // Manually parsed data for testing // Note some entries were missing in the source spreadsheet so are treated as '' to match the spreadsheet behaviour. { @@ -110,8 +142,8 @@ export const LASER_CUTTER: ManualParsed = { fullMarks: true, timestampEpochMS: 1601298462_000 as EpochTimestampMilliseconds, }, - ], -}; + ] +); export const getLatestEvent = (data: ManualParsed) => data.entries.sort((a, b) => a.timestampEpochMS - b.timestampEpochMS)[ @@ -129,8 +161,8 @@ export const NOT_FOUND_ERROR = { code: '404', }; export const TRAINING_SHEETS = { - [EMPTY.data.spreadsheetId!]: EMPTY, - [METAL_LATHE.data.spreadsheetId!]: METAL_LATHE, - [LASER_CUTTER.data.spreadsheetId!]: LASER_CUTTER, - [BAMBU.data.spreadsheetId!]: BAMBU, + [EMPTY.apiResp.spreadsheetId!]: EMPTY, + [METAL_LATHE.apiResp.spreadsheetId!]: METAL_LATHE, + [LASER_CUTTER.apiResp.spreadsheetId!]: LASER_CUTTER, + [BAMBU.apiResp.spreadsheetId!]: BAMBU, }; diff --git a/tests/data/google_spreadsheets_empty.json b/tests/data/google_spreadsheets_empty.json index 17fcc40c..825af9f8 100644 --- a/tests/data/google_spreadsheets_empty.json +++ b/tests/data/google_spreadsheets_empty.json @@ -139,7 +139,7 @@ "index": 0, "sheetType": "GRID", "gridProperties": { - "rowCount": 190, + "rowCount": 1, "columnCount": 25, "frozenRowCount": 1 } diff --git a/tests/init-dependencies/happy-path-adapters.helper.ts b/tests/init-dependencies/happy-path-adapters.helper.ts index 15264ca8..f5f516ee 100644 --- a/tests/init-dependencies/happy-path-adapters.helper.ts +++ b/tests/init-dependencies/happy-path-adapters.helper.ts @@ -7,7 +7,7 @@ import {faker} from '@faker-js/faker'; import {EventName} from '../../src/types/domain-event'; import {initSharedReadModel} from '../../src/read-models/shared-state'; import * as libsqlClient from '@libsql/client'; -import {localPullGoogleSheetData} from './pull-local-google'; +import {localGoogleHelpers} from './pull-local-google'; export const happyPathAdapters: Dependencies = { commitEvent: () => () => @@ -20,7 +20,7 @@ export const happyPathAdapters: Dependencies = { level: 'fatal', timestamp: pino.stdTimeFunctions.isoTime, }), - O.some(localPullGoogleSheetData), + O.some(localGoogleHelpers), 120_000 ), logger: (() => undefined) as never as Logger, diff --git a/tests/init-dependencies/pull-local-google.ts b/tests/init-dependencies/pull-local-google.ts index 6918582b..e4b0eb79 100644 --- a/tests/init-dependencies/pull-local-google.ts +++ b/tests/init-dependencies/pull-local-google.ts @@ -1,17 +1,55 @@ import {Logger} from 'pino'; import * as TE from 'fp-ts/TaskEither'; import * as gsheetData from '../data/google_sheet_data'; -import {PullSheetData} from '../../src/read-models/shared-state/async-apply-external-event-sources'; +import { + GoogleHelpers, + GoogleSpreadsheetDataForSheet, + GoogleSpreadsheetInitialMetadata, +} from '../../src/init-dependencies/google/pull_sheet_data'; +import {NonEmptyArray} from 'fp-ts/lib/NonEmptyArray'; -export const localPullGoogleSheetData: PullSheetData = ( +const localPullGoogleSheetDataMetadata = ( logger: Logger, trainingSheetId: string -) => { +): TE.TaskEither => { + logger.debug(`Pulling local google sheet metadata '${trainingSheetId}'`); + const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].metadata; + return sheet ? TE.right(sheet) : TE.left('Spreadsheet not found'); +}; + +const clone = (v: T): T => JSON.parse(JSON.stringify(v)) as T; + +const localPullGoogleSheetData = ( + logger: Logger, + trainingSheetId: string, + sheetName: string, + rowStart: number, + rowEnd: number, + _columnStartIndex: number, + _columnEndIndex: number +): TE.TaskEither => { logger.debug(`Pulling local google sheet '${trainingSheetId}'`); - const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].data; - return sheet - ? TE.right(sheet) - : TE.left({ - message: 'Sheet not found', - }); + const sheet = clone( + gsheetData.TRAINING_SHEETS[trainingSheetId].sheets[sheetName] + ); + if (sheet) { + if (rowStart > sheet.sheets[0].data[0].rowData.length) { + return TE.left('Sheet no more data'); + } + sheet.sheets[0].data[0].rowData = sheet.sheets[0].data[0].rowData.slice( + rowStart - 1, // 1 indexed. + rowEnd + ) as NonEmptyArray<{ + values: NonEmptyArray<{ + formattedValue: string; + }>; + }>; + return TE.right(sheet); + } + return TE.left('Sheet not found'); +}; + +export const localGoogleHelpers: GoogleHelpers = { + pullGoogleSheetData: localPullGoogleSheetData, + pullGoogleSheetDataMetadata: localPullGoogleSheetDataMetadata, }; diff --git a/tests/read-models/test-framework.ts b/tests/read-models/test-framework.ts index 14606bb6..5fee0ad4 100644 --- a/tests/read-models/test-framework.ts +++ b/tests/read-models/test-framework.ts @@ -18,7 +18,7 @@ import {EventName, EventOfType} from '../../src/types/domain-event'; import {Dependencies} from '../../src/dependencies'; import {applyToResource} from '../../src/commands/apply-command-to-resource'; import {initSharedReadModel} from '../../src/read-models/shared-state'; -import {localPullGoogleSheetData} from '../init-dependencies/pull-local-google'; +import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; type ToFrameworkCommands = { [K in keyof T]: { @@ -57,7 +57,7 @@ export const initTestFramework = async ( const sharedReadModel = initSharedReadModel( dbClient, logger, - O.some(localPullGoogleSheetData), + O.some(localGoogleHelpers), googleRateLimitMs ); const frameworkCommitEvent = commitEvent( diff --git a/tests/training-sheets/async-apply-external.test.ts b/tests/training-sheets/async-apply-external.test.ts index edfd0584..998fd8d9 100644 --- a/tests/training-sheets/async-apply-external.test.ts +++ b/tests/training-sheets/async-apply-external.test.ts @@ -61,13 +61,13 @@ describe('Integration asyncApplyExternalEventSources', () => { framework, 'bambu', areaId, - O.some(gsheetData.BAMBU.data.spreadsheetId!) + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) ); const lathe = await addWithSheet( framework, 'Metal Lathe', areaId, - O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) + O.some(gsheetData.METAL_LATHE.apiResp.spreadsheetId!) ); const results = await runAsyncApplyExternalEventSources(framework); checkLastQuizSyncUpdated(results); @@ -111,65 +111,60 @@ describe('Integration asyncApplyExternalEventSources', () => { timestamp: new Date(gsheetData.METAL_LATHE.entries[0].timestampEpochMS), }); }); - // it('Handle no equipment', async () => { - // const framework = await initTestFramework(1000); - // const results = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results); - // expect(results.equipmentAfter.size).toStrictEqual(0); - // }); - // it('Handle equipment with no training sheet', async () => { - // const framework = await initTestFramework(1000); - // const areaId = await addArea(framework); - // const bambu = await addWithSheet(framework, 'bambu', areaId, O.none); - // const results = await runAsyncApplyExternalEventSources(framework); - // expect( - // results.equipmentAfter.get(bambu.id)!.lastQuizSync // No training sheet so not updated. - // ).toStrictEqual(O.none); - // expect( - // results.equipmentAfter.get(bambu.id)!.lastQuizResult - // ).toStrictEqual(O.none); - // expect(results.newEvents).toHaveLength(0); - // }); - // it('Rate limit equipment pull', async () => { - // const framework = await initTestFramework(1000); - // const areaId = await addArea(framework); - // const bambu = await addWithSheet( - // framework, - // 'bambu', - // areaId, - // O.some(gsheetData.BAMBU.data.spreadsheetId!) - // ); - // const results1 = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results1); - // const results2 = await runAsyncApplyExternalEventSources(framework); - // expect( - // results1.equipmentAfter.get(bambu.id)!.lastQuizSync - // ).toStrictEqual(results2.equipmentAfter.get(bambu.id)!.lastQuizSync); - // expect(results1.newEvents.length).toBeGreaterThan(0); - // expect(results2.newEvents).toHaveLength(0); - // }); - // it('Repeat equipment pull no rate limit', async () => { - // const rateLimitMs = 100; - // const framework = await initTestFramework(rateLimitMs); - // const areaId = await addArea(framework); - // const bambu = await addWithSheet( - // framework, - // 'bambu', - // areaId, - // O.some(gsheetData.BAMBU.data.spreadsheetId!) - // ); - // const results1 = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results1); + it('Handle no equipment', async () => { + const framework = await initTestFramework(1000); + const results = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results); + expect(results.equipmentAfter.size).toStrictEqual(0); + }); + it('Handle equipment with no training sheet', async () => { + const framework = await initTestFramework(1000); + const areaId = await addArea(framework); + const bambu = await addWithSheet(framework, 'bambu', areaId, O.none); + const results = await runAsyncApplyExternalEventSources(framework); + expect( + results.equipmentAfter.get(bambu.id)!.lastQuizSync // No training sheet so not updated. + ).toStrictEqual(O.none); + expect(results.equipmentAfter.get(bambu.id)!.lastQuizResult).toStrictEqual( + O.none + ); + }); + it('Rate limit equipment pull', async () => { + const framework = await initTestFramework(1000); + const areaId = await addArea(framework); + const bambu = await addWithSheet( + framework, + 'bambu', + areaId, + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) + ); + const results1 = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results1); + const results2 = await runAsyncApplyExternalEventSources(framework); + expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).toStrictEqual( + results2.equipmentAfter.get(bambu.id)!.lastQuizSync + ); + }); + it('Repeat equipment pull no rate limit', async () => { + const rateLimitMs = 100; + const framework = await initTestFramework(rateLimitMs); + const areaId = await addArea(framework); + const bambu = await addWithSheet( + framework, + 'bambu', + areaId, + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) + ); + const results1 = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results1); - // await new Promise(res => setTimeout(res, rateLimitMs)); - // const results2 = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results2); - // expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( - // results2.equipmentAfter.get(bambu.id)!.lastQuizSync - // ); - // expect(results1.newEvents.length).toBeGreaterThan(0); - // expect(results2.newEvents).toHaveLength(0); - // }); + await new Promise(res => setTimeout(res, rateLimitMs)); + const results2 = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results2); + expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( + results2.equipmentAfter.get(bambu.id)!.lastQuizSync + ); + }); }); type ApplyExternalEventsResults = { diff --git a/tests/training-sheets/col-bounds-required.test.ts b/tests/training-sheets/col-bounds-required.test.ts new file mode 100644 index 00000000..31e8905a --- /dev/null +++ b/tests/training-sheets/col-bounds-required.test.ts @@ -0,0 +1,50 @@ +import * as O from 'fp-ts/Option'; +import {columnBoundsRequired} from '../../src/training-sheets/google'; + +describe('columnBoundsRequired', () => { + [ + { + input: { + name: 'All populated', + rowCount: 0, + mappedColumns: { + timestamp: 1, + score: 0, + email: O.some(3), + memberNumber: O.some(2), + }, + }, + expected: [0, 3], + }, + { + input: { + name: 'Minimal required', + rowCount: 0, + mappedColumns: { + timestamp: 1, + score: 0, + email: O.none, + memberNumber: O.none, + }, + }, + expected: [0, 1], + }, + { + input: { + name: '1 populated', + rowCount: 0, + mappedColumns: { + timestamp: 1, + score: 4, + email: O.none, + memberNumber: O.some(0), + }, + }, + expected: [0, 4], + }, + ].forEach(({input, expected}) => { + it(`${input.name}`, () => { + expect(columnBoundsRequired(input)).toStrictEqual(expected); + }); + }); +}); diff --git a/tests/training-sheets/extract-metadata.test.ts b/tests/training-sheets/extract-metadata.test.ts new file mode 100644 index 00000000..a9bb69fd --- /dev/null +++ b/tests/training-sheets/extract-metadata.test.ts @@ -0,0 +1,25 @@ +import pino from 'pino'; +import * as O from 'fp-ts/Option'; +import {extractGoogleSheetMetadata} from '../../src/training-sheets/extract-metadata'; + +import * as gsheetData from '../data/google_sheet_data'; +import {getSomeOrFail} from '../helpers'; + +describe('extract metadata', () => { + it('Empty sheet', () => { + const metadata = gsheetData.EMPTY.metadata; + const result = getSomeOrFail( + extractGoogleSheetMetadata(pino())( + metadata.sheets[0], + Object.values(gsheetData.EMPTY.sheets)[0] + ) + ); + + expect(result.name).toStrictEqual('Form Responses 1'); + expect(result.rowCount).toStrictEqual(1); + expect(result.mappedColumns.timestamp).toStrictEqual(0); + expect(result.mappedColumns.email).toStrictEqual(O.some(1)); + expect(result.mappedColumns.score).toStrictEqual(2); + expect(result.mappedColumns.memberNumber).toStrictEqual(O.some(4)); + }); +}); diff --git a/tests/training-sheets/get-chunk-indexes.test.ts b/tests/training-sheets/get-chunk-indexes.test.ts new file mode 100644 index 00000000..6ca7101e --- /dev/null +++ b/tests/training-sheets/get-chunk-indexes.test.ts @@ -0,0 +1,26 @@ +import {getChunkIndexes} from '../../src/util'; + +describe('Get chunk indexes', () => { + [ + [2, 240, 500, [[2, 240]]], + [ + 2, + 2000, + 500, + [ + [2, 502], + [503, 1003], + [1004, 1504], + [1505, 2000], + ], + ], + [2, 1, 500, []], + [1000, 2, 500, []], + ].forEach(([start, end, chunkSize, expected]) => { + it(`${start.toString()}:${end.toString()}:${chunkSize.toString()}`, () => { + expect( + getChunkIndexes(start as number, end as number, chunkSize as number) + ).toStrictEqual(expected); + }); + }); +}); diff --git a/tests/training-sheets/google-timezone.test.ts b/tests/training-sheets/google-timezone.test.ts new file mode 100644 index 00000000..6e0aceb3 --- /dev/null +++ b/tests/training-sheets/google-timezone.test.ts @@ -0,0 +1,25 @@ +import {GoogleTimezone} from '../../src/init-dependencies/google/pull_sheet_data'; +import {getRightOrFail} from '../helpers'; + +describe('Google timezone parse', () => { + it('Empty default', () => { + expect(getRightOrFail(GoogleTimezone.decode(''))).toStrictEqual( + 'Europe/London' + ); + }); + it('Malformed default', () => { + expect(getRightOrFail(GoogleTimezone.decode(null))).toStrictEqual( + 'Europe/London' + ); + }); + it('Known timezone', () => { + expect(getRightOrFail(GoogleTimezone.decode('Africa/Cairo'))).toStrictEqual( + 'Africa/Cairo' + ); + }); + it('Unknown timezone', () => { + expect( + getRightOrFail(GoogleTimezone.decode('Makespace/Cambridge')) + ).toStrictEqual('Europe/London'); + }); +}); diff --git a/tests/training-sheets/process-events.test.ts b/tests/training-sheets/process-events.test.ts index 48b810e7..ff0014d8 100644 --- a/tests/training-sheets/process-events.test.ts +++ b/tests/training-sheets/process-events.test.ts @@ -1,16 +1,20 @@ import {UUID} from 'io-ts-types'; -import {EventOfType, isEventOfType} from '../../src/types/domain-event'; +import { + DomainEvent, + EventOfType, + isEventOfType, +} from '../../src/types/domain-event'; import pino from 'pino'; import * as RA from 'fp-ts/lib/ReadonlyArray'; import * as N from 'fp-ts/number'; import * as O from 'fp-ts/Option'; import * as gsheetData from '../data/google_sheet_data'; import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources'; -import {localPullGoogleSheetData} from '../init-dependencies/pull-local-google'; import { EpochTimestampMilliseconds, Equipment, } from '../../src/read-models/shared-state/return-types'; +import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; const sortQuizResults = RA.sort({ compare: (a, b) => @@ -25,15 +29,21 @@ const sortQuizResults = RA.sort({ ), }); -const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => - pullNewEquipmentQuizResults( +const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => { + const newEvents: DomainEvent[] = []; + await pullNewEquipmentQuizResults( pino({ level: 'fatal', timestamp: pino.stdTimeFunctions.isoTime, }), - localPullGoogleSheetData, - equipment - )(); + localGoogleHelpers, + equipment, + newEvent => { + newEvents.push(newEvent); + } + ); + return newEvents; +}; const defaultEquipment = (): Equipment => ({ id: 'ebedee32-49f4-4d36-a350-4fa7848792bf' as UUID, @@ -75,7 +85,7 @@ const pullEquipmentQuizResultsWrapper = async ( for (const event of events) { if (isEventOfType('EquipmentTrainingQuizResult')(event)) { result.quizResults.push(event); - } else if (isEventOfType('EquipmentTrainingQuizSync')) { + } else if (isEventOfType('EquipmentTrainingQuizSync')(event)) { result.quizSync.push(event); } else { throw new Error('Unexpected event type'); @@ -109,14 +119,14 @@ describe('Training sheets worker', () => { it('empty sheet produces no events, but does indicate a sync', async () => { const result = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.EMPTY.data.spreadsheetId!) + O.some(gsheetData.EMPTY.apiResp.spreadsheetId!) ); expect(result.quizResults).toHaveLength(0); checkQuizSync(result); }); it('metal lathe training sheet', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) + O.some(gsheetData.METAL_LATHE.apiResp.spreadsheetId!) ); checkQuizSync(results); expect(results.quizResults[0]).toMatchObject< @@ -124,13 +134,13 @@ describe('Training sheets worker', () => { >({ type: 'EquipmentTrainingQuizResult', equipmentId: defaultEquipment().id, - trainingSheetId: gsheetData.METAL_LATHE.data.spreadsheetId!, + trainingSheetId: gsheetData.METAL_LATHE.apiResp.spreadsheetId!, ...gsheetData.METAL_LATHE.entries[0], }); }); it('training sheet with a summary page', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.LASER_CUTTER.data.spreadsheetId!) + O.some(gsheetData.LASER_CUTTER.apiResp.spreadsheetId!) ); checkQuizSync(results); const expected: readonly Partial< @@ -138,7 +148,7 @@ describe('Training sheets worker', () => { >[] = gsheetData.LASER_CUTTER.entries.map(e => ({ type: 'EquipmentTrainingQuizResult', equipmentId: defaultEquipment().id, - trainingSheetId: gsheetData.LASER_CUTTER.data.spreadsheetId!, + trainingSheetId: gsheetData.LASER_CUTTER.apiResp.spreadsheetId!, actor: { tag: 'system', }, @@ -157,7 +167,7 @@ describe('Training sheets worker', () => { }); it('training sheet with multiple response pages (different quiz questions)', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!) + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) ); checkQuizSync(results); const expected: readonly Partial< @@ -165,7 +175,7 @@ describe('Training sheets worker', () => { >[] = gsheetData.BAMBU.entries.map(e => ({ type: 'EquipmentTrainingQuizResult', equipmentId: defaultEquipment().id, - trainingSheetId: gsheetData.BAMBU.data.spreadsheetId!, + trainingSheetId: gsheetData.BAMBU.apiResp.spreadsheetId!, actor: { tag: 'system', }, @@ -184,7 +194,7 @@ describe('Training sheets worker', () => { }); it('Only take new rows, date in future', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), O.some(Date.now() as EpochTimestampMilliseconds) ); checkQuizSync(results); @@ -192,7 +202,7 @@ describe('Training sheets worker', () => { }); it('Only take new rows, date in far past', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), O.some(0 as EpochTimestampMilliseconds) ); checkQuizSync(results); @@ -209,7 +219,7 @@ describe('Training sheets worker', () => { it('Only take new rows, exclude 1', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), O.some(1700768963_000 as EpochTimestampMilliseconds) ); checkQuizSync(results); @@ -218,7 +228,7 @@ describe('Training sheets worker', () => { it('Only take new rows, exclude 2', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), O.some(1700769348_000 as EpochTimestampMilliseconds) ); checkQuizSync(results); @@ -227,7 +237,7 @@ describe('Training sheets worker', () => { it('Only take new rows, exclude 3', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), O.some(1710249052_000 as EpochTimestampMilliseconds) ); checkQuizSync(results); @@ -236,7 +246,7 @@ describe('Training sheets worker', () => { it('Only take new rows, exclude all (already have latest)', async () => { const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), O.some(1710249842_000 as EpochTimestampMilliseconds) ); checkQuizSync(results);