From dca8070a9da1a301618718508380e82c504eb4df Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 00:13:49 +0100 Subject: [PATCH 01/20] We can be much more memory efficient if we are selective about what we pull back from google --- src/index.ts | 2 +- .../google/pull_sheet_data.ts | 50 +++++- src/init-dependencies/init-dependencies.ts | 36 ++-- .../async-apply-external-event-sources.ts | 104 +++++++---- src/read-models/shared-state/index.ts | 10 +- src/training-sheets/extract-metadata.ts | 162 ++++++++++++++++++ src/training-sheets/google.ts | 60 +------ .../happy-path-adapters.helper.ts | 4 +- tests/init-dependencies/pull-local-google.ts | 12 +- tests/read-models/test-framework.ts | 4 +- tests/training-sheets/process-events.test.ts | 4 +- 11 files changed, 324 insertions(+), 124 deletions(-) create mode 100644 src/training-sheets/extract-metadata.ts diff --git a/src/index.ts b/src/index.ts index f8297e0c..e8e5dae8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -73,7 +73,7 @@ const periodicExternalReadModelRefresh = setInterval(() => { 'Unexpected error when refreshing read model with external sources' ) ); -}, 60_000); +}, 30_000); server.on('close', () => { clearInterval(periodicReadModelRefresh); clearInterval(periodicExternalReadModelRefresh); diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 06e23003..1afb24a5 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -6,12 +6,48 @@ import {pipe} from 'fp-ts/lib/function'; import {sheets, sheets_v4} from '@googleapis/sheets'; import {GoogleAuth} from 'google-auth-library'; -export const pullGoogleSheetData = +export type GoogleSpreadsheetInitialMetadata = sheets_v4.Schema$Spreadsheet & { + readonly GoogleSpreadsheetInitialMetadata: unique symbol; +}; + +export type GoogleSpreadsheetDataForSheet = sheets_v4.Schema$Spreadsheet & { + readonly GoogleSpreadsheetDataForSheet: unique symbol; +}; + +export const pullGoogleSheetDataMetadata = (auth: GoogleAuth) => ( logger: Logger, trainingSheetId: string - ): TE.TaskEither => + ): TE.TaskEither => + pipe( + TE.tryCatch( + () => + sheets({ + version: 'v4', + auth, + }).spreadsheets.get({ + spreadsheetId: trainingSheetId, + includeGridData: false, // Only the metadata. + fields: 'sheets(properties)', // Only the metadata about the sheets. + }), + reason => { + logger.error(reason, 'Failed to get spreadsheet metadata'); + return `Failed to get training spreadsheet metadata ${trainingSheetId}`; + } + ), + TE.map(resp => resp.data as GoogleSpreadsheetInitialMetadata) + ); + +export const pullGoogleSheetData = + (auth: GoogleAuth) => + ( + logger: Logger, + trainingSheetId: string, + sheetName: string, + rowStart: number, // 1 indexed. + rowEnd: number + ): TE.TaskEither => pipe( TE.tryCatch( () => @@ -20,7 +56,8 @@ export const pullGoogleSheetData = auth, }).spreadsheets.get({ spreadsheetId: trainingSheetId, - includeGridData: true, + fields: 'sheets(data(rowData(values(formattedValue))))', + ranges: [`${sheetName}!${rowStart}:${rowEnd}`], }), reason => { logger.error(reason, 'Failed to get spreadsheet'); @@ -30,5 +67,10 @@ export const pullGoogleSheetData = }; } ), - TE.map(resp => resp.data) + TE.map(resp => resp.data as GoogleSpreadsheetDataForSheet) ); + +export interface GoogleHelpers { + pullGoogleSheetData: ReturnType; + pullGoogleSheetDataMetadata: ReturnType; +} diff --git a/src/init-dependencies/init-dependencies.ts b/src/init-dependencies/init-dependencies.ts index 34636063..b57c6799 100644 --- a/src/init-dependencies/init-dependencies.ts +++ b/src/init-dependencies/init-dependencies.ts @@ -10,7 +10,11 @@ import {commitEvent} from './event-store/commit-event'; import {getAllEvents, getAllEventsByType} from './event-store/get-all-events'; import {getResourceEvents} from './event-store/get-resource-events'; import {Client} from '@libsql/client'; -import {pullGoogleSheetData} from './google/pull_sheet_data'; +import { + GoogleHelpers, + pullGoogleSheetData, + pullGoogleSheetDataMetadata, +} from './google/pull_sheet_data'; import {initSharedReadModel} from '../read-models/shared-state'; import {GoogleAuth} from 'google-auth-library'; @@ -57,24 +61,26 @@ export const initDependencies = ( }) ); - const googleAuth = - conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON.toLowerCase().trim() === 'disabled' - ? O.none - : O.some( - pullGoogleSheetData( - new GoogleAuth({ - // Google issues the credentials file and validates it. - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - credentials: JSON.parse(conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON), - scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'], - }) - ) - ); + let googleHelpers: O.Option = O.none; + if ( + conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON.toLowerCase().trim() !== 'disabled' + ) { + const googleAuth = new GoogleAuth({ + // Google issues the credentials file and validates it. + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + credentials: JSON.parse(conf.GOOGLE_SERVICE_ACCOUNT_KEY_JSON), + scopes: ['https://www.googleapis.com/auth/spreadsheets.readonly'], + }); + googleHelpers = O.some({ + pullGoogleSheetData: pullGoogleSheetData(googleAuth), + pullGoogleSheetDataMetadata: pullGoogleSheetDataMetadata(googleAuth), + }); + } const sharedReadModel = initSharedReadModel( dbClient, logger, - googleAuth, + googleHelpers, conf.GOOGLE_RATELIMIT_MS ); diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 6101a3ca..88aee33c 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -1,43 +1,33 @@ import {Logger} from 'pino'; import * as T from 'fp-ts/Task'; +import * as E from 'fp-ts/Either'; import * as TE from 'fp-ts/TaskEither'; import * as O from 'fp-ts/Option'; import * as RA from 'fp-ts/ReadonlyArray'; -import {DomainEvent, Failure} from '../../types'; +import {DomainEvent} from '../../types'; import {BetterSQLite3Database} from 'drizzle-orm/better-sqlite3'; import {getAllEquipment} from './get-equipment'; import {pipe} from 'fp-ts/lib/function'; -import {sheets_v4} from '@googleapis/sheets'; import {EpochTimestampMilliseconds, Equipment} from './return-types'; -import {extractGoogleSheetData} from '../../training-sheets/google'; +import {extractGoogleSheetData, shouldPullFromSheet} from '../../training-sheets/google'; import {constructEvent, EventOfType} from '../../types/domain-event'; - -export type PullSheetData = ( - logger: Logger, - trainingSheetId: string -) => TE.TaskEither; +import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data'; +import { extractGoogleSheetMetadata, extractInitialGoogleSheetMetadata, GoogleSheetMetadata, GoogleSheetsMetadataInital } from '../../training-sheets/extract-metadata'; export const pullNewEquipmentQuizResults = ( logger: Logger, - pullGoogleSheetData: PullSheetData, - equipment: Equipment -): T.Task< - ReadonlyArray< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - > -> => { + googleHelpers: GoogleHelpers, + equipment: Equipment, + updateState: (event: DomainEvent) => void, +): T.Task => { + // TODO - Refactor this into fp-ts style. if (O.isNone(equipment.trainingSheetId)) { logger.warn( 'No training sheet registered for equipment %s, skipping training data ingestion', equipment.name ); // eslint-disable-next-line @typescript-eslint/require-await - return async () => - [] as ReadonlyArray< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - >; + return async () => {}; } const trainingSheetId = equipment.trainingSheetId.value; logger = logger.child({trainingSheetId}); @@ -45,8 +35,60 @@ export const pullNewEquipmentQuizResults = ( 'Scanning training sheet. Pulling google sheet data from %s...', equipment.lastQuizResult ); + + const initialRaw = await googleHelpers.pullGoogleSheetDataMetadata(logger, trainingSheetId)(); + if (E.isLeft(initialRaw)) { + logger.warn(initialRaw.left); + return async() => {}; + } + + const initialMeta = extractInitialGoogleSheetMetadata(initialRaw.right); + + if (E.isLeft(initialMeta)) { + logger.warn('Failed to get google sheet metadata for training sheet %s, skipping', trainingSheetId); + logger.warn(initialMeta.left); + return async () => {}; + } + + const sheets: GoogleSheetMetadata[] = []; + for (const sheet of initialMeta.right.sheets) { + if (!shouldPullFromSheet(sheet)) { + logger.warn( + `Skipping sheet as doesn't match expected for form responses` + ); + continue; + } + + const firstRowData = await googleHelpers.pullGoogleSheetData( + logger, + trainingSheetId, + sheet.name, + 1, + 1, + )(); + if (E.isLeft(firstRowData)) { + logger.warn('Failed to get google sheet first row data for sheet %s, skipping', sheet.name); + continue; + } + + const meta = extractGoogleSheetMetadata(logger)(sheet, firstRowData.right); + if (O.isNone(meta)) { + continue; + } + + logger.info('Got metadata for sheet: %s: %o', sheet.name, meta.value); + sheets.push(meta.value); + } + + + + + + return pipe( - pullGoogleSheetData(logger, trainingSheetId), + , + , + ) TE.map( extractGoogleSheetData( logger, @@ -84,13 +126,13 @@ export const pullNewEquipmentQuizResults = ( export const asyncApplyExternalEventSources = ( logger: Logger, currentState: BetterSQLite3Database, - pullGoogleSheetData: O.Option, + googleHelpers: O.Option, updateState: (event: DomainEvent) => void, googleRateLimitMs: number ) => { return () => async () => { logger.info('Applying external event sources...'); - if (O.isNone(pullGoogleSheetData)) { + if (O.isNone(googleHelpers)) { logger.info('Google external event source disabled'); return; } @@ -106,14 +148,12 @@ export const asyncApplyExternalEventSources = ( 'Triggering event update from google training sheets for %s...', equipment.name ); - pipe( - await pullNewEquipmentQuizResults( - logger, - pullGoogleSheetData.value, - equipment - )(), - RA.map(updateState) - ); + await pullNewEquipmentQuizResults( + logger, + googleHelpers.value, + equipment, + updateState + )(), logger.info( 'Finished pulling events from google training sheet for %s', equipment.name diff --git a/src/read-models/shared-state/index.ts b/src/read-models/shared-state/index.ts index 0ec2931e..888abaeb 100644 --- a/src/read-models/shared-state/index.ts +++ b/src/read-models/shared-state/index.ts @@ -10,11 +10,9 @@ import {Client} from '@libsql/client/.'; import {asyncRefresh} from './async-refresh'; import {updateState} from './update-state'; import {Logger} from 'pino'; -import { - asyncApplyExternalEventSources, - PullSheetData, -} from './async-apply-external-event-sources'; +import {asyncApplyExternalEventSources} from './async-apply-external-event-sources'; import {UUID} from 'io-ts-types'; +import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data'; export {replayState} from './deprecated-replay'; @@ -35,7 +33,7 @@ export type SharedReadModel = { export const initSharedReadModel = ( eventStoreClient: Client, logger: Logger, - pullGoogleSheetData: O.Option, + googleHelpers: O.Option, googleRateLimitMs: number ): SharedReadModel => { const readModelDb = drizzle(new Database()); @@ -48,7 +46,7 @@ export const initSharedReadModel = ( asyncApplyExternalEventSources: asyncApplyExternalEventSources( logger, readModelDb, - pullGoogleSheetData, + googleHelpers, updateState_, googleRateLimitMs ), diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts new file mode 100644 index 00000000..8c0200e7 --- /dev/null +++ b/src/training-sheets/extract-metadata.ts @@ -0,0 +1,162 @@ +import {pipe} from 'fp-ts/lib/function'; +import * as RA from 'fp-ts/ReadonlyArray'; +import * as O from 'fp-ts/Option'; +import * as t from 'io-ts'; +import * as tt from 'io-ts-types'; +import * as E from 'fp-ts/Either'; + +import {Logger} from 'pino'; +import { + GoogleSpreadsheetDataForSheet, + GoogleSpreadsheetInitialMetadata, +} from '../init-dependencies/google/pull_sheet_data'; + +const EMAIL_COLUMN_NAMES = ['email address', 'email']; + +type GoogleSheetName = string; +// What we can get from an initial call to google sheets without any rows. +export interface GoogleSheetMetadataInital { + name: GoogleSheetName; + rowCount: number; +} +export interface GoogleSheetsMetadataInital { + sheets: GoogleSheetMetadataInital[]; +} + +type ColumnLetter = string; +type ColumnIndex = number; // 0-indexed. +// Requires a subsequent call to get the column names. +export interface GoogleSheetMetadata { + name: GoogleSheetName; + rowCount: number; + requiredColumns: { + timestamp: ColumnIndex; + email: ColumnIndex; + score: ColumnIndex; + memberNumber: ColumnIndex; + }; +} +export interface GoogleSheetsMetadata { + sheets: GoogleSheetMetadata[]; +} + +// Doesn't support beyond 26 columns but actually thats fine for the current data. +export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter => + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index); + +const SheetProperties = t.strict({ + sheets: t.array( + t.strict({ + properties: t.strict({ + title: t.string, + gridProperties: t.strict({ + rowCount: t.number, + }), + }), + }) + ), +}); + +export const extractInitialGoogleSheetMetadata = ( + spreadsheet: GoogleSpreadsheetInitialMetadata +): E.Either => + pipe( + spreadsheet, + SheetProperties.decode, + E.mapLeft(_e => 'Failed to extract initial google sheet metadata'), + E.map(properties => ({ + sheets: properties.sheets.map(sheet => ({ + name: sheet.properties.title, + rowCount: sheet.properties.gridProperties.rowCount, + })), + })) + ); + +const SpreadsheetData = t.strict({ + sheets: tt.nonEmptyArray( + t.strict({ + data: tt.nonEmptyArray( + t.strict({ + rowData: tt.nonEmptyArray( + t.strict({ + values: tt.nonEmptyArray( + t.strict({ + formattedValue: t.string, + }) + ), + }) + ), + }) + ), + }) + ), +}); + +export const extractGoogleSheetMetadata = + (logger: Logger) => + ( + initialMeta: GoogleSheetMetadataInital, + spreadsheetDataForSheet: GoogleSpreadsheetDataForSheet + ): O.Option => { + logger = logger.child({sheetName: initialMeta.name}); + const validated = SpreadsheetData.decode(spreadsheetDataForSheet); + if (E.isLeft(validated)) { + logger.warn('Failed to validate spreadsheet data, skipping sheet'); + return O.none; + } + + const columnNames = validated.right.sheets[0].data[0].rowData[0].values.map( + col => col.formattedValue + ); + logger.trace('Found column names for sheet: %o', columnNames); + const timestamp = RA.findIndex( + val => val.toLowerCase() === 'timestamp' + )(columnNames); + if (O.isNone(timestamp)) { + logger.warn( + 'Failed to find timestamp column, skipping sheet: %s', + initialMeta.name + ); + return O.none; + } + const email = RA.findIndex(val => + EMAIL_COLUMN_NAMES.includes(val.toLowerCase()) + )(columnNames); + if (O.isNone(email)) { + logger.warn( + 'Failed to find email column, skipping sheet: %s', + initialMeta.name + ); + return O.none; + } + const score = RA.findIndex(val => val.toLowerCase() === 'score')( + columnNames + ); + if (O.isNone(score)) { + logger.warn( + 'Failed to find score column, skipping sheet: %s', + initialMeta.name + ); + return O.none; + } + const memberNumber = RA.findIndex( + val => val.toLowerCase() === 'membership number' + )(columnNames); + if (O.isNone(memberNumber)) { + logger.warn( + 'Failed to find member number column, skipping sheet: %s', + initialMeta.name + ); + return O.none; + } + + return O.some({ + ...initialMeta, + requiredColumns: { + timestamp: timestamp.value, + email: email.value, + score: score.value, + memberNumber: memberNumber.value, + }, + }); + }; diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index a442d6d4..dc6ae247 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -9,6 +9,7 @@ import {UUID} from 'io-ts-types'; import {DateTime} from 'luxon'; import {sheets_v4} from '@googleapis/sheets'; import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-types'; +import {GoogleSheetMetadataInital} from './extract-metadata'; // Bounds to prevent clearly broken parsing. const MIN_RECOGNISED_MEMBER_NUMBER = 0; @@ -21,15 +22,6 @@ const MAX_VALID_TIMESTAMP_EPOCH_MS = const FORM_RESPONSES_SHEET_REGEX = /^Form Responses [0-9]*/i; -const extractRowFormattedValues = ( - row: sheets_v4.Schema$RowData -): O.Option => { - if (row.values) { - return O.some(row.values.map(cd => cd.formattedValue ?? '')); - } - return O.none; -}; - const extractScore = ( rowValue: string | undefined | null ): O.Option<{ @@ -130,47 +122,6 @@ const extractTimestamp = ( } }; -const EMAIL_COLUMN_NAMES = ['email address', 'email']; - -type SheetInfo = { - columnIndexes: { - timestamp: O.Option; - email: O.Option; - score: O.Option; - memberNumber: O.Option; - }; - columnNames: string[]; -}; - -const extractQuizSheetInformation = - (logger: Logger) => - (firstRow: sheets_v4.Schema$RowData): O.Option => { - const columnNames = extractRowFormattedValues(firstRow); - if (O.isNone(columnNames)) { - logger.debug('Failed to find column names'); - return O.none; - } - logger.trace('Found column names for sheet %o', columnNames.value); - - return O.some({ - columnIndexes: { - timestamp: RA.findIndex( - val => val.toLowerCase() === 'timestamp' - )(columnNames.value), - email: RA.findIndex(val => - EMAIL_COLUMN_NAMES.includes(val.toLowerCase()) - )(columnNames.value), - score: RA.findIndex(val => val.toLowerCase() === 'score')( - columnNames.value - ), - memberNumber: RA.findIndex( - val => val.toLowerCase() === 'membership number' - )(columnNames.value), - }, - columnNames: columnNames.value, - }); - }; - const extractFromRow = ( logger: Logger, @@ -259,11 +210,6 @@ export const extractGoogleSheetData = logger.warn('Skipping sheet due to missing title'); return []; } - if (!FORM_RESPONSES_SHEET_REGEX.test(title)) { - logger.warn( - `Skipping sheet '${title}' as title doesn't match expected for form responses` - ); - } if ( !sheet.data || @@ -317,3 +263,7 @@ export const extractGoogleSheetData = ) ); }); + +export const shouldPullFromSheet = ( + sheet: GoogleSheetMetadataInital +): boolean => FORM_RESPONSES_SHEET_REGEX.test(sheet.name); diff --git a/tests/init-dependencies/happy-path-adapters.helper.ts b/tests/init-dependencies/happy-path-adapters.helper.ts index 15264ca8..f5f516ee 100644 --- a/tests/init-dependencies/happy-path-adapters.helper.ts +++ b/tests/init-dependencies/happy-path-adapters.helper.ts @@ -7,7 +7,7 @@ import {faker} from '@faker-js/faker'; import {EventName} from '../../src/types/domain-event'; import {initSharedReadModel} from '../../src/read-models/shared-state'; import * as libsqlClient from '@libsql/client'; -import {localPullGoogleSheetData} from './pull-local-google'; +import {localGoogleHelpers} from './pull-local-google'; export const happyPathAdapters: Dependencies = { commitEvent: () => () => @@ -20,7 +20,7 @@ export const happyPathAdapters: Dependencies = { level: 'fatal', timestamp: pino.stdTimeFunctions.isoTime, }), - O.some(localPullGoogleSheetData), + O.some(localGoogleHelpers), 120_000 ), logger: (() => undefined) as never as Logger, diff --git a/tests/init-dependencies/pull-local-google.ts b/tests/init-dependencies/pull-local-google.ts index 6918582b..db7d8bae 100644 --- a/tests/init-dependencies/pull-local-google.ts +++ b/tests/init-dependencies/pull-local-google.ts @@ -1,12 +1,9 @@ import {Logger} from 'pino'; import * as TE from 'fp-ts/TaskEither'; import * as gsheetData from '../data/google_sheet_data'; -import {PullSheetData} from '../../src/read-models/shared-state/async-apply-external-event-sources'; +import {GoogleHelpers} from '../../src/init-dependencies/google/pull_sheet_data'; -export const localPullGoogleSheetData: PullSheetData = ( - logger: Logger, - trainingSheetId: string -) => { +const localPullGoogleSheetData = (logger: Logger, trainingSheetId: string) => { logger.debug(`Pulling local google sheet '${trainingSheetId}'`); const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].data; return sheet @@ -15,3 +12,8 @@ export const localPullGoogleSheetData: PullSheetData = ( message: 'Sheet not found', }); }; + +export const localGoogleHelpers: GoogleHelpers = { + pullGoogleSheetData: localPullGoogleSheetData, + pullGoogleSheetDataMetadata: localPullGoogleSheetData, +}; diff --git a/tests/read-models/test-framework.ts b/tests/read-models/test-framework.ts index 14606bb6..5fee0ad4 100644 --- a/tests/read-models/test-framework.ts +++ b/tests/read-models/test-framework.ts @@ -18,7 +18,7 @@ import {EventName, EventOfType} from '../../src/types/domain-event'; import {Dependencies} from '../../src/dependencies'; import {applyToResource} from '../../src/commands/apply-command-to-resource'; import {initSharedReadModel} from '../../src/read-models/shared-state'; -import {localPullGoogleSheetData} from '../init-dependencies/pull-local-google'; +import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; type ToFrameworkCommands = { [K in keyof T]: { @@ -57,7 +57,7 @@ export const initTestFramework = async ( const sharedReadModel = initSharedReadModel( dbClient, logger, - O.some(localPullGoogleSheetData), + O.some(localGoogleHelpers), googleRateLimitMs ); const frameworkCommitEvent = commitEvent( diff --git a/tests/training-sheets/process-events.test.ts b/tests/training-sheets/process-events.test.ts index 48b810e7..f43aeb0f 100644 --- a/tests/training-sheets/process-events.test.ts +++ b/tests/training-sheets/process-events.test.ts @@ -6,11 +6,11 @@ import * as N from 'fp-ts/number'; import * as O from 'fp-ts/Option'; import * as gsheetData from '../data/google_sheet_data'; import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources'; -import {localPullGoogleSheetData} from '../init-dependencies/pull-local-google'; import { EpochTimestampMilliseconds, Equipment, } from '../../src/read-models/shared-state/return-types'; +import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; const sortQuizResults = RA.sort({ compare: (a, b) => @@ -31,7 +31,7 @@ const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => level: 'fatal', timestamp: pino.stdTimeFunctions.isoTime, }), - localPullGoogleSheetData, + localGoogleHelpers, equipment )(); From 2506c165b597b816f960435cb8dcae9b9e099e5f Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 00:37:13 +0100 Subject: [PATCH 02/20] Rough pass of how everything fits together --- .../async-apply-external-event-sources.ts | 154 +++++++++++------- src/training-sheets/google.ts | 2 +- src/util.ts | 16 ++ tests/get-chunk-indexes.test.ts | 25 +++ 4 files changed, 138 insertions(+), 59 deletions(-) create mode 100644 tests/get-chunk-indexes.test.ts diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 88aee33c..9d0704ae 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -1,7 +1,5 @@ import {Logger} from 'pino'; -import * as T from 'fp-ts/Task'; import * as E from 'fp-ts/Either'; -import * as TE from 'fp-ts/TaskEither'; import * as O from 'fp-ts/Option'; import * as RA from 'fp-ts/ReadonlyArray'; import {DomainEvent} from '../../types'; @@ -9,17 +7,73 @@ import {BetterSQLite3Database} from 'drizzle-orm/better-sqlite3'; import {getAllEquipment} from './get-equipment'; import {pipe} from 'fp-ts/lib/function'; import {EpochTimestampMilliseconds, Equipment} from './return-types'; -import {extractGoogleSheetData, shouldPullFromSheet} from '../../training-sheets/google'; -import {constructEvent, EventOfType} from '../../types/domain-event'; +import { + extractGoogleSheetData, + shouldPullFromSheet, +} from '../../training-sheets/google'; +import {constructEvent} from '../../types/domain-event'; import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data'; -import { extractGoogleSheetMetadata, extractInitialGoogleSheetMetadata, GoogleSheetMetadata, GoogleSheetsMetadataInital } from '../../training-sheets/extract-metadata'; +import { + extractGoogleSheetMetadata, + extractInitialGoogleSheetMetadata, + GoogleSheetMetadata, +} from '../../training-sheets/extract-metadata'; +import {getChunkIndexes} from '../../util'; -export const pullNewEquipmentQuizResults = ( +const ROW_BATCH_SIZE = 500; + +export const pullNewEquipmentQuizResultsForSheet = async ( logger: Logger, googleHelpers: GoogleHelpers, equipment: Equipment, - updateState: (event: DomainEvent) => void, -): T.Task => { + trainingSheetId: string, + sheet: GoogleSheetMetadata, + updateState: (event: DomainEvent) => void +) => { + logger.info('Processing sheet %s', sheet.name); + for (const [rowStart, rowEnd] of getChunkIndexes( + 2, + sheet.rowCount, + ROW_BATCH_SIZE + )) { + logger.debug( + 'Pulling data for sheet %s rows %s to %s', + sheet.name, + rowStart, + rowEnd + ); + const data = await googleHelpers.pullGoogleSheetData( + logger, + trainingSheetId, + sheet.name, + rowStart, + rowEnd + )(); + if (E.isLeft(data)) { + logger.debug( + 'Failed to pull data for sheet %s rows %s to %s, skipping rest of sheet' + ); + return; + } + pipe( + data.right, + extractGoogleSheetData( + logger, + trainingSheetId, + equipment.id, + equipment.lastQuizResult + ), + RA.map(updateState) + ); + } +}; + +export const pullNewEquipmentQuizResults = async ( + logger: Logger, + googleHelpers: GoogleHelpers, + equipment: Equipment, + updateState: (event: DomainEvent) => void +): Promise => { // TODO - Refactor this into fp-ts style. if (O.isNone(equipment.trainingSheetId)) { logger.warn( @@ -27,7 +81,7 @@ export const pullNewEquipmentQuizResults = ( equipment.name ); // eslint-disable-next-line @typescript-eslint/require-await - return async () => {}; + return; } const trainingSheetId = equipment.trainingSheetId.value; logger = logger.child({trainingSheetId}); @@ -36,25 +90,31 @@ export const pullNewEquipmentQuizResults = ( equipment.lastQuizResult ); - const initialRaw = await googleHelpers.pullGoogleSheetDataMetadata(logger, trainingSheetId)(); + const initialRaw = await googleHelpers.pullGoogleSheetDataMetadata( + logger, + trainingSheetId + )(); if (E.isLeft(initialRaw)) { logger.warn(initialRaw.left); - return async() => {}; + return; } const initialMeta = extractInitialGoogleSheetMetadata(initialRaw.right); if (E.isLeft(initialMeta)) { - logger.warn('Failed to get google sheet metadata for training sheet %s, skipping', trainingSheetId); + logger.warn( + 'Failed to get google sheet metadata for training sheet %s, skipping', + trainingSheetId + ); logger.warn(initialMeta.left); - return async () => {}; + return; } const sheets: GoogleSheetMetadata[] = []; for (const sheet of initialMeta.right.sheets) { if (!shouldPullFromSheet(sheet)) { logger.warn( - `Skipping sheet as doesn't match expected for form responses` + "Skipping sheet as doesn't match expected for form responses" ); continue; } @@ -64,10 +124,13 @@ export const pullNewEquipmentQuizResults = ( trainingSheetId, sheet.name, 1, - 1, + 1 )(); if (E.isLeft(firstRowData)) { - logger.warn('Failed to get google sheet first row data for sheet %s, skipping', sheet.name); + logger.warn( + 'Failed to get google sheet first row data for sheet %s, skipping', + sheet.name + ); continue; } @@ -80,45 +143,20 @@ export const pullNewEquipmentQuizResults = ( sheets.push(meta.value); } - + for (const sheet of sheets) { + await pullNewEquipmentQuizResultsForSheet( + logger, + googleHelpers, + equipment, + trainingSheetId, + sheet, + updateState + ); + } - - - - return pipe( - , - , - ) - TE.map( - extractGoogleSheetData( - logger, - trainingSheetId, - equipment.id, - equipment.lastQuizResult - ) - ), - TE.map(RA.flatten), - TE.map( - RA.append< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - >( - constructEvent('EquipmentTrainingQuizSync')({ - equipmentId: equipment.id, - }) - ) - ), - // eslint-disable-next-line @typescript-eslint/require-await - TE.getOrElse(err => async () => { - logger.error( - 'Failed to receive data from google sheets for equipment %s: %s', - equipment.name, - err.message - ); - return [] as ReadonlyArray< - | EventOfType<'EquipmentTrainingQuizResult'> - | EventOfType<'EquipmentTrainingQuizSync'> - >; + updateState( + constructEvent('EquipmentTrainingQuizSync')({ + equipmentId: equipment.id, }) ); }; @@ -154,10 +192,10 @@ export const asyncApplyExternalEventSources = ( equipment, updateState )(), - logger.info( - 'Finished pulling events from google training sheet for %s', - equipment.name - ); + logger.info( + 'Finished pulling events from google training sheet for %s', + equipment.name + ); } } logger.info('Finished applying external event sources'); diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index dc6ae247..4ac0503f 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -201,7 +201,7 @@ export const extractGoogleSheetData = ) => ( spreadsheet: sheets_v4.Schema$Spreadsheet - ): ReadonlyArray>> => + ): ReadonlyArray> => !spreadsheet.sheets || spreadsheet.sheets.length < 1 ? [] : spreadsheet.sheets.map(sheet => { diff --git a/src/util.ts b/src/util.ts index a9754059..ee5b2f3d 100644 --- a/src/util.ts +++ b/src/util.ts @@ -6,3 +6,19 @@ export const logPassThru = logger.info(msg); return input; }; + +export const getChunkIndexes = ( + startInclusive: number, + endInclusive: number, + chunkSize: number +): [number, number][] => { + const result: [number, number][] = []; + let start = startInclusive; + while (start < endInclusive) { + let end = start + chunkSize; + end = end > endInclusive ? endInclusive : end; + result.push([start, end]); + start = end + 1; + } + return result; +}; diff --git a/tests/get-chunk-indexes.test.ts b/tests/get-chunk-indexes.test.ts new file mode 100644 index 00000000..7227ecd0 --- /dev/null +++ b/tests/get-chunk-indexes.test.ts @@ -0,0 +1,25 @@ +import {getChunkIndexes} from '../src/util'; + +describe('Get chunk indexes', () => { + [ + [2, 240, 500, [[2, 240]]], + [ + 2, + 2000, + 500, + [ + [2, 502], + [503, 1003], + [1004, 1504], + [1505, 2000], + ], + ], + [2, 1, 500, []], + ].forEach(([start, end, chunkSize, expected]) => { + it(`${start.toString()}:${end.toString()}:${chunkSize.toString()}`, () => { + expect( + getChunkIndexes(start as number, end as number, chunkSize as number) + ).toStrictEqual(expected); + }); + }); +}); From 26b12ae17127b1ffaca78b207e4d24819090f7d3 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 20:10:08 +0100 Subject: [PATCH 03/20] Another test for get chunk indexes --- .../shared-state/async-apply-external-event-sources.ts | 10 +++++----- tests/get-chunk-indexes.test.ts | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 9d0704ae..4c42f9da 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -191,11 +191,11 @@ export const asyncApplyExternalEventSources = ( googleHelpers.value, equipment, updateState - )(), - logger.info( - 'Finished pulling events from google training sheet for %s', - equipment.name - ); + ); + logger.info( + 'Finished pulling events from google training sheet for %s', + equipment.name + ); } } logger.info('Finished applying external event sources'); diff --git a/tests/get-chunk-indexes.test.ts b/tests/get-chunk-indexes.test.ts index 7227ecd0..1af51139 100644 --- a/tests/get-chunk-indexes.test.ts +++ b/tests/get-chunk-indexes.test.ts @@ -15,6 +15,7 @@ describe('Get chunk indexes', () => { ], ], [2, 1, 500, []], + [1000, 2, 500, []], ].forEach(([start, end, chunkSize, expected]) => { it(`${start.toString()}:${end.toString()}:${chunkSize.toString()}`, () => { expect( From 193e1a049c8a5ed2cd2e4504f7e44491f801a144 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 20:13:21 +0100 Subject: [PATCH 04/20] Log google sheet get request --- .../google/pull_sheet_data.ts | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 1afb24a5..1930a872 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -5,6 +5,7 @@ import {Failure} from '../../types'; import {pipe} from 'fp-ts/lib/function'; import {sheets, sheets_v4} from '@googleapis/sheets'; import {GoogleAuth} from 'google-auth-library'; +import {columnIndexToLetter} from '../../training-sheets/extract-metadata'; export type GoogleSpreadsheetInitialMetadata = sheets_v4.Schema$Spreadsheet & { readonly GoogleSpreadsheetInitialMetadata: unique symbol; @@ -46,19 +47,32 @@ export const pullGoogleSheetData = trainingSheetId: string, sheetName: string, rowStart: number, // 1 indexed. - rowEnd: number + rowEnd: number, + columnStartIndex: number, + columnEndIndex: number ): TE.TaskEither => pipe( TE.tryCatch( - () => - sheets({ + () => { + const ranges = [ + `${sheetName}!${columnIndexToLetter(columnStartIndex)}${rowStart}:${columnIndexToLetter(columnEndIndex)}${rowEnd}`, + ]; + const fields = 'sheets(data(rowData(values(formattedValue))))'; + logger.info( + 'Querying sheet %s for fields %s range %s', + trainingSheetId, + fields, + ranges + ); + return sheets({ version: 'v4', auth, }).spreadsheets.get({ spreadsheetId: trainingSheetId, - fields: 'sheets(data(rowData(values(formattedValue))))', - ranges: [`${sheetName}!${rowStart}:${rowEnd}`], - }), + fields, + ranges, + }); + }, reason => { logger.error(reason, 'Failed to get spreadsheet'); return { From edaa97cae9bc9fb2a47b08540f63e36f0059eb53 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 20:14:18 +0100 Subject: [PATCH 05/20] Columns are 0-indexed --- src/init-dependencies/google/pull_sheet_data.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 1930a872..3c402b4e 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -48,7 +48,7 @@ export const pullGoogleSheetData = sheetName: string, rowStart: number, // 1 indexed. rowEnd: number, - columnStartIndex: number, + columnStartIndex: number, // 0 indexed, converted to a letter. columnEndIndex: number ): TE.TaskEither => pipe( From b9e6a40fbaa71d3eb66338964b578af302867592 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 20:18:04 +0100 Subject: [PATCH 06/20] Pull back only a subset of the columns --- .../shared-state/async-apply-external-event-sources.ts | 9 +++++++-- src/training-sheets/extract-metadata.ts | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 4c42f9da..bb527757 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -17,6 +17,7 @@ import { extractGoogleSheetMetadata, extractInitialGoogleSheetMetadata, GoogleSheetMetadata, + MAX_COLUMN_INDEX, } from '../../training-sheets/extract-metadata'; import {getChunkIndexes} from '../../util'; @@ -47,7 +48,9 @@ export const pullNewEquipmentQuizResultsForSheet = async ( trainingSheetId, sheet.name, rowStart, - rowEnd + rowEnd, + Math.min(...Object.values(sheet.requiredColumns)), + Math.max(...Object.values(sheet.requiredColumns)) )(); if (E.isLeft(data)) { logger.debug( @@ -124,7 +127,9 @@ export const pullNewEquipmentQuizResults = async ( trainingSheetId, sheet.name, 1, - 1 + 1, + 0, + MAX_COLUMN_INDEX )(); if (E.isLeft(firstRowData)) { logger.warn( diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index 8c0200e7..71e54f67 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -40,6 +40,7 @@ export interface GoogleSheetsMetadata { sheets: GoogleSheetMetadata[]; } +export const MAX_COLUMN_INDEX = 25; // Doesn't support beyond 26 columns but actually thats fine for the current data. export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter => 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index); From 6b30e4d46d818f3e4abaa46d7370d47d3fc79bf0 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 21:27:41 +0100 Subject: [PATCH 07/20] Update extract from row to new method --- .../google/pull_sheet_data.ts | 1 + src/training-sheets/extract-metadata.ts | 49 ++++--- src/training-sheets/google.ts | 138 ++++++------------ 3 files changed, 73 insertions(+), 115 deletions(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 3c402b4e..51b0bea1 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -11,6 +11,7 @@ export type GoogleSpreadsheetInitialMetadata = sheets_v4.Schema$Spreadsheet & { readonly GoogleSpreadsheetInitialMetadata: unique symbol; }; +// Contains only a single sheet export type GoogleSpreadsheetDataForSheet = sheets_v4.Schema$Spreadsheet & { readonly GoogleSpreadsheetDataForSheet: unique symbol; }; diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index 71e54f67..deffd699 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -29,15 +29,17 @@ type ColumnIndex = number; // 0-indexed. export interface GoogleSheetMetadata { name: GoogleSheetName; rowCount: number; - requiredColumns: { + mappedColumns: { + // Timestamp and score are required for every sheet, some other sheets only have email or member number. timestamp: ColumnIndex; - email: ColumnIndex; score: ColumnIndex; - memberNumber: ColumnIndex; + email: O.Option; + memberNumber: O.Option; }; } export interface GoogleSheetsMetadata { sheets: GoogleSheetMetadata[]; + timezone: string; } export const MAX_COLUMN_INDEX = 25; @@ -73,7 +75,7 @@ export const extractInitialGoogleSheetMetadata = ( })) ); -const SpreadsheetData = t.strict({ +export const SpreadsheetData = t.strict({ sheets: tt.nonEmptyArray( t.strict({ data: tt.nonEmptyArray( @@ -120,16 +122,6 @@ export const extractGoogleSheetMetadata = ); return O.none; } - const email = RA.findIndex(val => - EMAIL_COLUMN_NAMES.includes(val.toLowerCase()) - )(columnNames); - if (O.isNone(email)) { - logger.warn( - 'Failed to find email column, skipping sheet: %s', - initialMeta.name - ); - return O.none; - } const score = RA.findIndex(val => val.toLowerCase() === 'score')( columnNames ); @@ -143,21 +135,30 @@ export const extractGoogleSheetMetadata = const memberNumber = RA.findIndex( val => val.toLowerCase() === 'membership number' )(columnNames); - if (O.isNone(memberNumber)) { - logger.warn( - 'Failed to find member number column, skipping sheet: %s', - initialMeta.name - ); - return O.none; - } + const email = RA.findIndex(val => + EMAIL_COLUMN_NAMES.includes(val.toLowerCase()) + )(columnNames); return O.some({ ...initialMeta, - requiredColumns: { + mappedColumns: { timestamp: timestamp.value, - email: email.value, score: score.value, - memberNumber: memberNumber.value, + email: email, + memberNumber: memberNumber, }, }); }; + +const getTimezone = (metadata: GoogleSheetMetadata): string => { + let timezone = spreadsheet.properties?.timeZone; + if (!timezone || !DateTime.local().setZone(timezone).isValid) { + // Not all the google form sheets are actually in Europe/London. + // Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). + logger.info( + `Unable to determine timezone for google sheet '${spreadsheet.properties?.title}', '${timezone}' - defaulting to Europe/London` + ); + timezone = 'Europe/London'; + } + return timezone; +} diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index 4ac0503f..7bc4ee27 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -1,15 +1,20 @@ import {pipe} from 'fp-ts/lib/function'; import * as RA from 'fp-ts/ReadonlyArray'; import * as O from 'fp-ts/Option'; +import * as E from 'fp-ts/Either'; import {Logger} from 'pino'; import {constructEvent, EventOfType} from '../types/domain-event'; import {v4} from 'uuid'; import {UUID} from 'io-ts-types'; import {DateTime} from 'luxon'; -import {sheets_v4} from '@googleapis/sheets'; import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-types'; -import {GoogleSheetMetadataInital} from './extract-metadata'; +import { + GoogleSheetMetadata, + GoogleSheetMetadataInital, + SpreadsheetData, +} from './extract-metadata'; +import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data'; // Bounds to prevent clearly broken parsing. const MIN_RECOGNISED_MEMBER_NUMBER = 0; @@ -125,56 +130,48 @@ const extractTimestamp = ( const extractFromRow = ( logger: Logger, - sheetInfo: SheetInfo, + metadata: GoogleSheetMetadata, equipmentId: UUID, trainingSheetId: string, timezone: string ) => - ( - row: sheets_v4.Schema$RowData - ): O.Option> => { - if (!row.values) { - return O.none; - } - - const email = O.isSome(sheetInfo.columnIndexes.email) + (row: { + values: {formattedValue: string}[]; + }): O.Option> => { + const email = O.isSome(metadata.mappedColumns.email) ? extractEmail( - row.values[sheetInfo.columnIndexes.email.value].formattedValue + row.values[metadata.mappedColumns.email.value].formattedValue ) : O.none; - const memberNumber = O.isSome(sheetInfo.columnIndexes.memberNumber) + const memberNumber = O.isSome(metadata.mappedColumns.memberNumber) ? extractMemberNumber( - row.values[sheetInfo.columnIndexes.memberNumber.value].formattedValue - ) - : O.none; - const score = O.isSome(sheetInfo.columnIndexes.score) - ? extractScore( - row.values[sheetInfo.columnIndexes.score.value].formattedValue - ) - : O.none; - const timestampEpochMS = O.isSome(sheetInfo.columnIndexes.timestamp) - ? extractTimestamp( - timezone, - row.values[sheetInfo.columnIndexes.timestamp.value].formattedValue + row.values[metadata.mappedColumns.memberNumber.value].formattedValue ) : O.none; + const score = extractScore( + row.values[metadata.mappedColumns.score].formattedValue + ); + const timestampEpochMS = extractTimestamp( + timezone, + row.values[metadata.mappedColumns.timestamp].formattedValue + ); if (O.isNone(email) && O.isNone(memberNumber)) { // Note that some quizes only require the member number. logger.warn( 'Failed to extract email or member number from row, skipping quiz result' ); - logger.trace('Skipped quiz row: %O', row.values); + logger.trace('Skipped quiz row: %O', row); return O.none; } if (O.isNone(score)) { logger.warn('Failed to extract score from row, skipped row'); - logger.trace('Skipped quiz row: %o', row.values); + logger.trace('Skipped quiz row: %o', row); return O.none; } if (O.isNone(timestampEpochMS)) { logger.warn('Failed to extract timestamp from row, skipped row'); - logger.trace('Skipped quiz row: %o', row.values); + logger.trace('Skipped quiz row: %o', row); return O.none; } return O.some( @@ -197,72 +194,31 @@ export const extractGoogleSheetData = logger: Logger, trainingSheetId: string, equipmentId: UUID, + metadata: GoogleSheetMetadata, + timezone: string, eventsFromExclusive: O.Option ) => ( - spreadsheet: sheets_v4.Schema$Spreadsheet - ): ReadonlyArray> => - !spreadsheet.sheets || spreadsheet.sheets.length < 1 - ? [] - : spreadsheet.sheets.map(sheet => { - const title = sheet.properties?.title; - if (!title) { - logger.warn('Skipping sheet due to missing title'); - return []; - } - - if ( - !sheet.data || - sheet.data.length < 1 || - !sheet.data[0].rowData || - sheet.data[0].rowData.length < 1 - ) { - logger.warn(`Skipping sheet '${title}' as missing data`); - return []; - } - - let timezone = spreadsheet.properties?.timeZone; - if (!timezone || !DateTime.local().setZone(timezone).isValid) { - // Not all the google form sheets are actually in Europe/London. - // Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). - logger.info( - `Unable to determine timezone for google sheet '${spreadsheet.properties?.title}', '${timezone}' - defaulting to Europe/London` - ); - timezone = 'Europe/London'; - } - - const [headers, ...data] = sheet.data[0].rowData; - - return pipe( - headers, - extractQuizSheetInformation(logger), - O.match( - () => { - logger.warn('Failed to extract sheet info'); - return []; - }, - sheetInfo => - pipe( - data, - RA.map( - extractFromRow( - logger, - sheetInfo, - equipmentId, - trainingSheetId, - timezone - ) - ), - RA.filterMap(e => e), - RA.filter( - e => - O.isNone(eventsFromExclusive) || - e.timestampEpochMS > eventsFromExclusive.value - ) - ) - ) - ); - }); + spreadsheet: GoogleSpreadsheetDataForSheet + ): ReadonlyArray> => { + const data = SpreadsheetData.decode(spreadsheet); + if (E.isLeft(data)) { + logger.warn('Skipping sheet %s due to missing data', trainingSheetId); + return []; + } + return pipe( + data.right.sheets[0].data[0].rowData, + RA.map( + extractFromRow(logger, metadata, equipmentId, trainingSheetId, timezone) + ), + RA.filterMap(e => e), + RA.filter( + e => + O.isNone(eventsFromExclusive) || + e.timestampEpochMS > eventsFromExclusive.value + ) + ); + }; export const shouldPullFromSheet = ( sheet: GoogleSheetMetadataInital From ff1f7e1febbe0d4fce650ce87df72f1763fed7af Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 21:53:32 +0100 Subject: [PATCH 08/20] Updated event generating --- src/configuration.ts | 10 +--- .../async-apply-external-event-sources.ts | 17 +++++-- src/training-sheets/extract-metadata.ts | 25 ++++++---- src/training-sheets/google.ts | 9 ++++ src/util.ts | 16 ++++++ tests/col-bounds-required.test.ts | 50 +++++++++++++++++++ 6 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 tests/col-bounds-required.test.ts diff --git a/src/configuration.ts b/src/configuration.ts index a1e2bed9..548c72d8 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -3,15 +3,7 @@ import * as tt from 'io-ts-types'; import * as E from 'fp-ts/Either'; import {pipe} from 'fp-ts/lib/function'; import {formatValidationErrors} from 'io-ts-reporters'; - -const withDefaultIfEmpty = (codec: C, ifEmpty: t.TypeOf) => - tt.withValidate(codec, (input, context) => - pipe( - tt.NonEmptyString.validate(input, context), - E.orElse(() => t.success(String(ifEmpty))), - E.chain(nonEmptyString => codec.validate(nonEmptyString, context)) - ) - ); +import {withDefaultIfEmpty} from './util'; const LogLevel = t.keyof({ trace: null, diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index bb527757..75eb2dc0 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -8,6 +8,7 @@ import {getAllEquipment} from './get-equipment'; import {pipe} from 'fp-ts/lib/function'; import {EpochTimestampMilliseconds, Equipment} from './return-types'; import { + columnBoundsRequired, extractGoogleSheetData, shouldPullFromSheet, } from '../../training-sheets/google'; @@ -29,6 +30,7 @@ export const pullNewEquipmentQuizResultsForSheet = async ( equipment: Equipment, trainingSheetId: string, sheet: GoogleSheetMetadata, + timezone: string, updateState: (event: DomainEvent) => void ) => { logger.info('Processing sheet %s', sheet.name); @@ -43,14 +45,17 @@ export const pullNewEquipmentQuizResultsForSheet = async ( rowStart, rowEnd ); + + const [minCol, maxCol] = columnBoundsRequired(sheet); + const data = await googleHelpers.pullGoogleSheetData( logger, trainingSheetId, sheet.name, rowStart, rowEnd, - Math.min(...Object.values(sheet.requiredColumns)), - Math.max(...Object.values(sheet.requiredColumns)) + minCol, + maxCol )(); if (E.isLeft(data)) { logger.debug( @@ -64,6 +69,8 @@ export const pullNewEquipmentQuizResultsForSheet = async ( logger, trainingSheetId, equipment.id, + sheet, + timezone, equipment.lastQuizResult ), RA.map(updateState) @@ -102,7 +109,10 @@ export const pullNewEquipmentQuizResults = async ( return; } - const initialMeta = extractInitialGoogleSheetMetadata(initialRaw.right); + const initialMeta = extractInitialGoogleSheetMetadata( + logger, + initialRaw.right + ); if (E.isLeft(initialMeta)) { logger.warn( @@ -155,6 +165,7 @@ export const pullNewEquipmentQuizResults = async ( equipment, trainingSheetId, sheet, + initialMeta.right.timezone, updateState ); } diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index deffd699..90733c71 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -10,6 +10,8 @@ import { GoogleSpreadsheetDataForSheet, GoogleSpreadsheetInitialMetadata, } from '../init-dependencies/google/pull_sheet_data'; +import {withDefaultIfEmpty} from '../util'; +import {DateTime} from 'luxon'; const EMAIL_COLUMN_NAMES = ['email address', 'email']; @@ -21,6 +23,7 @@ export interface GoogleSheetMetadataInital { } export interface GoogleSheetsMetadataInital { sheets: GoogleSheetMetadataInital[]; + timezone: string; } type ColumnLetter = string; @@ -37,17 +40,18 @@ export interface GoogleSheetMetadata { memberNumber: O.Option; }; } -export interface GoogleSheetsMetadata { - sheets: GoogleSheetMetadata[]; - timezone: string; -} export const MAX_COLUMN_INDEX = 25; // Doesn't support beyond 26 columns but actually thats fine for the current data. export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter => 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index); +const DEFAULT_TIMEZONE = 'Europe/London'; + const SheetProperties = t.strict({ + properties: t.strict({ + timeZone: withDefaultIfEmpty(t.string, DEFAULT_TIMEZONE), + }), sheets: t.array( t.strict({ properties: t.strict({ @@ -61,6 +65,7 @@ const SheetProperties = t.strict({ }); export const extractInitialGoogleSheetMetadata = ( + logger: Logger, spreadsheet: GoogleSpreadsheetInitialMetadata ): E.Either => pipe( @@ -72,6 +77,7 @@ export const extractInitialGoogleSheetMetadata = ( name: sheet.properties.title, rowCount: sheet.properties.gridProperties.rowCount, })), + timezone: validateTimezone(logger, properties.properties.timeZone), })) ); @@ -150,15 +156,14 @@ export const extractGoogleSheetMetadata = }); }; -const getTimezone = (metadata: GoogleSheetMetadata): string => { - let timezone = spreadsheet.properties?.timeZone; - if (!timezone || !DateTime.local().setZone(timezone).isValid) { +const validateTimezone = (logger: Logger, timezone: string): string => { + if (!DateTime.local().setZone(timezone).isValid) { // Not all the google form sheets are actually in Europe/London. // Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). logger.info( - `Unable to determine timezone for google sheet '${spreadsheet.properties?.title}', '${timezone}' - defaulting to Europe/London` + `Unable to determine timezone for google sheet, timezone is invalid: '${timezone}' - defaulting to Europe/London` ); - timezone = 'Europe/London'; + timezone = DEFAULT_TIMEZONE; } return timezone; -} +}; diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index 7bc4ee27..103a0367 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -223,3 +223,12 @@ export const extractGoogleSheetData = export const shouldPullFromSheet = ( sheet: GoogleSheetMetadataInital ): boolean => FORM_RESPONSES_SHEET_REGEX.test(sheet.name); + +export const columnBoundsRequired = ( + sheet: GoogleSheetMetadata +): [number, number] => { + const colIndexes = Object.values(sheet.mappedColumns) + .filter(col => typeof col === 'number' || O.isSome(col)) + .map(col => (typeof col === 'number' ? col : col.value)); + return [Math.min(...colIndexes), Math.max(...colIndexes)]; +}; diff --git a/src/util.ts b/src/util.ts index ee5b2f3d..e06727b1 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,4 +1,8 @@ import {Logger} from 'pino'; +import * as t from 'io-ts'; +import * as tt from 'io-ts-types'; +import * as E from 'fp-ts/Either'; +import {pipe} from 'fp-ts/lib/function'; export const logPassThru = (logger: Logger, msg: string) => @@ -22,3 +26,15 @@ export const getChunkIndexes = ( } return result; }; + +export const withDefaultIfEmpty = ( + codec: C, + ifEmpty: t.TypeOf +) => + tt.withValidate(codec, (input, context) => + pipe( + tt.NonEmptyString.validate(input, context), + E.orElse(() => t.success(String(ifEmpty))), + E.chain(nonEmptyString => codec.validate(nonEmptyString, context)) + ) + ); diff --git a/tests/col-bounds-required.test.ts b/tests/col-bounds-required.test.ts new file mode 100644 index 00000000..d0c08a70 --- /dev/null +++ b/tests/col-bounds-required.test.ts @@ -0,0 +1,50 @@ +import * as O from 'fp-ts/Option'; +import {columnBoundsRequired} from '../src/training-sheets/google'; + +describe('columnBoundsRequired', () => { + [ + { + input: { + name: 'All populated', + rowCount: 0, + mappedColumns: { + timestamp: 1, + score: 0, + email: O.some(3), + memberNumber: O.some(2), + }, + }, + expected: [0, 3], + }, + { + input: { + name: 'Minimal required', + rowCount: 0, + mappedColumns: { + timestamp: 1, + score: 0, + email: O.none, + memberNumber: O.none, + }, + }, + expected: [0, 1], + }, + { + input: { + name: '1 populated', + rowCount: 0, + mappedColumns: { + timestamp: 1, + score: 4, + email: O.none, + memberNumber: O.some(0), + }, + }, + expected: [0, 4], + }, + ].forEach(({input, expected}) => { + it(`${input.name}`, () => { + expect(columnBoundsRequired(input)).toStrictEqual(expected); + }); + }); +}); From 0d4d3fccd4140044180e2da0ace17ac39aea6618 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 21:59:40 +0100 Subject: [PATCH 09/20] Temporarily disable tests to check new idea has legs --- tests/init-dependencies/pull-local-google.ts | 15 +- .../async-apply-external.test.ts | 410 +++++++-------- tests/training-sheets/process-events.test.ts | 466 +++++++++--------- 3 files changed, 446 insertions(+), 445 deletions(-) diff --git a/tests/init-dependencies/pull-local-google.ts b/tests/init-dependencies/pull-local-google.ts index db7d8bae..3049dea5 100644 --- a/tests/init-dependencies/pull-local-google.ts +++ b/tests/init-dependencies/pull-local-google.ts @@ -4,13 +4,14 @@ import * as gsheetData from '../data/google_sheet_data'; import {GoogleHelpers} from '../../src/init-dependencies/google/pull_sheet_data'; const localPullGoogleSheetData = (logger: Logger, trainingSheetId: string) => { - logger.debug(`Pulling local google sheet '${trainingSheetId}'`); - const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].data; - return sheet - ? TE.right(sheet) - : TE.left({ - message: 'Sheet not found', - }); + return '' as any; +// logger.debug(`Pulling local google sheet '${trainingSheetId}'`); +// const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].data; +// return sheet +// ? TE.right(sheet) +// : TE.left({ +// message: 'Sheet not found', +// }); }; export const localGoogleHelpers: GoogleHelpers = { diff --git a/tests/training-sheets/async-apply-external.test.ts b/tests/training-sheets/async-apply-external.test.ts index edfd0584..09784d12 100644 --- a/tests/training-sheets/async-apply-external.test.ts +++ b/tests/training-sheets/async-apply-external.test.ts @@ -1,216 +1,216 @@ -import * as O from 'fp-ts/Option'; -import {NonEmptyString, UUID} from 'io-ts-types'; -import {faker} from '@faker-js/faker'; -import * as gsheetData from '../data/google_sheet_data'; -import {initTestFramework, TestFramework} from '../read-models/test-framework'; -import {EmailAddress} from '../../src/types'; -import {getSomeOrFail} from '../helpers'; -import { - EpochTimestampMilliseconds, - Equipment, -} from '../../src/read-models/shared-state/return-types'; +// import * as O from 'fp-ts/Option'; +// import {NonEmptyString, UUID} from 'io-ts-types'; +// import {faker} from '@faker-js/faker'; +// import * as gsheetData from '../data/google_sheet_data'; +// import {initTestFramework, TestFramework} from '../read-models/test-framework'; +// import {EmailAddress} from '../../src/types'; +// import {getSomeOrFail} from '../helpers'; +// import { +// EpochTimestampMilliseconds, +// Equipment, +// } from '../../src/read-models/shared-state/return-types'; -describe('Integration asyncApplyExternalEventSources', () => { - const addArea = async (framework: TestFramework) => { - const createArea = { - id: faker.string.uuid() as UUID, - name: faker.company.buzzNoun() as NonEmptyString, - }; - await framework.commands.area.create(createArea); - return createArea.id; - }; +// describe('Integration asyncApplyExternalEventSources', () => { +// const addArea = async (framework: TestFramework) => { +// const createArea = { +// id: faker.string.uuid() as UUID, +// name: faker.company.buzzNoun() as NonEmptyString, +// }; +// await framework.commands.area.create(createArea); +// return createArea.id; +// }; - const addWithSheet = async ( - framework: TestFramework, - name: string, - areaId: UUID, - trainingSheetId: O.Option - ) => { - const equipment = { - id: faker.string.uuid() as UUID, - name: name as NonEmptyString, - areaId, - }; - await framework.commands.equipment.add(equipment); - if (O.isSome(trainingSheetId)) { - await framework.commands.equipment.trainingSheet({ - equipmentId: equipment.id, - trainingSheetId: trainingSheetId.value, - }); - } - return { - ...equipment, - trainingSheetId, - }; - }; +// const addWithSheet = async ( +// framework: TestFramework, +// name: string, +// areaId: UUID, +// trainingSheetId: O.Option +// ) => { +// const equipment = { +// id: faker.string.uuid() as UUID, +// name: name as NonEmptyString, +// areaId, +// }; +// await framework.commands.equipment.add(equipment); +// if (O.isSome(trainingSheetId)) { +// await framework.commands.equipment.trainingSheet({ +// equipmentId: equipment.id, +// trainingSheetId: trainingSheetId.value, +// }); +// } +// return { +// ...equipment, +// trainingSheetId, +// }; +// }; - it('Handle multiple equipment both populated', async () => { - const framework = await initTestFramework(1000); +// it('Handle multiple equipment both populated', async () => { +// const framework = await initTestFramework(1000); - // Create the users which the results are registered too. - await framework.commands.memberNumbers.linkNumberToEmail({ - memberNumber: gsheetData.BAMBU.entries[0].memberNumberProvided, - email: gsheetData.BAMBU.entries[0].emailProvided as EmailAddress, - }); - await framework.commands.memberNumbers.linkNumberToEmail({ - memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, - email: gsheetData.METAL_LATHE.entries[0].emailProvided as EmailAddress, - }); - const areaId = await addArea(framework); - const bambu = await addWithSheet( - framework, - 'bambu', - areaId, - O.some(gsheetData.BAMBU.data.spreadsheetId!) - ); - const lathe = await addWithSheet( - framework, - 'Metal Lathe', - areaId, - O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) - ); - const results = await runAsyncApplyExternalEventSources(framework); - checkLastQuizSyncUpdated(results); - checkLastQuizEventTimestamp( - gsheetData.BAMBU, - results.equipmentAfter.get(bambu.id)! - ); - checkLastQuizEventTimestamp( - gsheetData.METAL_LATHE, - results.equipmentAfter.get(lathe.id)! - ); +// // Create the users which the results are registered too. +// await framework.commands.memberNumbers.linkNumberToEmail({ +// memberNumber: gsheetData.BAMBU.entries[0].memberNumberProvided, +// email: gsheetData.BAMBU.entries[0].emailProvided as EmailAddress, +// }); +// await framework.commands.memberNumbers.linkNumberToEmail({ +// memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, +// email: gsheetData.METAL_LATHE.entries[0].emailProvided as EmailAddress, +// }); +// const areaId = await addArea(framework); +// const bambu = await addWithSheet( +// framework, +// 'bambu', +// areaId, +// O.some(gsheetData.BAMBU.data.spreadsheetId!) +// ); +// const lathe = await addWithSheet( +// framework, +// 'Metal Lathe', +// areaId, +// O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) +// ); +// const results = await runAsyncApplyExternalEventSources(framework); +// checkLastQuizSyncUpdated(results); +// checkLastQuizEventTimestamp( +// gsheetData.BAMBU, +// results.equipmentAfter.get(bambu.id)! +// ); +// checkLastQuizEventTimestamp( +// gsheetData.METAL_LATHE, +// results.equipmentAfter.get(lathe.id)! +// ); - // We already test the produced quiz result events above - // and testing updateState is also tested elsewhere so this integration - // test doesn't need to enumerate every combination it just needs to check - // that generally the equipment is getting updated. - const bambuAfter = results.equipmentAfter.get(bambu.id)!; - expect(bambuAfter.orphanedPassedQuizes).toHaveLength(0); - expect(bambuAfter.membersAwaitingTraining).toHaveLength(1); - expect(bambuAfter.membersAwaitingTraining[0].memberNumber).toStrictEqual( - gsheetData.BAMBU.entries[0].memberNumberProvided - ); - expect(bambuAfter.membersAwaitingTraining[0].emailAddress).toStrictEqual( - gsheetData.BAMBU.entries[0].emailProvided - ); - expect(bambuAfter.membersAwaitingTraining[0].waitingSince).toStrictEqual( - new Date(gsheetData.getLatestEvent(gsheetData.BAMBU).timestampEpochMS) - ); +// // We already test the produced quiz result events above +// // and testing updateState is also tested elsewhere so this integration +// // test doesn't need to enumerate every combination it just needs to check +// // that generally the equipment is getting updated. +// const bambuAfter = results.equipmentAfter.get(bambu.id)!; +// expect(bambuAfter.orphanedPassedQuizes).toHaveLength(0); +// expect(bambuAfter.membersAwaitingTraining).toHaveLength(1); +// expect(bambuAfter.membersAwaitingTraining[0].memberNumber).toStrictEqual( +// gsheetData.BAMBU.entries[0].memberNumberProvided +// ); +// expect(bambuAfter.membersAwaitingTraining[0].emailAddress).toStrictEqual( +// gsheetData.BAMBU.entries[0].emailProvided +// ); +// expect(bambuAfter.membersAwaitingTraining[0].waitingSince).toStrictEqual( +// new Date(gsheetData.getLatestEvent(gsheetData.BAMBU).timestampEpochMS) +// ); - // Lathe results only have a single failed entry. - const latheAfter = results.equipmentAfter.get(lathe.id)!; - expect(latheAfter.orphanedPassedQuizes).toHaveLength(0); - expect(latheAfter.failedQuizAttempts).toHaveLength(1); - expect(latheAfter.failedQuizAttempts[0]).toMatchObject({ - emailAddress: gsheetData.METAL_LATHE.entries[0] - .emailProvided as EmailAddress, - memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, - score: gsheetData.METAL_LATHE.entries[0].score, - maxScore: gsheetData.METAL_LATHE.entries[0].maxScore, - percentage: gsheetData.METAL_LATHE.entries[0].percentage, - timestamp: new Date(gsheetData.METAL_LATHE.entries[0].timestampEpochMS), - }); - }); - // it('Handle no equipment', async () => { - // const framework = await initTestFramework(1000); - // const results = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results); - // expect(results.equipmentAfter.size).toStrictEqual(0); - // }); - // it('Handle equipment with no training sheet', async () => { - // const framework = await initTestFramework(1000); - // const areaId = await addArea(framework); - // const bambu = await addWithSheet(framework, 'bambu', areaId, O.none); - // const results = await runAsyncApplyExternalEventSources(framework); - // expect( - // results.equipmentAfter.get(bambu.id)!.lastQuizSync // No training sheet so not updated. - // ).toStrictEqual(O.none); - // expect( - // results.equipmentAfter.get(bambu.id)!.lastQuizResult - // ).toStrictEqual(O.none); - // expect(results.newEvents).toHaveLength(0); - // }); - // it('Rate limit equipment pull', async () => { - // const framework = await initTestFramework(1000); - // const areaId = await addArea(framework); - // const bambu = await addWithSheet( - // framework, - // 'bambu', - // areaId, - // O.some(gsheetData.BAMBU.data.spreadsheetId!) - // ); - // const results1 = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results1); - // const results2 = await runAsyncApplyExternalEventSources(framework); - // expect( - // results1.equipmentAfter.get(bambu.id)!.lastQuizSync - // ).toStrictEqual(results2.equipmentAfter.get(bambu.id)!.lastQuizSync); - // expect(results1.newEvents.length).toBeGreaterThan(0); - // expect(results2.newEvents).toHaveLength(0); - // }); - // it('Repeat equipment pull no rate limit', async () => { - // const rateLimitMs = 100; - // const framework = await initTestFramework(rateLimitMs); - // const areaId = await addArea(framework); - // const bambu = await addWithSheet( - // framework, - // 'bambu', - // areaId, - // O.some(gsheetData.BAMBU.data.spreadsheetId!) - // ); - // const results1 = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results1); +// // Lathe results only have a single failed entry. +// const latheAfter = results.equipmentAfter.get(lathe.id)!; +// expect(latheAfter.orphanedPassedQuizes).toHaveLength(0); +// expect(latheAfter.failedQuizAttempts).toHaveLength(1); +// expect(latheAfter.failedQuizAttempts[0]).toMatchObject({ +// emailAddress: gsheetData.METAL_LATHE.entries[0] +// .emailProvided as EmailAddress, +// memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, +// score: gsheetData.METAL_LATHE.entries[0].score, +// maxScore: gsheetData.METAL_LATHE.entries[0].maxScore, +// percentage: gsheetData.METAL_LATHE.entries[0].percentage, +// timestamp: new Date(gsheetData.METAL_LATHE.entries[0].timestampEpochMS), +// }); +// }); +// // it('Handle no equipment', async () => { +// // const framework = await initTestFramework(1000); +// // const results = await runAsyncApplyExternalEventSources(framework); +// // checkLastQuizSync(results); +// // expect(results.equipmentAfter.size).toStrictEqual(0); +// // }); +// // it('Handle equipment with no training sheet', async () => { +// // const framework = await initTestFramework(1000); +// // const areaId = await addArea(framework); +// // const bambu = await addWithSheet(framework, 'bambu', areaId, O.none); +// // const results = await runAsyncApplyExternalEventSources(framework); +// // expect( +// // results.equipmentAfter.get(bambu.id)!.lastQuizSync // No training sheet so not updated. +// // ).toStrictEqual(O.none); +// // expect( +// // results.equipmentAfter.get(bambu.id)!.lastQuizResult +// // ).toStrictEqual(O.none); +// // expect(results.newEvents).toHaveLength(0); +// // }); +// // it('Rate limit equipment pull', async () => { +// // const framework = await initTestFramework(1000); +// // const areaId = await addArea(framework); +// // const bambu = await addWithSheet( +// // framework, +// // 'bambu', +// // areaId, +// // O.some(gsheetData.BAMBU.data.spreadsheetId!) +// // ); +// // const results1 = await runAsyncApplyExternalEventSources(framework); +// // checkLastQuizSync(results1); +// // const results2 = await runAsyncApplyExternalEventSources(framework); +// // expect( +// // results1.equipmentAfter.get(bambu.id)!.lastQuizSync +// // ).toStrictEqual(results2.equipmentAfter.get(bambu.id)!.lastQuizSync); +// // expect(results1.newEvents.length).toBeGreaterThan(0); +// // expect(results2.newEvents).toHaveLength(0); +// // }); +// // it('Repeat equipment pull no rate limit', async () => { +// // const rateLimitMs = 100; +// // const framework = await initTestFramework(rateLimitMs); +// // const areaId = await addArea(framework); +// // const bambu = await addWithSheet( +// // framework, +// // 'bambu', +// // areaId, +// // O.some(gsheetData.BAMBU.data.spreadsheetId!) +// // ); +// // const results1 = await runAsyncApplyExternalEventSources(framework); +// // checkLastQuizSync(results1); - // await new Promise(res => setTimeout(res, rateLimitMs)); - // const results2 = await runAsyncApplyExternalEventSources(framework); - // checkLastQuizSync(results2); - // expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( - // results2.equipmentAfter.get(bambu.id)!.lastQuizSync - // ); - // expect(results1.newEvents.length).toBeGreaterThan(0); - // expect(results2.newEvents).toHaveLength(0); - // }); -}); +// // await new Promise(res => setTimeout(res, rateLimitMs)); +// // const results2 = await runAsyncApplyExternalEventSources(framework); +// // checkLastQuizSync(results2); +// // expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( +// // results2.equipmentAfter.get(bambu.id)!.lastQuizSync +// // ); +// // expect(results1.newEvents.length).toBeGreaterThan(0); +// // expect(results2.newEvents).toHaveLength(0); +// // }); +// }); -type ApplyExternalEventsResults = { - startTime: EpochTimestampMilliseconds; - endTime: EpochTimestampMilliseconds; - equipmentAfter: Map; -}; +// type ApplyExternalEventsResults = { +// startTime: EpochTimestampMilliseconds; +// endTime: EpochTimestampMilliseconds; +// equipmentAfter: Map; +// }; -const runAsyncApplyExternalEventSources = async ( - framework: TestFramework -): Promise => { - const startTime = Date.now() as EpochTimestampMilliseconds; - await framework.sharedReadModel.asyncApplyExternalEventSources()(); - const endTime = Date.now() as EpochTimestampMilliseconds; - const equipmentAfter = new Map( - framework.sharedReadModel.equipment.getAll().map(e => [e.id, e]) - ); - return { - startTime, - endTime, - equipmentAfter, - }; -}; +// const runAsyncApplyExternalEventSources = async ( +// framework: TestFramework +// ): Promise => { +// const startTime = Date.now() as EpochTimestampMilliseconds; +// await framework.sharedReadModel.asyncApplyExternalEventSources()(); +// const endTime = Date.now() as EpochTimestampMilliseconds; +// const equipmentAfter = new Map( +// framework.sharedReadModel.equipment.getAll().map(e => [e.id, e]) +// ); +// return { +// startTime, +// endTime, +// equipmentAfter, +// }; +// }; -const checkLastQuizSyncUpdated = (results: ApplyExternalEventsResults) => { - // Check that the last quiz sync property is updated to reflect - // that a quiz sync was preformed. - for (const equipment of results.equipmentAfter.values()) { - expect(getSomeOrFail(equipment.lastQuizSync)).toBeGreaterThanOrEqual( - results.startTime - ); - expect(getSomeOrFail(equipment.lastQuizSync)).toBeLessThanOrEqual( - results.endTime - ); - } -}; +// const checkLastQuizSyncUpdated = (results: ApplyExternalEventsResults) => { +// // Check that the last quiz sync property is updated to reflect +// // that a quiz sync was preformed. +// for (const equipment of results.equipmentAfter.values()) { +// expect(getSomeOrFail(equipment.lastQuizSync)).toBeGreaterThanOrEqual( +// results.startTime +// ); +// expect(getSomeOrFail(equipment.lastQuizSync)).toBeLessThanOrEqual( +// results.endTime +// ); +// } +// }; -const checkLastQuizEventTimestamp = ( - data: gsheetData.ManualParsed, - equipmentAfter: Equipment -) => - expect(getSomeOrFail(equipmentAfter.lastQuizResult)).toStrictEqual( - gsheetData.getLatestEvent(data).timestampEpochMS - ); +// const checkLastQuizEventTimestamp = ( +// data: gsheetData.ManualParsed, +// equipmentAfter: Equipment +// ) => +// expect(getSomeOrFail(equipmentAfter.lastQuizResult)).toStrictEqual( +// gsheetData.getLatestEvent(data).timestampEpochMS +// ); diff --git a/tests/training-sheets/process-events.test.ts b/tests/training-sheets/process-events.test.ts index f43aeb0f..ad4376f8 100644 --- a/tests/training-sheets/process-events.test.ts +++ b/tests/training-sheets/process-events.test.ts @@ -1,247 +1,247 @@ -import {UUID} from 'io-ts-types'; -import {EventOfType, isEventOfType} from '../../src/types/domain-event'; -import pino from 'pino'; -import * as RA from 'fp-ts/lib/ReadonlyArray'; -import * as N from 'fp-ts/number'; -import * as O from 'fp-ts/Option'; -import * as gsheetData from '../data/google_sheet_data'; -import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources'; -import { - EpochTimestampMilliseconds, - Equipment, -} from '../../src/read-models/shared-state/return-types'; -import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; +// import {UUID} from 'io-ts-types'; +// import {EventOfType, isEventOfType} from '../../src/types/domain-event'; +// import pino from 'pino'; +// import * as RA from 'fp-ts/lib/ReadonlyArray'; +// import * as N from 'fp-ts/number'; +// import * as O from 'fp-ts/Option'; +// import * as gsheetData from '../data/google_sheet_data'; +// import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources'; +// import { +// EpochTimestampMilliseconds, +// Equipment, +// } from '../../src/read-models/shared-state/return-types'; +// import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; -const sortQuizResults = RA.sort({ - compare: (a, b) => - N.Ord.compare( - (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, - (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS - ), - equals: (a, b) => - N.Ord.equals( - (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, - (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS - ), -}); +// const sortQuizResults = RA.sort({ +// compare: (a, b) => +// N.Ord.compare( +// (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, +// (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS +// ), +// equals: (a, b) => +// N.Ord.equals( +// (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, +// (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS +// ), +// }); -const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => - pullNewEquipmentQuizResults( - pino({ - level: 'fatal', - timestamp: pino.stdTimeFunctions.isoTime, - }), - localGoogleHelpers, - equipment - )(); +// const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => +// pullNewEquipmentQuizResults( +// pino({ +// level: 'fatal', +// timestamp: pino.stdTimeFunctions.isoTime, +// }), +// localGoogleHelpers, +// equipment +// )(); -const defaultEquipment = (): Equipment => ({ - id: 'ebedee32-49f4-4d36-a350-4fa7848792bf' as UUID, - name: 'Metal Lathe', - trainers: [], - trainedMembers: [], - area: { - id: 'f9cee7aa-75c6-42cc-8585-0e658044fe8e', - name: 'Metal Shop', - }, - membersAwaitingTraining: [], - orphanedPassedQuizes: [], - failedQuizAttempts: [], - trainingSheetId: O.none, - lastQuizResult: O.none, - lastQuizSync: O.none, -}); +// const defaultEquipment = (): Equipment => ({ +// id: 'ebedee32-49f4-4d36-a350-4fa7848792bf' as UUID, +// name: 'Metal Lathe', +// trainers: [], +// trainedMembers: [], +// area: { +// id: 'f9cee7aa-75c6-42cc-8585-0e658044fe8e', +// name: 'Metal Shop', +// }, +// membersAwaitingTraining: [], +// orphanedPassedQuizes: [], +// failedQuizAttempts: [], +// trainingSheetId: O.none, +// lastQuizResult: O.none, +// lastQuizSync: O.none, +// }); -type EquipmentQuizResultEvents = { - quizResults: ReadonlyArray>; - quizSync: ReadonlyArray>; - startTime: Date; - endTime: Date; -}; -const pullEquipmentQuizResultsWrapper = async ( - spreadsheetId: O.Option, - lastQuizResult: O.Option = O.none -): Promise => { - const equipment = defaultEquipment(); - equipment.trainingSheetId = spreadsheetId; - equipment.lastQuizResult = lastQuizResult; - const startTime = new Date(); - const events = await pullNewEquipmentQuizResultsLocal(equipment); - const endTime = new Date(); - const result = { - quizResults: [] as EventOfType<'EquipmentTrainingQuizResult'>[], - quizSync: [] as EventOfType<'EquipmentTrainingQuizSync'>[], - }; - for (const event of events) { - if (isEventOfType('EquipmentTrainingQuizResult')(event)) { - result.quizResults.push(event); - } else if (isEventOfType('EquipmentTrainingQuizSync')) { - result.quizSync.push(event); - } else { - throw new Error('Unexpected event type'); - } - } - return { - ...result, - startTime, - endTime, - }; -}; +// type EquipmentQuizResultEvents = { +// quizResults: ReadonlyArray>; +// quizSync: ReadonlyArray>; +// startTime: Date; +// endTime: Date; +// }; +// const pullEquipmentQuizResultsWrapper = async ( +// spreadsheetId: O.Option, +// lastQuizResult: O.Option = O.none +// ): Promise => { +// const equipment = defaultEquipment(); +// equipment.trainingSheetId = spreadsheetId; +// equipment.lastQuizResult = lastQuizResult; +// const startTime = new Date(); +// const events = await pullNewEquipmentQuizResultsLocal(equipment); +// const endTime = new Date(); +// const result = { +// quizResults: [] as EventOfType<'EquipmentTrainingQuizResult'>[], +// quizSync: [] as EventOfType<'EquipmentTrainingQuizSync'>[], +// }; +// for (const event of events) { +// if (isEventOfType('EquipmentTrainingQuizResult')(event)) { +// result.quizResults.push(event); +// } else if (isEventOfType('EquipmentTrainingQuizSync')) { +// result.quizSync.push(event); +// } else { +// throw new Error('Unexpected event type'); +// } +// } +// return { +// ...result, +// startTime, +// endTime, +// }; +// }; -const checkQuizSync = (results: EquipmentQuizResultEvents) => { - expect(results.quizSync).toHaveLength(1); - expect(results.quizSync[0].recordedAt.getTime()).toBeGreaterThanOrEqual( - results.startTime.getTime() - ); - expect(results.quizSync[0].recordedAt.getTime()).toBeLessThanOrEqual( - results.endTime.getTime() - ); -}; +// const checkQuizSync = (results: EquipmentQuizResultEvents) => { +// expect(results.quizSync).toHaveLength(1); +// expect(results.quizSync[0].recordedAt.getTime()).toBeGreaterThanOrEqual( +// results.startTime.getTime() +// ); +// expect(results.quizSync[0].recordedAt.getTime()).toBeLessThanOrEqual( +// results.endTime.getTime() +// ); +// }; -describe('Training sheets worker', () => { - describe('Process results', () => { - describe('Processes a registered training sheet', () => { - it('Equipment with no training sheet', async () => { - const result = await pullEquipmentQuizResultsWrapper(O.none); - expect(result.quizResults).toHaveLength(0); - expect(result.quizSync).toHaveLength(0); - }); +// describe('Training sheets worker', () => { +// describe('Process results', () => { +// describe('Processes a registered training sheet', () => { +// it('Equipment with no training sheet', async () => { +// const result = await pullEquipmentQuizResultsWrapper(O.none); +// expect(result.quizResults).toHaveLength(0); +// expect(result.quizSync).toHaveLength(0); +// }); - it('empty sheet produces no events, but does indicate a sync', async () => { - const result = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.EMPTY.data.spreadsheetId!) - ); - expect(result.quizResults).toHaveLength(0); - checkQuizSync(result); - }); - it('metal lathe training sheet', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) - ); - checkQuizSync(results); - expect(results.quizResults[0]).toMatchObject< - Partial> - >({ - type: 'EquipmentTrainingQuizResult', - equipmentId: defaultEquipment().id, - trainingSheetId: gsheetData.METAL_LATHE.data.spreadsheetId!, - ...gsheetData.METAL_LATHE.entries[0], - }); - }); - it('training sheet with a summary page', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.LASER_CUTTER.data.spreadsheetId!) - ); - checkQuizSync(results); - const expected: readonly Partial< - EventOfType<'EquipmentTrainingQuizResult'> - >[] = gsheetData.LASER_CUTTER.entries.map(e => ({ - type: 'EquipmentTrainingQuizResult', - equipmentId: defaultEquipment().id, - trainingSheetId: gsheetData.LASER_CUTTER.data.spreadsheetId!, - actor: { - tag: 'system', - }, - ...e, - })); - expect(results.quizResults).toHaveLength(expected.length); +// it('empty sheet produces no events, but does indicate a sync', async () => { +// const result = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.EMPTY.data.spreadsheetId!) +// ); +// expect(result.quizResults).toHaveLength(0); +// checkQuizSync(result); +// }); +// it('metal lathe training sheet', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) +// ); +// checkQuizSync(results); +// expect(results.quizResults[0]).toMatchObject< +// Partial> +// >({ +// type: 'EquipmentTrainingQuizResult', +// equipmentId: defaultEquipment().id, +// trainingSheetId: gsheetData.METAL_LATHE.data.spreadsheetId!, +// ...gsheetData.METAL_LATHE.entries[0], +// }); +// }); +// it('training sheet with a summary page', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.LASER_CUTTER.data.spreadsheetId!) +// ); +// checkQuizSync(results); +// const expected: readonly Partial< +// EventOfType<'EquipmentTrainingQuizResult'> +// >[] = gsheetData.LASER_CUTTER.entries.map(e => ({ +// type: 'EquipmentTrainingQuizResult', +// equipmentId: defaultEquipment().id, +// trainingSheetId: gsheetData.LASER_CUTTER.data.spreadsheetId!, +// actor: { +// tag: 'system', +// }, +// ...e, +// })); +// expect(results.quizResults).toHaveLength(expected.length); - for (const [actualEvent, expectedEvent] of RA.zip( - sortQuizResults(results.quizResults), - sortQuizResults(expected) - )) { - expect(actualEvent).toMatchObject< - Partial> - >(expectedEvent); - } - }); - it('training sheet with multiple response pages (different quiz questions)', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!) - ); - checkQuizSync(results); - const expected: readonly Partial< - EventOfType<'EquipmentTrainingQuizResult'> - >[] = gsheetData.BAMBU.entries.map(e => ({ - type: 'EquipmentTrainingQuizResult', - equipmentId: defaultEquipment().id, - trainingSheetId: gsheetData.BAMBU.data.spreadsheetId!, - actor: { - tag: 'system', - }, - ...e, - })); - expect(results.quizResults).toHaveLength(expected.length); +// for (const [actualEvent, expectedEvent] of RA.zip( +// sortQuizResults(results.quizResults), +// sortQuizResults(expected) +// )) { +// expect(actualEvent).toMatchObject< +// Partial> +// >(expectedEvent); +// } +// }); +// it('training sheet with multiple response pages (different quiz questions)', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!) +// ); +// checkQuizSync(results); +// const expected: readonly Partial< +// EventOfType<'EquipmentTrainingQuizResult'> +// >[] = gsheetData.BAMBU.entries.map(e => ({ +// type: 'EquipmentTrainingQuizResult', +// equipmentId: defaultEquipment().id, +// trainingSheetId: gsheetData.BAMBU.data.spreadsheetId!, +// actor: { +// tag: 'system', +// }, +// ...e, +// })); +// expect(results.quizResults).toHaveLength(expected.length); - for (const [actualEvent, expectedEvent] of RA.zip( - sortQuizResults(results.quizResults), - sortQuizResults(expected) - )) { - expect(actualEvent).toMatchObject< - Partial> - >(expectedEvent); - } - }); - it('Only take new rows, date in future', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), - O.some(Date.now() as EpochTimestampMilliseconds) - ); - checkQuizSync(results); - expect(results.quizResults).toHaveLength(0); - }); - it('Only take new rows, date in far past', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), - O.some(0 as EpochTimestampMilliseconds) - ); - checkQuizSync(results); - expect(results.quizResults).toHaveLength( - gsheetData.BAMBU.entries.length - ); - }); +// for (const [actualEvent, expectedEvent] of RA.zip( +// sortQuizResults(results.quizResults), +// sortQuizResults(expected) +// )) { +// expect(actualEvent).toMatchObject< +// Partial> +// >(expectedEvent); +// } +// }); +// it('Only take new rows, date in future', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!), +// O.some(Date.now() as EpochTimestampMilliseconds) +// ); +// checkQuizSync(results); +// expect(results.quizResults).toHaveLength(0); +// }); +// it('Only take new rows, date in far past', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!), +// O.some(0 as EpochTimestampMilliseconds) +// ); +// checkQuizSync(results); +// expect(results.quizResults).toHaveLength( +// gsheetData.BAMBU.entries.length +// ); +// }); - // The quiz results have dates: - // 1700768963 Thursday, November 23, 2023 7:49:23 PM - // 1700769348 Thursday, November 23, 2023 7:55:48 PM - // 1710249052 Tuesday, March 12, 2024 1:10:52 PM - // 1710249842 Tuesday, March 12, 2024 1:24:02 PM +// // The quiz results have dates: +// // 1700768963 Thursday, November 23, 2023 7:49:23 PM +// // 1700769348 Thursday, November 23, 2023 7:55:48 PM +// // 1710249052 Tuesday, March 12, 2024 1:10:52 PM +// // 1710249842 Tuesday, March 12, 2024 1:24:02 PM - it('Only take new rows, exclude 1', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), - O.some(1700768963_000 as EpochTimestampMilliseconds) - ); - checkQuizSync(results); - expect(results.quizResults).toHaveLength(3); - }); +// it('Only take new rows, exclude 1', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!), +// O.some(1700768963_000 as EpochTimestampMilliseconds) +// ); +// checkQuizSync(results); +// expect(results.quizResults).toHaveLength(3); +// }); - it('Only take new rows, exclude 2', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), - O.some(1700769348_000 as EpochTimestampMilliseconds) - ); - checkQuizSync(results); - expect(results.quizResults).toHaveLength(2); - }); +// it('Only take new rows, exclude 2', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!), +// O.some(1700769348_000 as EpochTimestampMilliseconds) +// ); +// checkQuizSync(results); +// expect(results.quizResults).toHaveLength(2); +// }); - it('Only take new rows, exclude 3', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), - O.some(1710249052_000 as EpochTimestampMilliseconds) - ); - checkQuizSync(results); - expect(results.quizResults).toHaveLength(1); - }); +// it('Only take new rows, exclude 3', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!), +// O.some(1710249052_000 as EpochTimestampMilliseconds) +// ); +// checkQuizSync(results); +// expect(results.quizResults).toHaveLength(1); +// }); - it('Only take new rows, exclude all (already have latest)', async () => { - const results = await pullEquipmentQuizResultsWrapper( - O.some(gsheetData.BAMBU.data.spreadsheetId!), - O.some(1710249842_000 as EpochTimestampMilliseconds) - ); - checkQuizSync(results); - expect(results.quizResults).toHaveLength(0); - }); - }); - }); -}); +// it('Only take new rows, exclude all (already have latest)', async () => { +// const results = await pullEquipmentQuizResultsWrapper( +// O.some(gsheetData.BAMBU.data.spreadsheetId!), +// O.some(1710249842_000 as EpochTimestampMilliseconds) +// ); +// checkQuizSync(results); +// expect(results.quizResults).toHaveLength(0); +// }); +// }); +// }); +// }); From bc721df16f437ce6a0508fc713ebf83f0af3bb89 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 22:06:31 +0100 Subject: [PATCH 10/20] Fix missing data in sheet meta --- src/init-dependencies/google/pull_sheet_data.ts | 2 +- src/training-sheets/extract-metadata.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 51b0bea1..55e1737d 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -31,7 +31,7 @@ export const pullGoogleSheetDataMetadata = }).spreadsheets.get({ spreadsheetId: trainingSheetId, includeGridData: false, // Only the metadata. - fields: 'sheets(properties)', // Only the metadata about the sheets. + fields: 'sheets(properties),properties(timeZone)', // Only the metadata about the sheets. }), reason => { logger.error(reason, 'Failed to get spreadsheet metadata'); diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index 90733c71..b4623a41 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -12,6 +12,7 @@ import { } from '../init-dependencies/google/pull_sheet_data'; import {withDefaultIfEmpty} from '../util'; import {DateTime} from 'luxon'; +import {formatValidationErrors} from 'io-ts-reporters'; const EMAIL_COLUMN_NAMES = ['email address', 'email']; @@ -71,7 +72,10 @@ export const extractInitialGoogleSheetMetadata = ( pipe( spreadsheet, SheetProperties.decode, - E.mapLeft(_e => 'Failed to extract initial google sheet metadata'), + E.mapLeft(e => { + logger.warn(formatValidationErrors(e)); + return 'Failed to extract initial google sheet metadata'; + }), E.map(properties => ({ sheets: properties.sheets.map(sheet => ({ name: sheet.properties.title, From e7c3fc4c756fa856f3ed847f3a2416fe7536e902 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 22:28:15 +0100 Subject: [PATCH 11/20] Decode as soon as we get back from api --- .../google/pull_sheet_data.ts | 43 +++++++++++++++++-- .../async-apply-external-event-sources.ts | 4 +- src/training-sheets/extract-metadata.ts | 31 +------------ src/training-sheets/google.ts | 9 +--- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 55e1737d..ed71320f 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -1,20 +1,44 @@ import {Logger} from 'pino'; import * as TE from 'fp-ts/TaskEither'; +import * as t from 'io-ts'; +import * as tt from 'io-ts-types'; +import * as E from 'fp-ts/Either'; import {Failure} from '../../types'; import {pipe} from 'fp-ts/lib/function'; import {sheets, sheets_v4} from '@googleapis/sheets'; import {GoogleAuth} from 'google-auth-library'; import {columnIndexToLetter} from '../../training-sheets/extract-metadata'; +import {formatValidationErrors} from 'io-ts-reporters'; export type GoogleSpreadsheetInitialMetadata = sheets_v4.Schema$Spreadsheet & { readonly GoogleSpreadsheetInitialMetadata: unique symbol; }; // Contains only a single sheet -export type GoogleSpreadsheetDataForSheet = sheets_v4.Schema$Spreadsheet & { - readonly GoogleSpreadsheetDataForSheet: unique symbol; -}; +export const GoogleSpreadsheetDataForSheet = t.strict({ + sheets: tt.nonEmptyArray( + // Array always has length = 1 because this is data for a single sheet. + t.strict({ + data: tt.nonEmptyArray( + t.strict({ + rowData: tt.nonEmptyArray( + t.strict({ + values: tt.nonEmptyArray( + t.strict({ + formattedValue: t.string, + }) + ), + }) + ), + }) + ), + }) + ), +}); +export type GoogleSpreadsheetDataForSheet = t.TypeOf< + typeof GoogleSpreadsheetDataForSheet +>; export const pullGoogleSheetDataMetadata = (auth: GoogleAuth) => @@ -82,7 +106,18 @@ export const pullGoogleSheetData = }; } ), - TE.map(resp => resp.data as GoogleSpreadsheetDataForSheet) + TE.map(resp => resp.data), + TE.chain(data => + TE.fromEither( + pipe( + data, + GoogleSpreadsheetDataForSheet.decode, + E.mapLeft(e => ({ + message: `Failed to get all required google spreadsheet data from API response: ${formatValidationErrors(e).join(',')}`, + })) + ) + ) + ) ); export interface GoogleHelpers { diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 75eb2dc0..c52867e4 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -22,7 +22,7 @@ import { } from '../../training-sheets/extract-metadata'; import {getChunkIndexes} from '../../util'; -const ROW_BATCH_SIZE = 500; +const ROW_BATCH_SIZE = 10; export const pullNewEquipmentQuizResultsForSheet = async ( logger: Logger, @@ -35,7 +35,7 @@ export const pullNewEquipmentQuizResultsForSheet = async ( ) => { logger.info('Processing sheet %s', sheet.name); for (const [rowStart, rowEnd] of getChunkIndexes( - 2, + 2, // 1-indexed and first row is headers. sheet.rowCount, ROW_BATCH_SIZE )) { diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index b4623a41..886782d3 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -2,7 +2,6 @@ import {pipe} from 'fp-ts/lib/function'; import * as RA from 'fp-ts/ReadonlyArray'; import * as O from 'fp-ts/Option'; import * as t from 'io-ts'; -import * as tt from 'io-ts-types'; import * as E from 'fp-ts/Either'; import {Logger} from 'pino'; @@ -85,40 +84,14 @@ export const extractInitialGoogleSheetMetadata = ( })) ); -export const SpreadsheetData = t.strict({ - sheets: tt.nonEmptyArray( - t.strict({ - data: tt.nonEmptyArray( - t.strict({ - rowData: tt.nonEmptyArray( - t.strict({ - values: tt.nonEmptyArray( - t.strict({ - formattedValue: t.string, - }) - ), - }) - ), - }) - ), - }) - ), -}); - export const extractGoogleSheetMetadata = (logger: Logger) => ( initialMeta: GoogleSheetMetadataInital, - spreadsheetDataForSheet: GoogleSpreadsheetDataForSheet + sheetData: GoogleSpreadsheetDataForSheet ): O.Option => { logger = logger.child({sheetName: initialMeta.name}); - const validated = SpreadsheetData.decode(spreadsheetDataForSheet); - if (E.isLeft(validated)) { - logger.warn('Failed to validate spreadsheet data, skipping sheet'); - return O.none; - } - - const columnNames = validated.right.sheets[0].data[0].rowData[0].values.map( + const columnNames = sheetData.sheets[0].data[0].rowData[0].values.map( col => col.formattedValue ); logger.trace('Found column names for sheet: %o', columnNames); diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index 103a0367..8eb165d6 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -1,7 +1,6 @@ import {pipe} from 'fp-ts/lib/function'; import * as RA from 'fp-ts/ReadonlyArray'; import * as O from 'fp-ts/Option'; -import * as E from 'fp-ts/Either'; import {Logger} from 'pino'; import {constructEvent, EventOfType} from '../types/domain-event'; @@ -12,7 +11,6 @@ import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-typ import { GoogleSheetMetadata, GoogleSheetMetadataInital, - SpreadsheetData, } from './extract-metadata'; import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data'; @@ -201,13 +199,8 @@ export const extractGoogleSheetData = ( spreadsheet: GoogleSpreadsheetDataForSheet ): ReadonlyArray> => { - const data = SpreadsheetData.decode(spreadsheet); - if (E.isLeft(data)) { - logger.warn('Skipping sheet %s due to missing data', trainingSheetId); - return []; - } return pipe( - data.right.sheets[0].data[0].rowData, + spreadsheet.sheets[0].data[0].rowData, RA.map( extractFromRow(logger, metadata, equipmentId, trainingSheetId, timezone) ), From e1b004587b8bbf797201b782e0bdabfd13cccf8b Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 22:44:12 +0100 Subject: [PATCH 12/20] Validate google sheet metadata straight from api --- .../google/pull_sheet_data.ts | 78 +++++++++++++++---- .../async-apply-external-event-sources.ts | 19 +---- src/training-sheets/extract-metadata.ts | 62 ++------------- src/training-sheets/google.ts | 5 +- 4 files changed, 72 insertions(+), 92 deletions(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index ed71320f..ee00a297 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -3,19 +3,51 @@ import * as TE from 'fp-ts/TaskEither'; import * as t from 'io-ts'; import * as tt from 'io-ts-types'; import * as E from 'fp-ts/Either'; -import {Failure} from '../../types'; import {pipe} from 'fp-ts/lib/function'; -import {sheets, sheets_v4} from '@googleapis/sheets'; +import {sheets} from '@googleapis/sheets'; import {GoogleAuth} from 'google-auth-library'; import {columnIndexToLetter} from '../../training-sheets/extract-metadata'; import {formatValidationErrors} from 'io-ts-reporters'; +import {DateTime} from 'luxon'; -export type GoogleSpreadsheetInitialMetadata = sheets_v4.Schema$Spreadsheet & { - readonly GoogleSpreadsheetInitialMetadata: unique symbol; -}; +const DEFAULT_TIMEZONE = 'Europe/London'; -// Contains only a single sheet +// Not all the google form sheets are actually in Europe/London. +// Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). +export const GoogleTimezone = tt.withValidate(t.string, (input, context) => + pipe( + t.string.validate(input, context), + E.chain(timezoneRaw => + DateTime.local().setZone(timezoneRaw).isValid + ? E.right(timezoneRaw) + : E.left([]) + ), + E.orElse(() => t.success(DEFAULT_TIMEZONE)) + ) +); + +export const GoogleSpreadsheetInitialMetadata = t.strict({ + properties: t.strict({ + timeZone: GoogleTimezone, + }), + sheets: t.array( + t.strict({ + properties: t.strict({ + title: t.string, + gridProperties: t.strict({ + rowCount: t.number, + }), + }), + }) + ), +}); +export type GoogleSpreadsheetInitialMetadata = t.TypeOf< + typeof GoogleSpreadsheetInitialMetadata +>; + +// Contains only a single sheet. Structure is a little verbose to match the part of the +// google api it is taken from. export const GoogleSpreadsheetDataForSheet = t.strict({ sheets: tt.nonEmptyArray( // Array always has length = 1 because this is data for a single sheet. @@ -62,7 +94,19 @@ export const pullGoogleSheetDataMetadata = return `Failed to get training spreadsheet metadata ${trainingSheetId}`; } ), - TE.map(resp => resp.data as GoogleSpreadsheetInitialMetadata) + TE.map(resp => resp.data), + TE.chain(data => + TE.fromEither( + pipe( + data, + GoogleSpreadsheetInitialMetadata.decode, + E.mapLeft( + e => + `Failed to get google spreadsheet metadata from API response: ${formatValidationErrors(e).join(',')}` + ) + ) + ) + ) ); export const pullGoogleSheetData = @@ -75,7 +119,7 @@ export const pullGoogleSheetData = rowEnd: number, columnStartIndex: number, // 0 indexed, converted to a letter. columnEndIndex: number - ): TE.TaskEither => + ): TE.TaskEither => pipe( TE.tryCatch( () => { @@ -99,11 +143,12 @@ export const pullGoogleSheetData = }); }, reason => { - logger.error(reason, 'Failed to get spreadsheet'); - return { - // Expand failure reasons. - message: `Failed to get training spreadsheet ${trainingSheetId}`, - }; + logger.error( + reason, + 'Failed to get training spreadsheet %s', + trainingSheetId + ); + return `Failed to get training spreadsheet ${trainingSheetId}`; } ), TE.map(resp => resp.data), @@ -112,9 +157,10 @@ export const pullGoogleSheetData = pipe( data, GoogleSpreadsheetDataForSheet.decode, - E.mapLeft(e => ({ - message: `Failed to get all required google spreadsheet data from API response: ${formatValidationErrors(e).join(',')}`, - })) + E.mapLeft( + e => + `Failed to get all required google spreadsheet data from API response: ${formatValidationErrors(e).join(',')}` + ) ) ) ) diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index c52867e4..df0c627f 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -16,7 +16,6 @@ import {constructEvent} from '../../types/domain-event'; import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data'; import { extractGoogleSheetMetadata, - extractInitialGoogleSheetMetadata, GoogleSheetMetadata, MAX_COLUMN_INDEX, } from '../../training-sheets/extract-metadata'; @@ -100,25 +99,11 @@ export const pullNewEquipmentQuizResults = async ( equipment.lastQuizResult ); - const initialRaw = await googleHelpers.pullGoogleSheetDataMetadata( + const initialMeta = await googleHelpers.pullGoogleSheetDataMetadata( logger, trainingSheetId )(); - if (E.isLeft(initialRaw)) { - logger.warn(initialRaw.left); - return; - } - - const initialMeta = extractInitialGoogleSheetMetadata( - logger, - initialRaw.right - ); - if (E.isLeft(initialMeta)) { - logger.warn( - 'Failed to get google sheet metadata for training sheet %s, skipping', - trainingSheetId - ); logger.warn(initialMeta.left); return; } @@ -165,7 +150,7 @@ export const pullNewEquipmentQuizResults = async ( equipment, trainingSheetId, sheet, - initialMeta.right.timezone, + initialMeta.timezone, updateState ); } diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index 886782d3..d663c091 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -16,15 +16,6 @@ import {formatValidationErrors} from 'io-ts-reporters'; const EMAIL_COLUMN_NAMES = ['email address', 'email']; type GoogleSheetName = string; -// What we can get from an initial call to google sheets without any rows. -export interface GoogleSheetMetadataInital { - name: GoogleSheetName; - rowCount: number; -} -export interface GoogleSheetsMetadataInital { - sheets: GoogleSheetMetadataInital[]; - timezone: string; -} type ColumnLetter = string; type ColumnIndex = number; // 0-indexed. @@ -46,43 +37,16 @@ export const MAX_COLUMN_INDEX = 25; export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter => 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index); -const DEFAULT_TIMEZONE = 'Europe/London'; - -const SheetProperties = t.strict({ - properties: t.strict({ - timeZone: withDefaultIfEmpty(t.string, DEFAULT_TIMEZONE), - }), - sheets: t.array( - t.strict({ - properties: t.strict({ - title: t.string, - gridProperties: t.strict({ - rowCount: t.number, - }), - }), - }) - ), -}); - export const extractInitialGoogleSheetMetadata = ( logger: Logger, spreadsheet: GoogleSpreadsheetInitialMetadata -): E.Either => - pipe( - spreadsheet, - SheetProperties.decode, - E.mapLeft(e => { - logger.warn(formatValidationErrors(e)); - return 'Failed to extract initial google sheet metadata'; - }), - E.map(properties => ({ - sheets: properties.sheets.map(sheet => ({ - name: sheet.properties.title, - rowCount: sheet.properties.gridProperties.rowCount, - })), - timezone: validateTimezone(logger, properties.properties.timeZone), - })) - ); +): GoogleSheetsMetadataInital => ({ + sheets: spreadsheet.sheets.map(sheet => ({ + name: sheet.properties.title, + rowCount: sheet.properties.gridProperties.rowCount, + })), + timezone: spreadsheet.properties.timeZone, +}); export const extractGoogleSheetMetadata = (logger: Logger) => @@ -132,15 +96,3 @@ export const extractGoogleSheetMetadata = }, }); }; - -const validateTimezone = (logger: Logger, timezone: string): string => { - if (!DateTime.local().setZone(timezone).isValid) { - // Not all the google form sheets are actually in Europe/London. - // Issue first noticed because CI is in a different zone (UTC) than local test machine (BST). - logger.info( - `Unable to determine timezone for google sheet, timezone is invalid: '${timezone}' - defaulting to Europe/London` - ); - timezone = DEFAULT_TIMEZONE; - } - return timezone; -}; diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index 8eb165d6..cdce62d8 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -8,10 +8,7 @@ import {v4} from 'uuid'; import {UUID} from 'io-ts-types'; import {DateTime} from 'luxon'; import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-types'; -import { - GoogleSheetMetadata, - GoogleSheetMetadataInital, -} from './extract-metadata'; +import {GoogleSheetMetadata} from './extract-metadata'; import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data'; // Bounds to prevent clearly broken parsing. From 816d5cf6eb5572c7f84a04b846ea0c561a338ab7 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 22:47:43 +0100 Subject: [PATCH 13/20] Don't need 2 versions of the initial metadata for a spreadsheet --- .../async-apply-external-event-sources.ts | 12 ++++--- src/training-sheets/extract-metadata.ts | 35 +++++-------------- src/training-sheets/google.ts | 8 +++-- 3 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index df0c627f..ff112144 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -120,7 +120,7 @@ export const pullNewEquipmentQuizResults = async ( const firstRowData = await googleHelpers.pullGoogleSheetData( logger, trainingSheetId, - sheet.name, + sheet.properties.title, 1, 1, 0, @@ -129,7 +129,7 @@ export const pullNewEquipmentQuizResults = async ( if (E.isLeft(firstRowData)) { logger.warn( 'Failed to get google sheet first row data for sheet %s, skipping', - sheet.name + sheet.properties.title ); continue; } @@ -139,7 +139,11 @@ export const pullNewEquipmentQuizResults = async ( continue; } - logger.info('Got metadata for sheet: %s: %o', sheet.name, meta.value); + logger.info( + 'Got metadata for sheet: %s: %o', + sheet.properties.title, + meta.value + ); sheets.push(meta.value); } @@ -150,7 +154,7 @@ export const pullNewEquipmentQuizResults = async ( equipment, trainingSheetId, sheet, - initialMeta.timezone, + initialMeta.right.properties.timeZone, updateState ); } diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index d663c091..b95b74f1 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -1,17 +1,8 @@ -import {pipe} from 'fp-ts/lib/function'; import * as RA from 'fp-ts/ReadonlyArray'; import * as O from 'fp-ts/Option'; -import * as t from 'io-ts'; -import * as E from 'fp-ts/Either'; import {Logger} from 'pino'; -import { - GoogleSpreadsheetDataForSheet, - GoogleSpreadsheetInitialMetadata, -} from '../init-dependencies/google/pull_sheet_data'; -import {withDefaultIfEmpty} from '../util'; -import {DateTime} from 'luxon'; -import {formatValidationErrors} from 'io-ts-reporters'; +import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data'; const EMAIL_COLUMN_NAMES = ['email address', 'email']; @@ -37,24 +28,15 @@ export const MAX_COLUMN_INDEX = 25; export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter => 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index); -export const extractInitialGoogleSheetMetadata = ( - logger: Logger, - spreadsheet: GoogleSpreadsheetInitialMetadata -): GoogleSheetsMetadataInital => ({ - sheets: spreadsheet.sheets.map(sheet => ({ - name: sheet.properties.title, - rowCount: sheet.properties.gridProperties.rowCount, - })), - timezone: spreadsheet.properties.timeZone, -}); - export const extractGoogleSheetMetadata = (logger: Logger) => ( - initialMeta: GoogleSheetMetadataInital, + initialMeta: { + properties: {title: string; gridProperties: {rowCount: number}}; + }, sheetData: GoogleSpreadsheetDataForSheet ): O.Option => { - logger = logger.child({sheetName: initialMeta.name}); + logger = logger.child({sheetName: initialMeta.properties.title}); const columnNames = sheetData.sheets[0].data[0].rowData[0].values.map( col => col.formattedValue ); @@ -65,7 +47,7 @@ export const extractGoogleSheetMetadata = if (O.isNone(timestamp)) { logger.warn( 'Failed to find timestamp column, skipping sheet: %s', - initialMeta.name + initialMeta.properties.title ); return O.none; } @@ -75,7 +57,7 @@ export const extractGoogleSheetMetadata = if (O.isNone(score)) { logger.warn( 'Failed to find score column, skipping sheet: %s', - initialMeta.name + initialMeta.properties.title ); return O.none; } @@ -87,7 +69,8 @@ export const extractGoogleSheetMetadata = )(columnNames); return O.some({ - ...initialMeta, + name: initialMeta.properties.title, + rowCount: initialMeta.properties.gridProperties.rowCount, mappedColumns: { timestamp: timestamp.value, score: score.value, diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index cdce62d8..e033c7ce 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -210,9 +210,11 @@ export const extractGoogleSheetData = ); }; -export const shouldPullFromSheet = ( - sheet: GoogleSheetMetadataInital -): boolean => FORM_RESPONSES_SHEET_REGEX.test(sheet.name); +export const shouldPullFromSheet = (sheet: { + properties: { + title: string; + }; +}): boolean => FORM_RESPONSES_SHEET_REGEX.test(sheet.properties.title); export const columnBoundsRequired = ( sheet: GoogleSheetMetadata From efc77d6d56a32bc0304d8e2c4d08db85916bdcf0 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 23:28:43 +0100 Subject: [PATCH 14/20] Testing local google mocks for pulling data with new method --- .../google/pull_sheet_data.ts | 3 +- src/training-sheets/extract-metadata.ts | 2 +- tests/data/google_sheet_data.ts | 82 +++++++++++++------ tests/init-dependencies/pull-local-google.ts | 38 ++++++--- .../training-sheets/extract-metadata.test.ts | 0 5 files changed, 87 insertions(+), 38 deletions(-) create mode 100644 tests/training-sheets/extract-metadata.test.ts diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index ee00a297..6c075fa5 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -10,6 +10,7 @@ import {GoogleAuth} from 'google-auth-library'; import {columnIndexToLetter} from '../../training-sheets/extract-metadata'; import {formatValidationErrors} from 'io-ts-reporters'; import {DateTime} from 'luxon'; +import { withDefaultIfEmpty } from '../../util'; const DEFAULT_TIMEZONE = 'Europe/London'; @@ -58,7 +59,7 @@ export const GoogleSpreadsheetDataForSheet = t.strict({ t.strict({ values: tt.nonEmptyArray( t.strict({ - formattedValue: t.string, + formattedValue: withDefaultIfEmpty(t.string, ''), }) ), }) diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index b95b74f1..8399b6c6 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -6,7 +6,7 @@ import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sh const EMAIL_COLUMN_NAMES = ['email address', 'email']; -type GoogleSheetName = string; +export type GoogleSheetName = string; type ColumnLetter = string; type ColumnIndex = number; // 0-indexed. diff --git a/tests/data/google_sheet_data.ts b/tests/data/google_sheet_data.ts index 7c612a3f..a8c4965e 100644 --- a/tests/data/google_sheet_data.ts +++ b/tests/data/google_sheet_data.ts @@ -1,6 +1,12 @@ import {sheets_v4} from '@googleapis/sheets'; import {readFileSync} from 'node:fs'; import {EpochTimestampMilliseconds} from '../../src/read-models/shared-state/return-types'; +import { + GoogleSpreadsheetDataForSheet, + GoogleSpreadsheetInitialMetadata, +} from '../../src/init-dependencies/google/pull_sheet_data'; +import {GoogleSheetName} from '../../src/training-sheets/extract-metadata'; +import {getRightOrFail} from '../helpers'; type ManualParsedEntry = { emailProvided: string; @@ -13,22 +19,47 @@ type ManualParsedEntry = { }; export type ManualParsed = { - data: sheets_v4.Schema$Spreadsheet; + apiResp: sheets_v4.Schema$Spreadsheet; + metadata: GoogleSpreadsheetInitialMetadata; + sheets: Record; entries: ManualParsedEntry[]; }; -export const EMPTY: ManualParsed = { - data: JSON.parse( +const genManualParsed = ( + // Enumlate the google api which filters out other sheets using the range parameter. + apiResp: sheets_v4.Schema$Spreadsheet, + entries: ManualParsedEntry[] +): ManualParsed => { + const sheets: Record = {}; + for (const sheet of apiResp.sheets!) { + const apiRespCopy = JSON.parse(JSON.stringify(apiResp)) as typeof apiResp; + apiRespCopy.sheets = apiRespCopy.sheets!.filter( + s => s.properties!.title === sheet.properties!.title + ); + sheets[sheet.properties!.title as string] = getRightOrFail( + GoogleSpreadsheetDataForSheet.decode(apiRespCopy) + ); + } + return { + apiResp, + entries, + sheets, + metadata: getRightOrFail(GoogleSpreadsheetInitialMetadata.decode(apiResp)), + }; +}; + +export const EMPTY = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_empty.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - entries: [], -}; -export const METAL_LATHE: ManualParsed = { - data: JSON.parse( + [] +); + +export const METAL_LATHE: ManualParsed = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_metal_lathe.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - // Manually parsed data for testing: - entries: [ + [ { emailProvided: 'test@makespace.com', memberNumberProvided: 1234, @@ -38,14 +69,14 @@ export const METAL_LATHE: ManualParsed = { fullMarks: false, timestampEpochMS: 1705770960_000 as EpochTimestampMilliseconds, }, - ], -}; + ] +); -export const BAMBU: ManualParsed = { - data: JSON.parse( +export const BAMBU: ManualParsed = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_bambu.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - entries: [ + [ // Manually parsed data for testing: { emailProvided: 'flonn@example.com', @@ -83,13 +114,14 @@ export const BAMBU: ManualParsed = { fullMarks: true, timestampEpochMS: 1710249842_000 as EpochTimestampMilliseconds, }, - ], -}; -export const LASER_CUTTER: ManualParsed = { - data: JSON.parse( + ] +); + +export const LASER_CUTTER: ManualParsed = genManualParsed( + JSON.parse( readFileSync('./tests/data/google_spreadsheets_laser_cutter.json', 'utf8') ) as sheets_v4.Schema$Spreadsheet, - entries: [ + [ // Manually parsed data for testing // Note some entries were missing in the source spreadsheet so are treated as '' to match the spreadsheet behaviour. { @@ -110,8 +142,8 @@ export const LASER_CUTTER: ManualParsed = { fullMarks: true, timestampEpochMS: 1601298462_000 as EpochTimestampMilliseconds, }, - ], -}; + ] +); export const getLatestEvent = (data: ManualParsed) => data.entries.sort((a, b) => a.timestampEpochMS - b.timestampEpochMS)[ @@ -129,8 +161,8 @@ export const NOT_FOUND_ERROR = { code: '404', }; export const TRAINING_SHEETS = { - [EMPTY.data.spreadsheetId!]: EMPTY, - [METAL_LATHE.data.spreadsheetId!]: METAL_LATHE, - [LASER_CUTTER.data.spreadsheetId!]: LASER_CUTTER, - [BAMBU.data.spreadsheetId!]: BAMBU, + [EMPTY.apiResp.spreadsheetId!]: EMPTY, + [METAL_LATHE.apiResp.spreadsheetId!]: METAL_LATHE, + [LASER_CUTTER.apiResp.spreadsheetId!]: LASER_CUTTER, + [BAMBU.apiResp.spreadsheetId!]: BAMBU, }; diff --git a/tests/init-dependencies/pull-local-google.ts b/tests/init-dependencies/pull-local-google.ts index 3049dea5..768d07e5 100644 --- a/tests/init-dependencies/pull-local-google.ts +++ b/tests/init-dependencies/pull-local-google.ts @@ -1,20 +1,36 @@ import {Logger} from 'pino'; import * as TE from 'fp-ts/TaskEither'; import * as gsheetData from '../data/google_sheet_data'; -import {GoogleHelpers} from '../../src/init-dependencies/google/pull_sheet_data'; +import { + GoogleHelpers, + GoogleSpreadsheetDataForSheet, + GoogleSpreadsheetInitialMetadata, +} from '../../src/init-dependencies/google/pull_sheet_data'; -const localPullGoogleSheetData = (logger: Logger, trainingSheetId: string) => { - return '' as any; -// logger.debug(`Pulling local google sheet '${trainingSheetId}'`); -// const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].data; -// return sheet -// ? TE.right(sheet) -// : TE.left({ -// message: 'Sheet not found', -// }); +const localPullGoogleSheetDataMetadata = ( + logger: Logger, + trainingSheetId: string +): TE.TaskEither => { + logger.debug(`Pulling local google sheet metadata '${trainingSheetId}'`); + const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].metadata; + return sheet ? TE.right(sheet) : TE.left('Spreadsheet not found'); +}; + +const localPullGoogleSheetData = ( + logger: Logger, + trainingSheetId: string, + sheetName: string, + _rowStart: number, + _rowEnd: number, + _columnStartIndex: number, + _columnEndIndex: number +): TE.TaskEither => { + logger.debug(`Pulling local google sheet '${trainingSheetId}'`); + const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].sheets[sheetName]; + return sheet ? TE.right(sheet) : TE.left('Sheet not found'); }; export const localGoogleHelpers: GoogleHelpers = { pullGoogleSheetData: localPullGoogleSheetData, - pullGoogleSheetDataMetadata: localPullGoogleSheetData, + pullGoogleSheetDataMetadata: localPullGoogleSheetDataMetadata, }; diff --git a/tests/training-sheets/extract-metadata.test.ts b/tests/training-sheets/extract-metadata.test.ts new file mode 100644 index 00000000..e69de29b From 26220fc56147fc08e864bf48091e2369d15d46dd Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 23:30:08 +0100 Subject: [PATCH 15/20] Use withFallback rather than slightly confusing withDefaultIfEmpty with an empty default --- src/init-dependencies/google/pull_sheet_data.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/init-dependencies/google/pull_sheet_data.ts b/src/init-dependencies/google/pull_sheet_data.ts index 6c075fa5..e66961fb 100644 --- a/src/init-dependencies/google/pull_sheet_data.ts +++ b/src/init-dependencies/google/pull_sheet_data.ts @@ -10,7 +10,6 @@ import {GoogleAuth} from 'google-auth-library'; import {columnIndexToLetter} from '../../training-sheets/extract-metadata'; import {formatValidationErrors} from 'io-ts-reporters'; import {DateTime} from 'luxon'; -import { withDefaultIfEmpty } from '../../util'; const DEFAULT_TIMEZONE = 'Europe/London'; @@ -59,7 +58,7 @@ export const GoogleSpreadsheetDataForSheet = t.strict({ t.strict({ values: tt.nonEmptyArray( t.strict({ - formattedValue: withDefaultIfEmpty(t.string, ''), + formattedValue: tt.withFallback(t.string, ''), }) ), }) From 858936d102bf78c373b6e2aaa63b74e19de538cf Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 23:39:48 +0100 Subject: [PATCH 16/20] Initial extract metadata test --- src/training-sheets/extract-metadata.ts | 4 +-- tests/data/google_spreadsheets_empty.json | 2 +- .../training-sheets/extract-metadata.test.ts | 25 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/training-sheets/extract-metadata.ts b/src/training-sheets/extract-metadata.ts index 8399b6c6..300fe844 100644 --- a/src/training-sheets/extract-metadata.ts +++ b/src/training-sheets/extract-metadata.ts @@ -34,10 +34,10 @@ export const extractGoogleSheetMetadata = initialMeta: { properties: {title: string; gridProperties: {rowCount: number}}; }, - sheetData: GoogleSpreadsheetDataForSheet + firstRowData: GoogleSpreadsheetDataForSheet ): O.Option => { logger = logger.child({sheetName: initialMeta.properties.title}); - const columnNames = sheetData.sheets[0].data[0].rowData[0].values.map( + const columnNames = firstRowData.sheets[0].data[0].rowData[0].values.map( col => col.formattedValue ); logger.trace('Found column names for sheet: %o', columnNames); diff --git a/tests/data/google_spreadsheets_empty.json b/tests/data/google_spreadsheets_empty.json index 17fcc40c..825af9f8 100644 --- a/tests/data/google_spreadsheets_empty.json +++ b/tests/data/google_spreadsheets_empty.json @@ -139,7 +139,7 @@ "index": 0, "sheetType": "GRID", "gridProperties": { - "rowCount": 190, + "rowCount": 1, "columnCount": 25, "frozenRowCount": 1 } diff --git a/tests/training-sheets/extract-metadata.test.ts b/tests/training-sheets/extract-metadata.test.ts index e69de29b..a9bb69fd 100644 --- a/tests/training-sheets/extract-metadata.test.ts +++ b/tests/training-sheets/extract-metadata.test.ts @@ -0,0 +1,25 @@ +import pino from 'pino'; +import * as O from 'fp-ts/Option'; +import {extractGoogleSheetMetadata} from '../../src/training-sheets/extract-metadata'; + +import * as gsheetData from '../data/google_sheet_data'; +import {getSomeOrFail} from '../helpers'; + +describe('extract metadata', () => { + it('Empty sheet', () => { + const metadata = gsheetData.EMPTY.metadata; + const result = getSomeOrFail( + extractGoogleSheetMetadata(pino())( + metadata.sheets[0], + Object.values(gsheetData.EMPTY.sheets)[0] + ) + ); + + expect(result.name).toStrictEqual('Form Responses 1'); + expect(result.rowCount).toStrictEqual(1); + expect(result.mappedColumns.timestamp).toStrictEqual(0); + expect(result.mappedColumns.email).toStrictEqual(O.some(1)); + expect(result.mappedColumns.score).toStrictEqual(2); + expect(result.mappedColumns.memberNumber).toStrictEqual(O.some(4)); + }); +}); From 5b9f22d83be2de5b87072744b42e94bd50e8c074 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 23:44:37 +0100 Subject: [PATCH 17/20] Check I got google timezone parsing with fallback right --- .../col-bounds-required.test.ts | 2 +- .../get-chunk-indexes.test.ts | 2 +- tests/training-sheets/google-timezone.test.ts | 25 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) rename tests/{ => training-sheets}/col-bounds-required.test.ts (93%) rename tests/{ => training-sheets}/get-chunk-indexes.test.ts (91%) create mode 100644 tests/training-sheets/google-timezone.test.ts diff --git a/tests/col-bounds-required.test.ts b/tests/training-sheets/col-bounds-required.test.ts similarity index 93% rename from tests/col-bounds-required.test.ts rename to tests/training-sheets/col-bounds-required.test.ts index d0c08a70..31e8905a 100644 --- a/tests/col-bounds-required.test.ts +++ b/tests/training-sheets/col-bounds-required.test.ts @@ -1,5 +1,5 @@ import * as O from 'fp-ts/Option'; -import {columnBoundsRequired} from '../src/training-sheets/google'; +import {columnBoundsRequired} from '../../src/training-sheets/google'; describe('columnBoundsRequired', () => { [ diff --git a/tests/get-chunk-indexes.test.ts b/tests/training-sheets/get-chunk-indexes.test.ts similarity index 91% rename from tests/get-chunk-indexes.test.ts rename to tests/training-sheets/get-chunk-indexes.test.ts index 1af51139..6ca7101e 100644 --- a/tests/get-chunk-indexes.test.ts +++ b/tests/training-sheets/get-chunk-indexes.test.ts @@ -1,4 +1,4 @@ -import {getChunkIndexes} from '../src/util'; +import {getChunkIndexes} from '../../src/util'; describe('Get chunk indexes', () => { [ diff --git a/tests/training-sheets/google-timezone.test.ts b/tests/training-sheets/google-timezone.test.ts new file mode 100644 index 00000000..6e0aceb3 --- /dev/null +++ b/tests/training-sheets/google-timezone.test.ts @@ -0,0 +1,25 @@ +import {GoogleTimezone} from '../../src/init-dependencies/google/pull_sheet_data'; +import {getRightOrFail} from '../helpers'; + +describe('Google timezone parse', () => { + it('Empty default', () => { + expect(getRightOrFail(GoogleTimezone.decode(''))).toStrictEqual( + 'Europe/London' + ); + }); + it('Malformed default', () => { + expect(getRightOrFail(GoogleTimezone.decode(null))).toStrictEqual( + 'Europe/London' + ); + }); + it('Known timezone', () => { + expect(getRightOrFail(GoogleTimezone.decode('Africa/Cairo'))).toStrictEqual( + 'Africa/Cairo' + ); + }); + it('Unknown timezone', () => { + expect( + getRightOrFail(GoogleTimezone.decode('Makespace/Cambridge')) + ).toStrictEqual('Europe/London'); + }); +}); From 2bffc83e9770a491fbfeb50f5cd541dcff20ab02 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 23:51:12 +0100 Subject: [PATCH 18/20] Re-enable integration tests --- .../async-apply-external-event-sources.ts | 6 +- .../async-apply-external.test.ts | 405 +++++++++--------- 2 files changed, 203 insertions(+), 208 deletions(-) diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index ff112144..6790454f 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -21,9 +21,9 @@ import { } from '../../training-sheets/extract-metadata'; import {getChunkIndexes} from '../../util'; -const ROW_BATCH_SIZE = 10; +const ROW_BATCH_SIZE = 200; -export const pullNewEquipmentQuizResultsForSheet = async ( +const pullNewEquipmentQuizResultsForSheet = async ( logger: Logger, googleHelpers: GoogleHelpers, equipment: Equipment, @@ -77,7 +77,7 @@ export const pullNewEquipmentQuizResultsForSheet = async ( } }; -export const pullNewEquipmentQuizResults = async ( +const pullNewEquipmentQuizResults = async ( logger: Logger, googleHelpers: GoogleHelpers, equipment: Equipment, diff --git a/tests/training-sheets/async-apply-external.test.ts b/tests/training-sheets/async-apply-external.test.ts index 09784d12..998fd8d9 100644 --- a/tests/training-sheets/async-apply-external.test.ts +++ b/tests/training-sheets/async-apply-external.test.ts @@ -1,216 +1,211 @@ -// import * as O from 'fp-ts/Option'; -// import {NonEmptyString, UUID} from 'io-ts-types'; -// import {faker} from '@faker-js/faker'; -// import * as gsheetData from '../data/google_sheet_data'; -// import {initTestFramework, TestFramework} from '../read-models/test-framework'; -// import {EmailAddress} from '../../src/types'; -// import {getSomeOrFail} from '../helpers'; -// import { -// EpochTimestampMilliseconds, -// Equipment, -// } from '../../src/read-models/shared-state/return-types'; +import * as O from 'fp-ts/Option'; +import {NonEmptyString, UUID} from 'io-ts-types'; +import {faker} from '@faker-js/faker'; +import * as gsheetData from '../data/google_sheet_data'; +import {initTestFramework, TestFramework} from '../read-models/test-framework'; +import {EmailAddress} from '../../src/types'; +import {getSomeOrFail} from '../helpers'; +import { + EpochTimestampMilliseconds, + Equipment, +} from '../../src/read-models/shared-state/return-types'; -// describe('Integration asyncApplyExternalEventSources', () => { -// const addArea = async (framework: TestFramework) => { -// const createArea = { -// id: faker.string.uuid() as UUID, -// name: faker.company.buzzNoun() as NonEmptyString, -// }; -// await framework.commands.area.create(createArea); -// return createArea.id; -// }; +describe('Integration asyncApplyExternalEventSources', () => { + const addArea = async (framework: TestFramework) => { + const createArea = { + id: faker.string.uuid() as UUID, + name: faker.company.buzzNoun() as NonEmptyString, + }; + await framework.commands.area.create(createArea); + return createArea.id; + }; -// const addWithSheet = async ( -// framework: TestFramework, -// name: string, -// areaId: UUID, -// trainingSheetId: O.Option -// ) => { -// const equipment = { -// id: faker.string.uuid() as UUID, -// name: name as NonEmptyString, -// areaId, -// }; -// await framework.commands.equipment.add(equipment); -// if (O.isSome(trainingSheetId)) { -// await framework.commands.equipment.trainingSheet({ -// equipmentId: equipment.id, -// trainingSheetId: trainingSheetId.value, -// }); -// } -// return { -// ...equipment, -// trainingSheetId, -// }; -// }; + const addWithSheet = async ( + framework: TestFramework, + name: string, + areaId: UUID, + trainingSheetId: O.Option + ) => { + const equipment = { + id: faker.string.uuid() as UUID, + name: name as NonEmptyString, + areaId, + }; + await framework.commands.equipment.add(equipment); + if (O.isSome(trainingSheetId)) { + await framework.commands.equipment.trainingSheet({ + equipmentId: equipment.id, + trainingSheetId: trainingSheetId.value, + }); + } + return { + ...equipment, + trainingSheetId, + }; + }; -// it('Handle multiple equipment both populated', async () => { -// const framework = await initTestFramework(1000); + it('Handle multiple equipment both populated', async () => { + const framework = await initTestFramework(1000); -// // Create the users which the results are registered too. -// await framework.commands.memberNumbers.linkNumberToEmail({ -// memberNumber: gsheetData.BAMBU.entries[0].memberNumberProvided, -// email: gsheetData.BAMBU.entries[0].emailProvided as EmailAddress, -// }); -// await framework.commands.memberNumbers.linkNumberToEmail({ -// memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, -// email: gsheetData.METAL_LATHE.entries[0].emailProvided as EmailAddress, -// }); -// const areaId = await addArea(framework); -// const bambu = await addWithSheet( -// framework, -// 'bambu', -// areaId, -// O.some(gsheetData.BAMBU.data.spreadsheetId!) -// ); -// const lathe = await addWithSheet( -// framework, -// 'Metal Lathe', -// areaId, -// O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) -// ); -// const results = await runAsyncApplyExternalEventSources(framework); -// checkLastQuizSyncUpdated(results); -// checkLastQuizEventTimestamp( -// gsheetData.BAMBU, -// results.equipmentAfter.get(bambu.id)! -// ); -// checkLastQuizEventTimestamp( -// gsheetData.METAL_LATHE, -// results.equipmentAfter.get(lathe.id)! -// ); + // Create the users which the results are registered too. + await framework.commands.memberNumbers.linkNumberToEmail({ + memberNumber: gsheetData.BAMBU.entries[0].memberNumberProvided, + email: gsheetData.BAMBU.entries[0].emailProvided as EmailAddress, + }); + await framework.commands.memberNumbers.linkNumberToEmail({ + memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, + email: gsheetData.METAL_LATHE.entries[0].emailProvided as EmailAddress, + }); + const areaId = await addArea(framework); + const bambu = await addWithSheet( + framework, + 'bambu', + areaId, + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) + ); + const lathe = await addWithSheet( + framework, + 'Metal Lathe', + areaId, + O.some(gsheetData.METAL_LATHE.apiResp.spreadsheetId!) + ); + const results = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results); + checkLastQuizEventTimestamp( + gsheetData.BAMBU, + results.equipmentAfter.get(bambu.id)! + ); + checkLastQuizEventTimestamp( + gsheetData.METAL_LATHE, + results.equipmentAfter.get(lathe.id)! + ); -// // We already test the produced quiz result events above -// // and testing updateState is also tested elsewhere so this integration -// // test doesn't need to enumerate every combination it just needs to check -// // that generally the equipment is getting updated. -// const bambuAfter = results.equipmentAfter.get(bambu.id)!; -// expect(bambuAfter.orphanedPassedQuizes).toHaveLength(0); -// expect(bambuAfter.membersAwaitingTraining).toHaveLength(1); -// expect(bambuAfter.membersAwaitingTraining[0].memberNumber).toStrictEqual( -// gsheetData.BAMBU.entries[0].memberNumberProvided -// ); -// expect(bambuAfter.membersAwaitingTraining[0].emailAddress).toStrictEqual( -// gsheetData.BAMBU.entries[0].emailProvided -// ); -// expect(bambuAfter.membersAwaitingTraining[0].waitingSince).toStrictEqual( -// new Date(gsheetData.getLatestEvent(gsheetData.BAMBU).timestampEpochMS) -// ); + // We already test the produced quiz result events above + // and testing updateState is also tested elsewhere so this integration + // test doesn't need to enumerate every combination it just needs to check + // that generally the equipment is getting updated. + const bambuAfter = results.equipmentAfter.get(bambu.id)!; + expect(bambuAfter.orphanedPassedQuizes).toHaveLength(0); + expect(bambuAfter.membersAwaitingTraining).toHaveLength(1); + expect(bambuAfter.membersAwaitingTraining[0].memberNumber).toStrictEqual( + gsheetData.BAMBU.entries[0].memberNumberProvided + ); + expect(bambuAfter.membersAwaitingTraining[0].emailAddress).toStrictEqual( + gsheetData.BAMBU.entries[0].emailProvided + ); + expect(bambuAfter.membersAwaitingTraining[0].waitingSince).toStrictEqual( + new Date(gsheetData.getLatestEvent(gsheetData.BAMBU).timestampEpochMS) + ); -// // Lathe results only have a single failed entry. -// const latheAfter = results.equipmentAfter.get(lathe.id)!; -// expect(latheAfter.orphanedPassedQuizes).toHaveLength(0); -// expect(latheAfter.failedQuizAttempts).toHaveLength(1); -// expect(latheAfter.failedQuizAttempts[0]).toMatchObject({ -// emailAddress: gsheetData.METAL_LATHE.entries[0] -// .emailProvided as EmailAddress, -// memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, -// score: gsheetData.METAL_LATHE.entries[0].score, -// maxScore: gsheetData.METAL_LATHE.entries[0].maxScore, -// percentage: gsheetData.METAL_LATHE.entries[0].percentage, -// timestamp: new Date(gsheetData.METAL_LATHE.entries[0].timestampEpochMS), -// }); -// }); -// // it('Handle no equipment', async () => { -// // const framework = await initTestFramework(1000); -// // const results = await runAsyncApplyExternalEventSources(framework); -// // checkLastQuizSync(results); -// // expect(results.equipmentAfter.size).toStrictEqual(0); -// // }); -// // it('Handle equipment with no training sheet', async () => { -// // const framework = await initTestFramework(1000); -// // const areaId = await addArea(framework); -// // const bambu = await addWithSheet(framework, 'bambu', areaId, O.none); -// // const results = await runAsyncApplyExternalEventSources(framework); -// // expect( -// // results.equipmentAfter.get(bambu.id)!.lastQuizSync // No training sheet so not updated. -// // ).toStrictEqual(O.none); -// // expect( -// // results.equipmentAfter.get(bambu.id)!.lastQuizResult -// // ).toStrictEqual(O.none); -// // expect(results.newEvents).toHaveLength(0); -// // }); -// // it('Rate limit equipment pull', async () => { -// // const framework = await initTestFramework(1000); -// // const areaId = await addArea(framework); -// // const bambu = await addWithSheet( -// // framework, -// // 'bambu', -// // areaId, -// // O.some(gsheetData.BAMBU.data.spreadsheetId!) -// // ); -// // const results1 = await runAsyncApplyExternalEventSources(framework); -// // checkLastQuizSync(results1); -// // const results2 = await runAsyncApplyExternalEventSources(framework); -// // expect( -// // results1.equipmentAfter.get(bambu.id)!.lastQuizSync -// // ).toStrictEqual(results2.equipmentAfter.get(bambu.id)!.lastQuizSync); -// // expect(results1.newEvents.length).toBeGreaterThan(0); -// // expect(results2.newEvents).toHaveLength(0); -// // }); -// // it('Repeat equipment pull no rate limit', async () => { -// // const rateLimitMs = 100; -// // const framework = await initTestFramework(rateLimitMs); -// // const areaId = await addArea(framework); -// // const bambu = await addWithSheet( -// // framework, -// // 'bambu', -// // areaId, -// // O.some(gsheetData.BAMBU.data.spreadsheetId!) -// // ); -// // const results1 = await runAsyncApplyExternalEventSources(framework); -// // checkLastQuizSync(results1); + // Lathe results only have a single failed entry. + const latheAfter = results.equipmentAfter.get(lathe.id)!; + expect(latheAfter.orphanedPassedQuizes).toHaveLength(0); + expect(latheAfter.failedQuizAttempts).toHaveLength(1); + expect(latheAfter.failedQuizAttempts[0]).toMatchObject({ + emailAddress: gsheetData.METAL_LATHE.entries[0] + .emailProvided as EmailAddress, + memberNumber: gsheetData.METAL_LATHE.entries[0].memberNumberProvided, + score: gsheetData.METAL_LATHE.entries[0].score, + maxScore: gsheetData.METAL_LATHE.entries[0].maxScore, + percentage: gsheetData.METAL_LATHE.entries[0].percentage, + timestamp: new Date(gsheetData.METAL_LATHE.entries[0].timestampEpochMS), + }); + }); + it('Handle no equipment', async () => { + const framework = await initTestFramework(1000); + const results = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results); + expect(results.equipmentAfter.size).toStrictEqual(0); + }); + it('Handle equipment with no training sheet', async () => { + const framework = await initTestFramework(1000); + const areaId = await addArea(framework); + const bambu = await addWithSheet(framework, 'bambu', areaId, O.none); + const results = await runAsyncApplyExternalEventSources(framework); + expect( + results.equipmentAfter.get(bambu.id)!.lastQuizSync // No training sheet so not updated. + ).toStrictEqual(O.none); + expect(results.equipmentAfter.get(bambu.id)!.lastQuizResult).toStrictEqual( + O.none + ); + }); + it('Rate limit equipment pull', async () => { + const framework = await initTestFramework(1000); + const areaId = await addArea(framework); + const bambu = await addWithSheet( + framework, + 'bambu', + areaId, + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) + ); + const results1 = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results1); + const results2 = await runAsyncApplyExternalEventSources(framework); + expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).toStrictEqual( + results2.equipmentAfter.get(bambu.id)!.lastQuizSync + ); + }); + it('Repeat equipment pull no rate limit', async () => { + const rateLimitMs = 100; + const framework = await initTestFramework(rateLimitMs); + const areaId = await addArea(framework); + const bambu = await addWithSheet( + framework, + 'bambu', + areaId, + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) + ); + const results1 = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results1); -// // await new Promise(res => setTimeout(res, rateLimitMs)); -// // const results2 = await runAsyncApplyExternalEventSources(framework); -// // checkLastQuizSync(results2); -// // expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( -// // results2.equipmentAfter.get(bambu.id)!.lastQuizSync -// // ); -// // expect(results1.newEvents.length).toBeGreaterThan(0); -// // expect(results2.newEvents).toHaveLength(0); -// // }); -// }); + await new Promise(res => setTimeout(res, rateLimitMs)); + const results2 = await runAsyncApplyExternalEventSources(framework); + checkLastQuizSyncUpdated(results2); + expect(results1.equipmentAfter.get(bambu.id)!.lastQuizSync).not.toEqual( + results2.equipmentAfter.get(bambu.id)!.lastQuizSync + ); + }); +}); -// type ApplyExternalEventsResults = { -// startTime: EpochTimestampMilliseconds; -// endTime: EpochTimestampMilliseconds; -// equipmentAfter: Map; -// }; +type ApplyExternalEventsResults = { + startTime: EpochTimestampMilliseconds; + endTime: EpochTimestampMilliseconds; + equipmentAfter: Map; +}; -// const runAsyncApplyExternalEventSources = async ( -// framework: TestFramework -// ): Promise => { -// const startTime = Date.now() as EpochTimestampMilliseconds; -// await framework.sharedReadModel.asyncApplyExternalEventSources()(); -// const endTime = Date.now() as EpochTimestampMilliseconds; -// const equipmentAfter = new Map( -// framework.sharedReadModel.equipment.getAll().map(e => [e.id, e]) -// ); -// return { -// startTime, -// endTime, -// equipmentAfter, -// }; -// }; +const runAsyncApplyExternalEventSources = async ( + framework: TestFramework +): Promise => { + const startTime = Date.now() as EpochTimestampMilliseconds; + await framework.sharedReadModel.asyncApplyExternalEventSources()(); + const endTime = Date.now() as EpochTimestampMilliseconds; + const equipmentAfter = new Map( + framework.sharedReadModel.equipment.getAll().map(e => [e.id, e]) + ); + return { + startTime, + endTime, + equipmentAfter, + }; +}; -// const checkLastQuizSyncUpdated = (results: ApplyExternalEventsResults) => { -// // Check that the last quiz sync property is updated to reflect -// // that a quiz sync was preformed. -// for (const equipment of results.equipmentAfter.values()) { -// expect(getSomeOrFail(equipment.lastQuizSync)).toBeGreaterThanOrEqual( -// results.startTime -// ); -// expect(getSomeOrFail(equipment.lastQuizSync)).toBeLessThanOrEqual( -// results.endTime -// ); -// } -// }; +const checkLastQuizSyncUpdated = (results: ApplyExternalEventsResults) => { + // Check that the last quiz sync property is updated to reflect + // that a quiz sync was preformed. + for (const equipment of results.equipmentAfter.values()) { + expect(getSomeOrFail(equipment.lastQuizSync)).toBeGreaterThanOrEqual( + results.startTime + ); + expect(getSomeOrFail(equipment.lastQuizSync)).toBeLessThanOrEqual( + results.endTime + ); + } +}; -// const checkLastQuizEventTimestamp = ( -// data: gsheetData.ManualParsed, -// equipmentAfter: Equipment -// ) => -// expect(getSomeOrFail(equipmentAfter.lastQuizResult)).toStrictEqual( -// gsheetData.getLatestEvent(data).timestampEpochMS -// ); +const checkLastQuizEventTimestamp = ( + data: gsheetData.ManualParsed, + equipmentAfter: Equipment +) => + expect(getSomeOrFail(equipmentAfter.lastQuizResult)).toStrictEqual( + gsheetData.getLatestEvent(data).timestampEpochMS + ); From d3caa0e90943a355bc3a870075f953042bf97e0d Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Wed, 25 Sep 2024 23:57:14 +0100 Subject: [PATCH 19/20] All existing tests are now re-enabled --- .../async-apply-external-event-sources.ts | 2 +- src/training-sheets/google.ts | 2 + tests/training-sheets/process-events.test.ts | 476 +++++++++--------- 3 files changed, 246 insertions(+), 234 deletions(-) diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts index 6790454f..2bb6d0b8 100644 --- a/src/read-models/shared-state/async-apply-external-event-sources.ts +++ b/src/read-models/shared-state/async-apply-external-event-sources.ts @@ -77,7 +77,7 @@ const pullNewEquipmentQuizResultsForSheet = async ( } }; -const pullNewEquipmentQuizResults = async ( +export const pullNewEquipmentQuizResults = async ( logger: Logger, googleHelpers: GoogleHelpers, equipment: Equipment, diff --git a/src/training-sheets/google.ts b/src/training-sheets/google.ts index e033c7ce..3c8a4144 100644 --- a/src/training-sheets/google.ts +++ b/src/training-sheets/google.ts @@ -191,6 +191,8 @@ export const extractGoogleSheetData = equipmentId: UUID, metadata: GoogleSheetMetadata, timezone: string, + // Note we filter events on timestamp rather than last row currently to handle + // blank rows but potentially we could switch if we added detection for blank rows. eventsFromExclusive: O.Option ) => ( diff --git a/tests/training-sheets/process-events.test.ts b/tests/training-sheets/process-events.test.ts index ad4376f8..ff0014d8 100644 --- a/tests/training-sheets/process-events.test.ts +++ b/tests/training-sheets/process-events.test.ts @@ -1,247 +1,257 @@ -// import {UUID} from 'io-ts-types'; -// import {EventOfType, isEventOfType} from '../../src/types/domain-event'; -// import pino from 'pino'; -// import * as RA from 'fp-ts/lib/ReadonlyArray'; -// import * as N from 'fp-ts/number'; -// import * as O from 'fp-ts/Option'; -// import * as gsheetData from '../data/google_sheet_data'; -// import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources'; -// import { -// EpochTimestampMilliseconds, -// Equipment, -// } from '../../src/read-models/shared-state/return-types'; -// import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; +import {UUID} from 'io-ts-types'; +import { + DomainEvent, + EventOfType, + isEventOfType, +} from '../../src/types/domain-event'; +import pino from 'pino'; +import * as RA from 'fp-ts/lib/ReadonlyArray'; +import * as N from 'fp-ts/number'; +import * as O from 'fp-ts/Option'; +import * as gsheetData from '../data/google_sheet_data'; +import {pullNewEquipmentQuizResults} from '../../src/read-models/shared-state/async-apply-external-event-sources'; +import { + EpochTimestampMilliseconds, + Equipment, +} from '../../src/read-models/shared-state/return-types'; +import {localGoogleHelpers} from '../init-dependencies/pull-local-google'; -// const sortQuizResults = RA.sort({ -// compare: (a, b) => -// N.Ord.compare( -// (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, -// (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS -// ), -// equals: (a, b) => -// N.Ord.equals( -// (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, -// (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS -// ), -// }); +const sortQuizResults = RA.sort({ + compare: (a, b) => + N.Ord.compare( + (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, + (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS + ), + equals: (a, b) => + N.Ord.equals( + (a as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS, + (b as EventOfType<'EquipmentTrainingQuizResult'>).timestampEpochMS + ), +}); -// const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => -// pullNewEquipmentQuizResults( -// pino({ -// level: 'fatal', -// timestamp: pino.stdTimeFunctions.isoTime, -// }), -// localGoogleHelpers, -// equipment -// )(); +const pullNewEquipmentQuizResultsLocal = async (equipment: Equipment) => { + const newEvents: DomainEvent[] = []; + await pullNewEquipmentQuizResults( + pino({ + level: 'fatal', + timestamp: pino.stdTimeFunctions.isoTime, + }), + localGoogleHelpers, + equipment, + newEvent => { + newEvents.push(newEvent); + } + ); + return newEvents; +}; -// const defaultEquipment = (): Equipment => ({ -// id: 'ebedee32-49f4-4d36-a350-4fa7848792bf' as UUID, -// name: 'Metal Lathe', -// trainers: [], -// trainedMembers: [], -// area: { -// id: 'f9cee7aa-75c6-42cc-8585-0e658044fe8e', -// name: 'Metal Shop', -// }, -// membersAwaitingTraining: [], -// orphanedPassedQuizes: [], -// failedQuizAttempts: [], -// trainingSheetId: O.none, -// lastQuizResult: O.none, -// lastQuizSync: O.none, -// }); +const defaultEquipment = (): Equipment => ({ + id: 'ebedee32-49f4-4d36-a350-4fa7848792bf' as UUID, + name: 'Metal Lathe', + trainers: [], + trainedMembers: [], + area: { + id: 'f9cee7aa-75c6-42cc-8585-0e658044fe8e', + name: 'Metal Shop', + }, + membersAwaitingTraining: [], + orphanedPassedQuizes: [], + failedQuizAttempts: [], + trainingSheetId: O.none, + lastQuizResult: O.none, + lastQuizSync: O.none, +}); -// type EquipmentQuizResultEvents = { -// quizResults: ReadonlyArray>; -// quizSync: ReadonlyArray>; -// startTime: Date; -// endTime: Date; -// }; -// const pullEquipmentQuizResultsWrapper = async ( -// spreadsheetId: O.Option, -// lastQuizResult: O.Option = O.none -// ): Promise => { -// const equipment = defaultEquipment(); -// equipment.trainingSheetId = spreadsheetId; -// equipment.lastQuizResult = lastQuizResult; -// const startTime = new Date(); -// const events = await pullNewEquipmentQuizResultsLocal(equipment); -// const endTime = new Date(); -// const result = { -// quizResults: [] as EventOfType<'EquipmentTrainingQuizResult'>[], -// quizSync: [] as EventOfType<'EquipmentTrainingQuizSync'>[], -// }; -// for (const event of events) { -// if (isEventOfType('EquipmentTrainingQuizResult')(event)) { -// result.quizResults.push(event); -// } else if (isEventOfType('EquipmentTrainingQuizSync')) { -// result.quizSync.push(event); -// } else { -// throw new Error('Unexpected event type'); -// } -// } -// return { -// ...result, -// startTime, -// endTime, -// }; -// }; +type EquipmentQuizResultEvents = { + quizResults: ReadonlyArray>; + quizSync: ReadonlyArray>; + startTime: Date; + endTime: Date; +}; +const pullEquipmentQuizResultsWrapper = async ( + spreadsheetId: O.Option, + lastQuizResult: O.Option = O.none +): Promise => { + const equipment = defaultEquipment(); + equipment.trainingSheetId = spreadsheetId; + equipment.lastQuizResult = lastQuizResult; + const startTime = new Date(); + const events = await pullNewEquipmentQuizResultsLocal(equipment); + const endTime = new Date(); + const result = { + quizResults: [] as EventOfType<'EquipmentTrainingQuizResult'>[], + quizSync: [] as EventOfType<'EquipmentTrainingQuizSync'>[], + }; + for (const event of events) { + if (isEventOfType('EquipmentTrainingQuizResult')(event)) { + result.quizResults.push(event); + } else if (isEventOfType('EquipmentTrainingQuizSync')(event)) { + result.quizSync.push(event); + } else { + throw new Error('Unexpected event type'); + } + } + return { + ...result, + startTime, + endTime, + }; +}; -// const checkQuizSync = (results: EquipmentQuizResultEvents) => { -// expect(results.quizSync).toHaveLength(1); -// expect(results.quizSync[0].recordedAt.getTime()).toBeGreaterThanOrEqual( -// results.startTime.getTime() -// ); -// expect(results.quizSync[0].recordedAt.getTime()).toBeLessThanOrEqual( -// results.endTime.getTime() -// ); -// }; +const checkQuizSync = (results: EquipmentQuizResultEvents) => { + expect(results.quizSync).toHaveLength(1); + expect(results.quizSync[0].recordedAt.getTime()).toBeGreaterThanOrEqual( + results.startTime.getTime() + ); + expect(results.quizSync[0].recordedAt.getTime()).toBeLessThanOrEqual( + results.endTime.getTime() + ); +}; -// describe('Training sheets worker', () => { -// describe('Process results', () => { -// describe('Processes a registered training sheet', () => { -// it('Equipment with no training sheet', async () => { -// const result = await pullEquipmentQuizResultsWrapper(O.none); -// expect(result.quizResults).toHaveLength(0); -// expect(result.quizSync).toHaveLength(0); -// }); +describe('Training sheets worker', () => { + describe('Process results', () => { + describe('Processes a registered training sheet', () => { + it('Equipment with no training sheet', async () => { + const result = await pullEquipmentQuizResultsWrapper(O.none); + expect(result.quizResults).toHaveLength(0); + expect(result.quizSync).toHaveLength(0); + }); -// it('empty sheet produces no events, but does indicate a sync', async () => { -// const result = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.EMPTY.data.spreadsheetId!) -// ); -// expect(result.quizResults).toHaveLength(0); -// checkQuizSync(result); -// }); -// it('metal lathe training sheet', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.METAL_LATHE.data.spreadsheetId!) -// ); -// checkQuizSync(results); -// expect(results.quizResults[0]).toMatchObject< -// Partial> -// >({ -// type: 'EquipmentTrainingQuizResult', -// equipmentId: defaultEquipment().id, -// trainingSheetId: gsheetData.METAL_LATHE.data.spreadsheetId!, -// ...gsheetData.METAL_LATHE.entries[0], -// }); -// }); -// it('training sheet with a summary page', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.LASER_CUTTER.data.spreadsheetId!) -// ); -// checkQuizSync(results); -// const expected: readonly Partial< -// EventOfType<'EquipmentTrainingQuizResult'> -// >[] = gsheetData.LASER_CUTTER.entries.map(e => ({ -// type: 'EquipmentTrainingQuizResult', -// equipmentId: defaultEquipment().id, -// trainingSheetId: gsheetData.LASER_CUTTER.data.spreadsheetId!, -// actor: { -// tag: 'system', -// }, -// ...e, -// })); -// expect(results.quizResults).toHaveLength(expected.length); + it('empty sheet produces no events, but does indicate a sync', async () => { + const result = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.EMPTY.apiResp.spreadsheetId!) + ); + expect(result.quizResults).toHaveLength(0); + checkQuizSync(result); + }); + it('metal lathe training sheet', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.METAL_LATHE.apiResp.spreadsheetId!) + ); + checkQuizSync(results); + expect(results.quizResults[0]).toMatchObject< + Partial> + >({ + type: 'EquipmentTrainingQuizResult', + equipmentId: defaultEquipment().id, + trainingSheetId: gsheetData.METAL_LATHE.apiResp.spreadsheetId!, + ...gsheetData.METAL_LATHE.entries[0], + }); + }); + it('training sheet with a summary page', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.LASER_CUTTER.apiResp.spreadsheetId!) + ); + checkQuizSync(results); + const expected: readonly Partial< + EventOfType<'EquipmentTrainingQuizResult'> + >[] = gsheetData.LASER_CUTTER.entries.map(e => ({ + type: 'EquipmentTrainingQuizResult', + equipmentId: defaultEquipment().id, + trainingSheetId: gsheetData.LASER_CUTTER.apiResp.spreadsheetId!, + actor: { + tag: 'system', + }, + ...e, + })); + expect(results.quizResults).toHaveLength(expected.length); -// for (const [actualEvent, expectedEvent] of RA.zip( -// sortQuizResults(results.quizResults), -// sortQuizResults(expected) -// )) { -// expect(actualEvent).toMatchObject< -// Partial> -// >(expectedEvent); -// } -// }); -// it('training sheet with multiple response pages (different quiz questions)', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!) -// ); -// checkQuizSync(results); -// const expected: readonly Partial< -// EventOfType<'EquipmentTrainingQuizResult'> -// >[] = gsheetData.BAMBU.entries.map(e => ({ -// type: 'EquipmentTrainingQuizResult', -// equipmentId: defaultEquipment().id, -// trainingSheetId: gsheetData.BAMBU.data.spreadsheetId!, -// actor: { -// tag: 'system', -// }, -// ...e, -// })); -// expect(results.quizResults).toHaveLength(expected.length); + for (const [actualEvent, expectedEvent] of RA.zip( + sortQuizResults(results.quizResults), + sortQuizResults(expected) + )) { + expect(actualEvent).toMatchObject< + Partial> + >(expectedEvent); + } + }); + it('training sheet with multiple response pages (different quiz questions)', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!) + ); + checkQuizSync(results); + const expected: readonly Partial< + EventOfType<'EquipmentTrainingQuizResult'> + >[] = gsheetData.BAMBU.entries.map(e => ({ + type: 'EquipmentTrainingQuizResult', + equipmentId: defaultEquipment().id, + trainingSheetId: gsheetData.BAMBU.apiResp.spreadsheetId!, + actor: { + tag: 'system', + }, + ...e, + })); + expect(results.quizResults).toHaveLength(expected.length); -// for (const [actualEvent, expectedEvent] of RA.zip( -// sortQuizResults(results.quizResults), -// sortQuizResults(expected) -// )) { -// expect(actualEvent).toMatchObject< -// Partial> -// >(expectedEvent); -// } -// }); -// it('Only take new rows, date in future', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!), -// O.some(Date.now() as EpochTimestampMilliseconds) -// ); -// checkQuizSync(results); -// expect(results.quizResults).toHaveLength(0); -// }); -// it('Only take new rows, date in far past', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!), -// O.some(0 as EpochTimestampMilliseconds) -// ); -// checkQuizSync(results); -// expect(results.quizResults).toHaveLength( -// gsheetData.BAMBU.entries.length -// ); -// }); + for (const [actualEvent, expectedEvent] of RA.zip( + sortQuizResults(results.quizResults), + sortQuizResults(expected) + )) { + expect(actualEvent).toMatchObject< + Partial> + >(expectedEvent); + } + }); + it('Only take new rows, date in future', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), + O.some(Date.now() as EpochTimestampMilliseconds) + ); + checkQuizSync(results); + expect(results.quizResults).toHaveLength(0); + }); + it('Only take new rows, date in far past', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), + O.some(0 as EpochTimestampMilliseconds) + ); + checkQuizSync(results); + expect(results.quizResults).toHaveLength( + gsheetData.BAMBU.entries.length + ); + }); -// // The quiz results have dates: -// // 1700768963 Thursday, November 23, 2023 7:49:23 PM -// // 1700769348 Thursday, November 23, 2023 7:55:48 PM -// // 1710249052 Tuesday, March 12, 2024 1:10:52 PM -// // 1710249842 Tuesday, March 12, 2024 1:24:02 PM + // The quiz results have dates: + // 1700768963 Thursday, November 23, 2023 7:49:23 PM + // 1700769348 Thursday, November 23, 2023 7:55:48 PM + // 1710249052 Tuesday, March 12, 2024 1:10:52 PM + // 1710249842 Tuesday, March 12, 2024 1:24:02 PM -// it('Only take new rows, exclude 1', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!), -// O.some(1700768963_000 as EpochTimestampMilliseconds) -// ); -// checkQuizSync(results); -// expect(results.quizResults).toHaveLength(3); -// }); + it('Only take new rows, exclude 1', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), + O.some(1700768963_000 as EpochTimestampMilliseconds) + ); + checkQuizSync(results); + expect(results.quizResults).toHaveLength(3); + }); -// it('Only take new rows, exclude 2', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!), -// O.some(1700769348_000 as EpochTimestampMilliseconds) -// ); -// checkQuizSync(results); -// expect(results.quizResults).toHaveLength(2); -// }); + it('Only take new rows, exclude 2', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), + O.some(1700769348_000 as EpochTimestampMilliseconds) + ); + checkQuizSync(results); + expect(results.quizResults).toHaveLength(2); + }); -// it('Only take new rows, exclude 3', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!), -// O.some(1710249052_000 as EpochTimestampMilliseconds) -// ); -// checkQuizSync(results); -// expect(results.quizResults).toHaveLength(1); -// }); + it('Only take new rows, exclude 3', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), + O.some(1710249052_000 as EpochTimestampMilliseconds) + ); + checkQuizSync(results); + expect(results.quizResults).toHaveLength(1); + }); -// it('Only take new rows, exclude all (already have latest)', async () => { -// const results = await pullEquipmentQuizResultsWrapper( -// O.some(gsheetData.BAMBU.data.spreadsheetId!), -// O.some(1710249842_000 as EpochTimestampMilliseconds) -// ); -// checkQuizSync(results); -// expect(results.quizResults).toHaveLength(0); -// }); -// }); -// }); -// }); + it('Only take new rows, exclude all (already have latest)', async () => { + const results = await pullEquipmentQuizResultsWrapper( + O.some(gsheetData.BAMBU.apiResp.spreadsheetId!), + O.some(1710249842_000 as EpochTimestampMilliseconds) + ); + checkQuizSync(results); + expect(results.quizResults).toHaveLength(0); + }); + }); + }); +}); From 7788d8a6a1e183335d3c237b05a23df9d1c4fcb2 Mon Sep 17 00:00:00 2001 From: Paul Lancaster Date: Thu, 26 Sep 2024 00:08:32 +0100 Subject: [PATCH 20/20] All previously passing tests pass again after changes --- tests/init-dependencies/pull-local-google.ts | 27 +++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/init-dependencies/pull-local-google.ts b/tests/init-dependencies/pull-local-google.ts index 768d07e5..e4b0eb79 100644 --- a/tests/init-dependencies/pull-local-google.ts +++ b/tests/init-dependencies/pull-local-google.ts @@ -6,6 +6,7 @@ import { GoogleSpreadsheetDataForSheet, GoogleSpreadsheetInitialMetadata, } from '../../src/init-dependencies/google/pull_sheet_data'; +import {NonEmptyArray} from 'fp-ts/lib/NonEmptyArray'; const localPullGoogleSheetDataMetadata = ( logger: Logger, @@ -16,18 +17,36 @@ const localPullGoogleSheetDataMetadata = ( return sheet ? TE.right(sheet) : TE.left('Spreadsheet not found'); }; +const clone = (v: T): T => JSON.parse(JSON.stringify(v)) as T; + const localPullGoogleSheetData = ( logger: Logger, trainingSheetId: string, sheetName: string, - _rowStart: number, - _rowEnd: number, + rowStart: number, + rowEnd: number, _columnStartIndex: number, _columnEndIndex: number ): TE.TaskEither => { logger.debug(`Pulling local google sheet '${trainingSheetId}'`); - const sheet = gsheetData.TRAINING_SHEETS[trainingSheetId].sheets[sheetName]; - return sheet ? TE.right(sheet) : TE.left('Sheet not found'); + const sheet = clone( + gsheetData.TRAINING_SHEETS[trainingSheetId].sheets[sheetName] + ); + if (sheet) { + if (rowStart > sheet.sheets[0].data[0].rowData.length) { + return TE.left('Sheet no more data'); + } + sheet.sheets[0].data[0].rowData = sheet.sheets[0].data[0].rowData.slice( + rowStart - 1, // 1 indexed. + rowEnd + ) as NonEmptyArray<{ + values: NonEmptyArray<{ + formattedValue: string; + }>; + }>; + return TE.right(sheet); + } + return TE.left('Sheet not found'); }; export const localGoogleHelpers: GoogleHelpers = {