Store most recent quiz row timestamp + sync to minimise extra process…

…ing & for display so I can quickly see stuff is working
Makespace · Sep 17, 2024 · b666989 · b666989
1 parent 66bb749
commit b666989
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 49 deletions.
diff --git a/src/read-models/shared-state/async-apply-external-event-sources.ts b/src/read-models/shared-state/async-apply-external-event-sources.ts
@@ -11,52 +11,53 @@ import {sheets_v4} from '@googleapis/sheets';
 import {Equipment} from './return-types';
 import {QzEvent} from '../../types/qz-event';
 import {extractGoogleSheetData} from '../../training-sheets/google';
-import {isNewQuizEvents} from './is-new-quiz-events';
 
 const GOOGLE_UPDATE_INTERVAL_MS = 5 * 60 * 1000;
 
 export type PullSheetData = (
   logger: Logger,
-  trainingSheetId: string
+  trainingSheetId: string,
+  rowsSince: O.Option<Date>
 ) => TE.TaskEither<Failure, sheets_v4.Schema$Spreadsheet>;
 
-const pullNewEquipmentQuizResults =
-  (logger: Logger, pullGoogleSheetData: PullSheetData) =>
-  (equipment: Equipment): T.Task<ReadonlyArray<QzEvent>> => {
-    if (O.isNone(equipment.trainingSheetId)) {
-      logger.warn(
-        'No training sheet registered for equipment %s, skipping training data ingestion',
-        equipment.name
-      );
-      // eslint-disable-next-line @typescript-eslint/require-await
-      return async () => [] as ReadonlyArray<QzEvent>;
-    }
-    const trainingSheetId = equipment.trainingSheetId.value;
-    logger.info(
-      `Scanning training sheet ${trainingSheetId}. Pulling google sheet data...`
-    );
-    return pipe(
-      pullGoogleSheetData(logger, trainingSheetId),
-      TE.map(
-        extractGoogleSheetData(
-          logger.child({trainingSheetId: trainingSheetId}),
-          trainingSheetId
-        )
-      ),
-      TE.map(RA.flatten),
-      // eslint-disable-next-line @typescript-eslint/require-await
-      TE.getOrElse(err => async () => {
-        logger.error(
-          'Failed to receive data from google sheets for equipment %s training sheet %o: %s',
-          equipment.name,
-          equipment.trainingSheetId,
-          err.message
-        );
-        return [] as ReadonlyArray<QzEvent>;
-      }),
-      T.map(RA.filter(isNewQuizEvents(equipment)))
+const pullNewEquipmentQuizResults = (
+  logger: Logger,
+  pullGoogleSheetData: PullSheetData,
+  equipment: Equipment
+): T.Task<ReadonlyArray<QzEvent>> => {
+  if (O.isNone(equipment.trainingSheetId)) {
+    logger.warn(
+      'No training sheet registered for equipment %s, skipping training data ingestion',
+      equipment.name
     );
-  };
+    // eslint-disable-next-line @typescript-eslint/require-await
+    return async () => [] as ReadonlyArray<QzEvent>;
+  }
+  const trainingSheetId = equipment.trainingSheetId.value;
+  logger.info(
+    `Scanning training sheet ${trainingSheetId}. Pulling google sheet data...`
+  );
+  return pipe(
+    pullGoogleSheetData(logger, trainingSheetId, equipment.lastQuizResult),
+    TE.map(
+      extractGoogleSheetData(
+        logger.child({trainingSheetId: trainingSheetId}),
+        trainingSheetId
+      )
+    ),
+    TE.map(RA.flatten),
+    // eslint-disable-next-line @typescript-eslint/require-await
+    TE.getOrElse(err => async () => {
+      logger.error(
+        'Failed to receive data from google sheets for equipment %s training sheet %o: %s',
+        equipment.name,
+        equipment.trainingSheetId,
+        err.message
+      );
+      return [] as ReadonlyArray<QzEvent>;
+    })
+  );
+};
 
 export const asyncApplyExternalEventSources = (
   logger: Logger,
@@ -65,7 +66,6 @@ export const asyncApplyExternalEventSources = (
   updateState: (event: DomainEvent) => void
 ) => {
   let lastGoogleUpdate: number | null;
-
   return () => async () => {
     logger.info('Applying external event sources...');
     if (
@@ -77,8 +77,9 @@ export const asyncApplyExternalEventSources = (
         RA.map(updateState)(
           await pullNewEquipmentQuizResults(
             logger,
-            pullGoogleSheetData
-          )(equipment)()
+            pullGoogleSheetData,
+            equipment
+          )()
         );
       }
     }

diff --git a/src/read-models/shared-state/is-new-quiz-events.ts b/src/read-models/shared-state/is-new-quiz-events.ts
diff --git a/src/read-models/shared-state/return-types.ts b/src/read-models/shared-state/return-types.ts
@@ -39,6 +39,12 @@ export type Equipment = {
   orphanedPassedQuizes: ReadonlyArray<OrphanedPassedQuiz>;
   failedQuizAttempts: ReadonlyArray<FailedQuizAttempt>;
   trainingSheetId: O.Option<string>;
+
+  // Uses the actual spreadsheet timestamp rather than our local timestamp which could be
+  // different due to clock drift or eventual consistency issues on the google side.
+  lastQuizResult: O.Option<Date>;
+  // Uses local timestamp.
+  lastQuizSync: O.Option<Date>;
 };
 
 type TrainedOn = {