Skip to content

Commit

Permalink
Validate google sheet metadata straight from api
Browse files Browse the repository at this point in the history
  • Loading branch information
@Lan2u
Lan2u committed Sep 25, 2024
1 parent e7c3fc4 commit e1b0045
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 92 deletions.
78 changes: 62 additions & 16 deletions src/init-dependencies/google/pull_sheet_data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,51 @@ import * as TE from 'fp-ts/TaskEither';
import * as t from 'io-ts';
import * as tt from 'io-ts-types';
import * as E from 'fp-ts/Either';
import {Failure} from '../../types';

import {pipe} from 'fp-ts/lib/function';
import {sheets, sheets_v4} from '@googleapis/sheets';
import {sheets} from '@googleapis/sheets';
import {GoogleAuth} from 'google-auth-library';
import {columnIndexToLetter} from '../../training-sheets/extract-metadata';
import {formatValidationErrors} from 'io-ts-reporters';
import {DateTime} from 'luxon';

export type GoogleSpreadsheetInitialMetadata = sheets_v4.Schema$Spreadsheet & {
readonly GoogleSpreadsheetInitialMetadata: unique symbol;
};
const DEFAULT_TIMEZONE = 'Europe/London';

// Contains only a single sheet
// Not all the google form sheets are actually in Europe/London.
// Issue first noticed because CI is in a different zone (UTC) than local test machine (BST).
export const GoogleTimezone = tt.withValidate(t.string, (input, context) =>
pipe(
t.string.validate(input, context),
E.chain(timezoneRaw =>
DateTime.local().setZone(timezoneRaw).isValid
? E.right(timezoneRaw)
: E.left([])
),
E.orElse(() => t.success(DEFAULT_TIMEZONE))
)
);

export const GoogleSpreadsheetInitialMetadata = t.strict({
properties: t.strict({
timeZone: GoogleTimezone,
}),
sheets: t.array(
t.strict({
properties: t.strict({
title: t.string,
gridProperties: t.strict({
rowCount: t.number,
}),
}),
})
),
});
export type GoogleSpreadsheetInitialMetadata = t.TypeOf<
typeof GoogleSpreadsheetInitialMetadata
>;

// Contains only a single sheet. Structure is a little verbose to match the part of the
// google api it is taken from.
export const GoogleSpreadsheetDataForSheet = t.strict({
sheets: tt.nonEmptyArray(
// Array always has length = 1 because this is data for a single sheet.
Expand Down Expand Up @@ -62,7 +94,19 @@ export const pullGoogleSheetDataMetadata =
return `Failed to get training spreadsheet metadata ${trainingSheetId}`;
}
),
TE.map(resp => resp.data as GoogleSpreadsheetInitialMetadata)
TE.map(resp => resp.data),
TE.chain(data =>
TE.fromEither(
pipe(
data,
GoogleSpreadsheetInitialMetadata.decode,
E.mapLeft(
e =>
`Failed to get google spreadsheet metadata from API response: ${formatValidationErrors(e).join(',')}`
)
)
)
)
);

export const pullGoogleSheetData =
Expand All @@ -75,7 +119,7 @@ export const pullGoogleSheetData =
rowEnd: number,
columnStartIndex: number, // 0 indexed, converted to a letter.
columnEndIndex: number
): TE.TaskEither<Failure, GoogleSpreadsheetDataForSheet> =>
): TE.TaskEither<string, GoogleSpreadsheetDataForSheet> =>
pipe(
TE.tryCatch(
() => {
Expand All @@ -99,11 +143,12 @@ export const pullGoogleSheetData =
});
},
reason => {
logger.error(reason, 'Failed to get spreadsheet');
return {
// Expand failure reasons.
message: `Failed to get training spreadsheet ${trainingSheetId}`,
};
logger.error(
reason,
'Failed to get training spreadsheet %s',
trainingSheetId
);
return `Failed to get training spreadsheet ${trainingSheetId}`;
}
),
TE.map(resp => resp.data),
Expand All @@ -112,9 +157,10 @@ export const pullGoogleSheetData =
pipe(
data,
GoogleSpreadsheetDataForSheet.decode,
E.mapLeft(e => ({
message: `Failed to get all required google spreadsheet data from API response: ${formatValidationErrors(e).join(',')}`,
}))
E.mapLeft(
e =>
`Failed to get all required google spreadsheet data from API response: ${formatValidationErrors(e).join(',')}`
)
)
)
)
Expand Down
19 changes: 2 additions & 17 deletions src/read-models/shared-state/async-apply-external-event-sources.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import {constructEvent} from '../../types/domain-event';
import {GoogleHelpers} from '../../init-dependencies/google/pull_sheet_data';
import {
extractGoogleSheetMetadata,
extractInitialGoogleSheetMetadata,
GoogleSheetMetadata,
MAX_COLUMN_INDEX,
} from '../../training-sheets/extract-metadata';
Expand Down Expand Up @@ -100,25 +99,11 @@ export const pullNewEquipmentQuizResults = async (
equipment.lastQuizResult
);

const initialRaw = await googleHelpers.pullGoogleSheetDataMetadata(
const initialMeta = await googleHelpers.pullGoogleSheetDataMetadata(
logger,
trainingSheetId
)();
if (E.isLeft(initialRaw)) {
logger.warn(initialRaw.left);
return;
}

const initialMeta = extractInitialGoogleSheetMetadata(
logger,
initialRaw.right
);

if (E.isLeft(initialMeta)) {
logger.warn(
'Failed to get google sheet metadata for training sheet %s, skipping',
trainingSheetId
);
logger.warn(initialMeta.left);
return;
}
Expand Down Expand Up @@ -165,7 +150,7 @@ export const pullNewEquipmentQuizResults = async (
equipment,
trainingSheetId,
sheet,
initialMeta.right.timezone,
initialMeta.timezone,
updateState
);
}
Expand Down
62 changes: 7 additions & 55 deletions src/training-sheets/extract-metadata.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@ import {formatValidationErrors} from 'io-ts-reporters';
const EMAIL_COLUMN_NAMES = ['email address', 'email'];

type GoogleSheetName = string;
// What we can get from an initial call to google sheets without any rows.
export interface GoogleSheetMetadataInital {
name: GoogleSheetName;
rowCount: number;
}
export interface GoogleSheetsMetadataInital {
sheets: GoogleSheetMetadataInital[];
timezone: string;
}

type ColumnLetter = string;
type ColumnIndex = number; // 0-indexed.
Expand All @@ -46,43 +37,16 @@ export const MAX_COLUMN_INDEX = 25;
export const columnIndexToLetter = (index: ColumnIndex): ColumnLetter =>
'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.charAt(index);

const DEFAULT_TIMEZONE = 'Europe/London';

const SheetProperties = t.strict({
properties: t.strict({
timeZone: withDefaultIfEmpty(t.string, DEFAULT_TIMEZONE),
}),
sheets: t.array(
t.strict({
properties: t.strict({
title: t.string,
gridProperties: t.strict({
rowCount: t.number,
}),
}),
})
),
});

export const extractInitialGoogleSheetMetadata = (
logger: Logger,
spreadsheet: GoogleSpreadsheetInitialMetadata
): E.Either<string, GoogleSheetsMetadataInital> =>
pipe(
spreadsheet,
SheetProperties.decode,
E.mapLeft(e => {
logger.warn(formatValidationErrors(e));
return 'Failed to extract initial google sheet metadata';
}),
E.map(properties => ({
sheets: properties.sheets.map(sheet => ({
name: sheet.properties.title,
rowCount: sheet.properties.gridProperties.rowCount,
})),
timezone: validateTimezone(logger, properties.properties.timeZone),
}))
);
): GoogleSheetsMetadataInital => ({
sheets: spreadsheet.sheets.map(sheet => ({
name: sheet.properties.title,
rowCount: sheet.properties.gridProperties.rowCount,
})),
timezone: spreadsheet.properties.timeZone,
});

export const extractGoogleSheetMetadata =
(logger: Logger) =>
Expand Down Expand Up @@ -132,15 +96,3 @@ export const extractGoogleSheetMetadata =
},
});
};

const validateTimezone = (logger: Logger, timezone: string): string => {
if (!DateTime.local().setZone(timezone).isValid) {
// Not all the google form sheets are actually in Europe/London.
// Issue first noticed because CI is in a different zone (UTC) than local test machine (BST).
logger.info(
`Unable to determine timezone for google sheet, timezone is invalid: '${timezone}' - defaulting to Europe/London`
);
timezone = DEFAULT_TIMEZONE;
}
return timezone;
};
5 changes: 1 addition & 4 deletions src/training-sheets/google.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,7 @@ import {v4} from 'uuid';
import {UUID} from 'io-ts-types';
import {DateTime} from 'luxon';
import {EpochTimestampMilliseconds} from '../read-models/shared-state/return-types';
import {
GoogleSheetMetadata,
GoogleSheetMetadataInital,
} from './extract-metadata';
import {GoogleSheetMetadata} from './extract-metadata';
import {GoogleSpreadsheetDataForSheet} from '../init-dependencies/google/pull_sheet_data';

// Bounds to prevent clearly broken parsing.
Expand Down

0 comments on commit e1b0045

Please sign in to comment.