From 9322a6b40a9c205b9b47d822eb3202ce7f940b8f Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Fri, 8 Sep 2023 17:32:55 -0400 Subject: [PATCH] Refactor scheduled circulars Lambda I'm preparing to add yet a third action to generate sitemap files. To keep everything nice and neat, here are a few refactorings: - Collect all of the actions into the actions/ directory. - Factor out repeated S3 bucket config details into a common file. - Collect the mapCirculars function and the CircularsAction type into one file to keep the Lambda entry point itself clean as we add more imports for the actions. - Rename the mapCirculars function to forAllCirculars, because its behavior more closely resembles the .forEach() array method than the .map() method. --- .../circulars.archive[.]json[.]tar[.]gz.ts | 2 +- .../circulars.archive[.]txt[.]tar[.]gz.ts | 2 +- app/scheduled/circulars/actions.ts | 41 +++++++++++++++++++ .../circulars/{ => actions}/stats.ts | 8 ++-- .../{uploadTar.ts => actions/tar.ts} | 7 ++-- app/scheduled/circulars/circularAction.ts | 14 ------- app/scheduled/circulars/index.ts | 36 ++-------------- app/scheduled/circulars/storage.ts | 11 +++++ 8 files changed, 65 insertions(+), 56 deletions(-) create mode 100644 app/scheduled/circulars/actions.ts rename app/scheduled/circulars/{ => actions}/stats.ts (79%) rename app/scheduled/circulars/{uploadTar.ts => actions/tar.ts} (91%) delete mode 100644 app/scheduled/circulars/circularAction.ts create mode 100644 app/scheduled/circulars/storage.ts diff --git a/app/routes/circulars.archive[.]json[.]tar[.]gz.ts b/app/routes/circulars.archive[.]json[.]tar[.]gz.ts index 984b6404b..e1289c954 100644 --- a/app/routes/circulars.archive[.]json[.]tar[.]gz.ts +++ b/app/routes/circulars.archive[.]json[.]tar[.]gz.ts @@ -8,7 +8,7 @@ import { redirect } from '@remix-run/node' import { publicStaticShortTermCacheControlHeaders } from '~/lib/headers.server' -import { getArchiveURL } from '~/scheduled/circulars/uploadTar' +import { getArchiveURL } from '~/scheduled/circulars/actions/tar' export function loader() { return redirect(getArchiveURL('json'), { diff --git a/app/routes/circulars.archive[.]txt[.]tar[.]gz.ts b/app/routes/circulars.archive[.]txt[.]tar[.]gz.ts index 5afd2a26d..8f62a8010 100644 --- a/app/routes/circulars.archive[.]txt[.]tar[.]gz.ts +++ b/app/routes/circulars.archive[.]txt[.]tar[.]gz.ts @@ -8,7 +8,7 @@ import { redirect } from '@remix-run/node' import { publicStaticShortTermCacheControlHeaders } from '~/lib/headers.server' -import { getArchiveURL } from '~/scheduled/circulars/uploadTar' +import { getArchiveURL } from '~/scheduled/circulars/actions/tar' export function loader() { return redirect(getArchiveURL('txt'), { diff --git a/app/scheduled/circulars/actions.ts b/app/scheduled/circulars/actions.ts new file mode 100644 index 000000000..060539998 --- /dev/null +++ b/app/scheduled/circulars/actions.ts @@ -0,0 +1,41 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import { tables } from '@architect/functions' +import type { DynamoDBDocument } from '@aws-sdk/lib-dynamodb' +import { paginateScan } from '@aws-sdk/lib-dynamodb' + +import type { Circular } from '~/routes/circulars/circulars.lib' + +export interface CircularAction { + initialize: () => T | Promise + action: (circulars: Circular[], context: T) => void | Promise + finalize: (context: T) => void | Promise +} + +export async function forAllCirculars(...actions: CircularAction[]) { + const contexts = await Promise.all( + actions.map((action) => action.initialize()) + ) + for await (const circulars of getAllRecords()) { + await Promise.all( + actions.map(({ action }, i) => action(circulars, contexts[i])) + ) + } + await Promise.all(actions.map(({ finalize }, i) => finalize(contexts[i]))) +} + +async function* getAllRecords() { + const db = await tables() + const client = db._doc as unknown as DynamoDBDocument + const TableName = db.name('circulars') + const pages = paginateScan({ client }, { TableName }) + + for await (const page of pages) { + yield page.Items as Circular[] + } +} diff --git a/app/scheduled/circulars/stats.ts b/app/scheduled/circulars/actions/stats.ts similarity index 79% rename from app/scheduled/circulars/stats.ts rename to app/scheduled/circulars/actions/stats.ts index 6f6c58d44..1c20eb39f 100644 --- a/app/scheduled/circulars/stats.ts +++ b/app/scheduled/circulars/actions/stats.ts @@ -5,13 +5,13 @@ * * SPDX-License-Identifier: Apache-2.0 */ -import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3' +import { PutObjectCommand } from '@aws-sdk/client-s3' -import type { CircularAction } from './circularAction' +import type { CircularAction } from '../actions' +import { keyPrefix, s3 } from '../storage' import { staticBucket as Bucket } from '~/lib/env.server' -const s3 = new S3Client({}) -const Key = 'generated/circulars/stats.json' +const Key = `${keyPrefix}/stats.json` export const statsAction: CircularAction> = { initialize() { diff --git a/app/scheduled/circulars/uploadTar.ts b/app/scheduled/circulars/actions/tar.ts similarity index 91% rename from app/scheduled/circulars/uploadTar.ts rename to app/scheduled/circulars/actions/tar.ts index 319134d94..fe02b03f2 100644 --- a/app/scheduled/circulars/uploadTar.ts +++ b/app/scheduled/circulars/actions/tar.ts @@ -5,7 +5,6 @@ * * SPDX-License-Identifier: Apache-2.0 */ -import { S3Client } from '@aws-sdk/client-s3' import { Upload } from '@aws-sdk/lib-storage' import { basename } from 'node:path' import { PassThrough } from 'node:stream' @@ -13,7 +12,8 @@ import { createGzip } from 'node:zlib' import type { Pack } from 'tar-stream' import { pack as tarPack } from 'tar-stream' -import type { CircularAction } from './circularAction' +import type { CircularAction } from '../actions' +import { keyPrefix, s3 } from '../storage' import { staticBucket as Bucket, region } from '~/lib/env.server' import type { Circular } from '~/routes/circulars/circulars.lib' import { @@ -21,11 +21,10 @@ import { formatCircularText, } from '~/routes/circulars/circulars.lib' -const s3 = new S3Client({}) const archiveSuffix = '.tar.gz' function getBucketKey(suffix: string) { - return `generated/circulars/archive.${suffix}${archiveSuffix}` + return `${keyPrefix}/archive.${suffix}${archiveSuffix}` } function getBucketUrl(region: string, bucket: string, key: string) { diff --git a/app/scheduled/circulars/circularAction.ts b/app/scheduled/circulars/circularAction.ts deleted file mode 100644 index e8c83c28a..000000000 --- a/app/scheduled/circulars/circularAction.ts +++ /dev/null @@ -1,14 +0,0 @@ -/*! - * Copyright © 2023 United States Government as represented by the - * Administrator of the National Aeronautics and Space Administration. - * All Rights Reserved. - * - * SPDX-License-Identifier: Apache-2.0 - */ -import type { Circular } from '~/routes/circulars/circulars.lib' - -export interface CircularAction { - initialize: () => T | Promise - action: (circulars: Circular[], context: T) => void | Promise - finalize: (context: T) => void | Promise -} diff --git a/app/scheduled/circulars/index.ts b/app/scheduled/circulars/index.ts index 91c9f671f..8d66a8c31 100644 --- a/app/scheduled/circulars/index.ts +++ b/app/scheduled/circulars/index.ts @@ -5,40 +5,12 @@ * * SPDX-License-Identifier: Apache-2.0 */ -import { tables } from '@architect/functions' -import type { DynamoDBDocument } from '@aws-sdk/lib-dynamodb' -import { paginateScan } from '@aws-sdk/lib-dynamodb' - -import type { CircularAction } from './circularAction' -import { statsAction } from './stats' -import { jsonUploadAction, txtUploadAction } from './uploadTar' -import { type Circular } from '~/routes/circulars/circulars.lib' - -async function mapCirculars(...actions: CircularAction[]) { - const contexts = await Promise.all( - actions.map((action) => action.initialize()) - ) - for await (const circulars of getAllRecords()) { - await Promise.all( - actions.map(({ action }, i) => action(circulars, contexts[i])) - ) - } - await Promise.all(actions.map(({ finalize }, i) => finalize(contexts[i]))) -} - -async function* getAllRecords() { - const db = await tables() - const client = db._doc as unknown as DynamoDBDocument - const TableName = db.name('circulars') - const pages = paginateScan({ client }, { TableName }) - - for await (const page of pages) { - yield page.Items as Circular[] - } -} +import { forAllCirculars } from './actions' +import { statsAction } from './actions/stats' +import { jsonUploadAction, txtUploadAction } from './actions/tar' // FIXME: must use module.exports here for OpenTelemetry shim to work correctly. // See https://dev.to/heymarkkop/how-to-solve-cannot-redefine-property-handler-on-aws-lambda-3j67 module.exports.handler = async () => { - await mapCirculars(jsonUploadAction, txtUploadAction, statsAction) + await forAllCirculars(jsonUploadAction, txtUploadAction, statsAction) } diff --git a/app/scheduled/circulars/storage.ts b/app/scheduled/circulars/storage.ts new file mode 100644 index 000000000..7a4118d8b --- /dev/null +++ b/app/scheduled/circulars/storage.ts @@ -0,0 +1,11 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ +import { S3Client } from '@aws-sdk/client-s3' + +export const s3 = new S3Client({}) +export const keyPrefix = 'generated/circulars'