Add redirect to Circulars archive tarballs in static bucket

API Gateways have a limit on response size of 10 MB, so we cannot
rely on the API Gateway to server the Circulars archive.

__tests__/env.ts

@@ -15,6 +15,14 @@ afterEach(() => {
   process.env = { ...oldEnv }
 })
 
+beforeEach(() => {
+  setEnv('ORIGIN', 'http://example.gov')
+  setEnv('ARC_STATIC_BUCKET', 'example-bucket')
+  setEnv('AWS_REGION', 'example-region')
+  setEnv('AWS_DEFAULT_REGION', 'example-default-region')
+  setEnv('SESSION_SECRET', 'example-secret')
+})
+
 function importEnv() {
   let result: typeof EnvModule
   jest.isolateModules(() => {
@@ -32,11 +40,6 @@ function setEnv(key: string, value: string | undefined) {
 describe('features', () => {
   const key = 'GCN_FEATURES'
 
-  // ORIGIN must be defined
-  beforeEach(() => {
-    setEnv('ORIGIN', 'http://example.gov')
-  })
-
   test.each([undefined, '', ',', ',,,'])(
     'environment variable is %p',
     async (value) => {
@@ -74,10 +77,6 @@ describe('features', () => {
 describe('getEnvOrDie', () => {
   const key = 'FOO'
 
-  beforeEach(() => {
-    setEnv('ORIGIN', 'http://example.gov')
-  })
-
   test('returns the value if the environment variable exists', () => {
     setEnv(key, 'BAR')
     const { getEnvOrDie } = importEnv()
@@ -97,36 +96,29 @@ describe('getEnvOrDie', () => {
 })
 
 describe('getOrigin', () => {
-  const key = 'ORIGIN'
-
   test('gets sandbox origin when ORIGIN is not defined', () => {
+    setEnv('ORIGIN', undefined)
     setEnv('ARC_SANDBOX', JSON.stringify({ ports: { http: 1234 } }))
     const { origin } = importEnv()
     expect(origin).toBe('http://localhost:1234')
   })
 
   test('gets env.ORIGIN when ORIGIN is defined', () => {
-    setEnv(key, 'https://gcn.nasa.gov')
+    setEnv('ORIGIN', 'https://gcn.nasa.gov')
     const { origin } = importEnv()
     expect(origin).toBe('https://gcn.nasa.gov')
   })
 })
 
 describe('getSessionSecret', () => {
-  const key = 'SESSION_SECRET'
-
-  beforeAll(() => {
-    setEnv('ARC_SANDBOX', JSON.stringify({ ports: { http: 1234 } }))
-  })
-
   test('gets sandbox value when SESSION_SECRET is not defined', () => {
-    setEnv(key, undefined)
+    setEnv('SESSION_SECRET', undefined)
     const { sessionSecret } = importEnv()
     expect(sessionSecret).toBe('fallback-secret-for-dev')
   })
 
   test('gets env.SESSION_SECRET when SESSION_SECRET is defined', () => {
-    setEnv(key, 'xyzzy')
+    setEnv('SESSION_SECRET', 'xyzzy')
     const { sessionSecret } = importEnv()
     expect(sessionSecret).toBe('xyzzy')
   })
@@ -148,14 +140,28 @@ describe('getHostname', () => {
   })
 })
 
-describe('getEnvOrDieInProduction', () => {
-  const key = 'FOO'
+describe('getRegion', () => {
+  test('returns the value of AWS_REGION if it is defined', () => {
+    const { region } = importEnv()
+    expect(region).toBe('example-region')
+  })
 
-  // ORIGIN must be defined
-  beforeEach(() => {
-    setEnv('ORIGIN', 'http://example.gov')
-    setEnv('SESSION_SECRET', 'foobar')
+  test('returns the value of AWS_DEFAULT_REGION if AWS_REGION is undefined', () => {
+    setEnv('AWS_REGION', undefined)
+    const { region } = importEnv()
+    expect(region).toBe('example-default-region')
   })
+})
+
+describe('getStaticBucket', () => {
+  test('returns the value of ARC_STATIC_BUCKET', () => {
+    const { staticBucket } = importEnv()
+    expect(staticBucket).toBe('example-bucket')
+  })
+})
+
+describe('getEnvOrDieInProduction', () => {
+  const key = 'FOO'
 
   test('returns undefined if the variable does not exist', () => {
     setEnv('TEST', undefined)

app/lib/env.server.ts

@@ -50,10 +50,23 @@ function getSessionSecret() {
   return getEnvOrDieInProduction('SESSION_SECRET') || 'fallback-secret-for-dev'
 }
 
+function getStaticBucket() {
+  return getEnvOrDie('ARC_STATIC_BUCKET')
+}
+
+function getRegion() {
+  const result = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION
+  if (!result)
+    throw new Error('Either AWS_REGION or AWS_DEFAULT_REGION must be defined')
+  return result
+}
+
 export const origin = /* @__PURE__ */ getOrigin()
 export const hostname = /* @__PURE__ */ getHostname()
 export const features = /* @__PURE__ */ getFeatures()
 export const sessionSecret = /* @__PURE__ */ getSessionSecret()
+export const staticBucket = /* @__PURE__ */ getStaticBucket()
+export const region = /* @__PURE__ */ getRegion()
 
 /**
  * Return true if the given feature flag is enabled.

app/routes/circulars._index.tsx

@@ -211,7 +211,6 @@ function DownloadModal() {
           <p id="modal-1-description">
             This is a download of the entire GCN Circulars database.
           </p>
-          <p>It may take a moment.</p>
           <p>Select a file format to begin download.</p>
         </div>
         <ModalFooter>

app/routes/circulars.archive.$suffix[.]tar.ts

@@ -0,0 +1,21 @@
+/*!
+ * Copyright © 2023 United States Government as represented by the
+ * Administrator of the National Aeronautics and Space Administration.
+ * All Rights Reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { type LoaderArgs, redirect } from '@remix-run/node'
+import invariant from 'tiny-invariant'
+
+import { region, staticBucket } from '~/lib/env.server'
+import { getBucketKey } from '~/scheduled/circulars/uploadTar'
+
+function getBucketUrl(region: string, bucket: string, key: string) {
+  return `https://s3.${region}.amazonaws.com/${bucket}/key`
+}
+
+export async function loader({ params: { suffix } }: LoaderArgs) {
+  invariant(suffix)
+  return redirect(getBucketUrl(region, staticBucket, getBucketKey(suffix)))
+}

app/scheduled/circulars/uploadTar.ts

@@ -7,26 +7,31 @@
  */
 import { S3Client } from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
+import { basename } from 'node:path'
 import { PassThrough } from 'node:stream'
 import type { Pack } from 'tar-stream'
 import { pack as tarPack } from 'tar-stream'
 
 import type { CircularAction } from './circularAction'
-import { getEnvOrDie } from '~/lib/env.server'
+import { staticBucket as Bucket } from '~/lib/env.server'
 import type { Circular } from '~/routes/circulars/circulars.lib'
 import {
   formatCircularJson,
   formatCircularText,
 } from '~/routes/circulars/circulars.lib'
 
 const s3 = new S3Client({})
-const Bucket = getEnvOrDie('ARC_STATIC_BUCKET')
+
+export function getBucketKey(suffix: string) {
+  return `circulars/archive.${suffix}.tar`
+}
 
 function createUploadAction(
   suffix: string,
   formatter: (circular: Circular) => string
 ): CircularAction<{ pack: Pack; promise: Promise<any> }> {
-  const baseFilename = `circulars-archive.${suffix}`
+  const Key = getBucketKey(suffix)
+  const tarDir = basename(Key, '.tar')
 
   return {
     initialize() {
@@ -35,13 +40,13 @@ function createUploadAction(
       pack.pipe(Body)
       const promise = new Upload({
         client: s3,
-        params: { Body, Bucket, Key: `${baseFilename}.tar` },
+        params: { Body, Bucket, Key },
       }).done()
       return { pack, promise }
     },
     action(circulars, { pack }) {
       for (const circular of circulars) {
-        const name = `${baseFilename}/${circular.circularId}.${suffix}`
+        const name = `${tarDir}/${circular.circularId}.${suffix}`
         pack.entry({ name }, formatter(circular)).end()
       }
     },