From c968196c2f94206fe0343bb07f89987bff8505fb Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 18 Jul 2022 13:14:46 +0200 Subject: [PATCH 1/7] fix: support hourly CDP uploads --- types/response.ts | 5 +++++ util/constants.ts | 7 ++++++- util/ui.ts | 48 ++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/types/response.ts b/types/response.ts index 5e1d5c6..5c8b385 100644 --- a/types/response.ts +++ b/types/response.ts @@ -14,3 +14,8 @@ export type CDPFile = { } export type ConvertedMD = { markdown: string } + +export type FileNameParserResponse = { + timestamp: number + dateFormatterOptions: Intl.DateTimeFormatOptions +} | null diff --git a/util/constants.ts b/util/constants.ts index 038d34d..3d443df 100644 --- a/util/constants.ts +++ b/util/constants.ts @@ -18,9 +18,14 @@ export const USER_TYPES: { label: string; url: string; value: USER_TYPE }[] = [ { label: 'OTP Users', url: OTP_USER_URL, value: 'otp' } ] -export const dateFormatterOptions: Intl.DateTimeFormatOptions = { +export const dateFormatterOptionsForDate: Intl.DateTimeFormatOptions = { day: 'numeric', month: 'long', timeZone: 'UTC', year: 'numeric' } + +export const dateFormatterOptionsForTime: Intl.DateTimeFormatOptions = { + dateStyle: 'long', + timeStyle: 'short' +} diff --git a/util/ui.ts b/util/ui.ts index f355ede..6ff72f5 100644 --- a/util/ui.ts +++ b/util/ui.ts @@ -1,6 +1,11 @@ import { Children, isValidElement, cloneElement } from 'react' -import { dateFormatterOptions, USER_TYPES } from '../util/constants' +import { FileNameParserResponse } from '../types/response' +import { + dateFormatterOptionsForDate, + dateFormatterOptionsForTime, + USER_TYPES +} from '../util/constants' import type { USER_TYPE } from './constants' @@ -94,9 +99,42 @@ export const getDateFromCDPFileName = (filename: string): string => { const date = filename.split('-anon-trip-data')?.[0].split('/')?.[1] if (!date) return filename - const parsedDate = Date.parse(date) - if (!parsedDate) return filename + const parsedDate = + getDateOnlyFromCDPFileName(date) || getDateAndTimeFromCDPFileName(date) + if (!parsedDate || !parsedDate.timestamp) return filename - const dateFormatter = new Intl.DateTimeFormat('en-US', dateFormatterOptions) - return dateFormatter.format(new Date(parsedDate)) + const dateFormatter = new Intl.DateTimeFormat( + 'en-US', + parsedDate.dateFormatterOptions + ) + return dateFormatter.format(new Date(parsedDate.timestamp)) +} +/** + * Some CDP file names are only a date. These are easy to parse + */ +const getDateOnlyFromCDPFileName = (date: string): FileNameParserResponse => { + const timestamp = Date.parse(date) + if (!timestamp) return null + + return { dateFormatterOptions: dateFormatterOptionsForDate, timestamp } +} + +/** + * Some CDP zip file names include the hour in a very non-standard format. This method + * attempts to process this file name + */ +const getDateAndTimeFromCDPFileName = ( + date: string +): FileNameParserResponse => { + const datePortion = date.split('-') + if (!datePortion) return null + + // The last number after the last dash is the hour + const timePortion = datePortion.pop() + if (!timePortion) return null + + // :00 helps the parser know we are talking about a time + const timestamp = Date.parse(`${datePortion.join('-')} ${timePortion}:00`) + + return { dateFormatterOptions: dateFormatterOptionsForTime, timestamp } } From d015deb3edeedcc26c9a7f41f127e1bf9ee8b39e Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 18 Jul 2022 13:19:33 +0200 Subject: [PATCH 2/7] chore(e2e tests): update e2e tests for new middleware file naming --- __tests__/e2e/e2e.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/__tests__/e2e/e2e.ts b/__tests__/e2e/e2e.ts index b0919b0..6db8534 100644 --- a/__tests__/e2e/e2e.ts +++ b/__tests__/e2e/e2e.ts @@ -1,7 +1,7 @@ import 'expect-puppeteer' import { server } from '../../jest-puppeteer.config' -import { dateFormatterOptions } from '../../util/constants' +import { dateFormatterOptionsForDate } from '../../util/constants' import { waitForDownload } from '../util/waitForDownload' jest.setTimeout(50000) @@ -185,7 +185,10 @@ describe('end-to-end tests', () => { await expect(page).toMatch('Raw Request Data Download', { timeout: 6000 }) }) - const dateFormatter = new Intl.DateTimeFormat('en-US', dateFormatterOptions) + const dateFormatter = new Intl.DateTimeFormat( + 'en-US', + dateFormatterOptionsForDate + ) const yesterday = new Date() yesterday.setDate(yesterday.getDate() - 1) const todaysUploadString = dateFormatter.format(yesterday) From 96f61cf54b4cf93b509978506d048083f2159b8f Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 18 Jul 2022 14:00:16 +0200 Subject: [PATCH 3/7] chore(e2e tests): update e2e tests for latest middleware changes --- .github/workflows/e2e.yml | 2 -- __tests__/e2e/e2e.ts | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 93e5df4..724514f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -26,8 +26,6 @@ jobs: - name: Download middleware run: "git clone https://github.com/ibi-group/otp-middleware.git" - - name: "[TEMP] checkout docker branch" - run: cd otp-middleware && git checkout docker - name: Load e2e middleware config run: echo $E2E_OTP_MIDDLEWARE_DOCKER_CONFIG_BASE64 | base64 -d > otp-middleware/configurations/default/env.docker.yml env: diff --git a/__tests__/e2e/e2e.ts b/__tests__/e2e/e2e.ts index 6db8534..7505224 100644 --- a/__tests__/e2e/e2e.ts +++ b/__tests__/e2e/e2e.ts @@ -191,6 +191,7 @@ describe('end-to-end tests', () => { ) const yesterday = new Date() yesterday.setDate(yesterday.getDate() - 1) + yesterday.setMinutes(0, 0, 0) const todaysUploadString = dateFormatter.format(yesterday) it('should see the file that the middleware uploaded', async () => { From 0bd1bcbfc23507a732df4c7ed3e2c4a0ddaedc12 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 21 Jul 2022 13:56:27 +0200 Subject: [PATCH 4/7] refactor: address pr comments --- types/response.ts | 2 +- util/ui.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/types/response.ts b/types/response.ts index 5c8b385..9596303 100644 --- a/types/response.ts +++ b/types/response.ts @@ -16,6 +16,6 @@ export type CDPFile = { export type ConvertedMD = { markdown: string } export type FileNameParserResponse = { - timestamp: number dateFormatterOptions: Intl.DateTimeFormatOptions + timestamp: number } | null diff --git a/util/ui.ts b/util/ui.ts index 6ff72f5..eca8eb9 100644 --- a/util/ui.ts +++ b/util/ui.ts @@ -94,6 +94,8 @@ export const getActiveUserTypes = (): { /** * Converts a CDP file name to a human-readable date. A bit fickle and * not universally browser supported. Attempts to fail as gracefully as possible + * + * Able to handle all the date/time formats that are generated by the CDP. */ export const getDateFromCDPFileName = (filename: string): string => { const date = filename.split('-anon-trip-data')?.[0].split('/')?.[1] @@ -110,7 +112,7 @@ export const getDateFromCDPFileName = (filename: string): string => { return dateFormatter.format(new Date(parsedDate.timestamp)) } /** - * Some CDP file names are only a date. These are easy to parse + * Some CDP file names are only a date (yyyy-mm-dd). These are easy to parse. */ const getDateOnlyFromCDPFileName = (date: string): FileNameParserResponse => { const timestamp = Date.parse(date) @@ -120,7 +122,7 @@ const getDateOnlyFromCDPFileName = (date: string): FileNameParserResponse => { } /** - * Some CDP zip file names include the hour in a very non-standard format. This method + * Some CDP zip file names include the hour in the very non-standard format yyyy-mm-dd-hh. This method * attempts to process this file name */ const getDateAndTimeFromCDPFileName = ( From 755855f8bacf1445c4d6698b8d9f411423ae5b55 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 21 Jul 2022 14:20:39 +0200 Subject: [PATCH 5/7] refactor(e2e tests): no longer have e2e tests depend on middleware file upload --- __tests__/e2e/e2e.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/__tests__/e2e/e2e.ts b/__tests__/e2e/e2e.ts index 7505224..66486c4 100644 --- a/__tests__/e2e/e2e.ts +++ b/__tests__/e2e/e2e.ts @@ -1,7 +1,6 @@ import 'expect-puppeteer' import { server } from '../../jest-puppeteer.config' -import { dateFormatterOptionsForDate } from '../../util/constants' import { waitForDownload } from '../util/waitForDownload' jest.setTimeout(50000) @@ -185,17 +184,13 @@ describe('end-to-end tests', () => { await expect(page).toMatch('Raw Request Data Download', { timeout: 6000 }) }) - const dateFormatter = new Intl.DateTimeFormat( - 'en-US', - dateFormatterOptionsForDate - ) - const yesterday = new Date() - yesterday.setDate(yesterday.getDate() - 1) - yesterday.setMinutes(0, 0, 0) - const todaysUploadString = dateFormatter.format(yesterday) - - it('should see the file that the middleware uploaded', async () => { - await expect(page).toMatch(todaysUploadString) + // Ideally this would be a file that the middleware uploads at e2e test launch, + // however the CDP refactor opt-middleware#170 changed the behavior so that this + // no longer happens. Therefore, we use a string known to exist + const uploadString = 'May 20, 2022' + + it('should see a file that a previous middleware uploaded', async () => { + await expect(page).toMatch(uploadString) }) it('should be able to download a CDP zip file', async () => { const cdpsession = await page.target().createCDPSession() @@ -204,7 +199,7 @@ describe('end-to-end tests', () => { downloadPath: '/tmp' }) - await expect(page).toClick('div', { text: todaysUploadString }) + await expect(page).toClick('div', { text: uploadString }) await waitForDownload('anon-trip-data') await expect(page).toMatch('You last downloaded', { timeout: 6000 }) From 1d543ecf44c2f21173238205835b26e09b9dd3de Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 21 Jul 2022 17:55:08 +0200 Subject: [PATCH 6/7] =?UTF-8?q?refactor:=20don=E2=80=99t=20allow=20Blink?= =?UTF-8?q?=20default=20date=20render?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- util/ui.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/ui.ts b/util/ui.ts index eca8eb9..3d1331e 100644 --- a/util/ui.ts +++ b/util/ui.ts @@ -129,7 +129,8 @@ const getDateAndTimeFromCDPFileName = ( date: string ): FileNameParserResponse => { const datePortion = date.split('-') - if (!datePortion) return null + if (!datePortion || datePortion.find((dp) => Number.isNaN(parseInt(dp)))) + return null // The last number after the last dash is the hour const timePortion = datePortion.pop() From d48823cd77baa1093d0aab15364768644ecc5f89 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Thu, 21 Jul 2022 19:03:45 +0200 Subject: [PATCH 7/7] refactor(util/ui): remove unneeded null check --- util/ui.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/ui.ts b/util/ui.ts index 3d1331e..a9d5e33 100644 --- a/util/ui.ts +++ b/util/ui.ts @@ -129,8 +129,7 @@ const getDateAndTimeFromCDPFileName = ( date: string ): FileNameParserResponse => { const datePortion = date.split('-') - if (!datePortion || datePortion.find((dp) => Number.isNaN(parseInt(dp)))) - return null + if (datePortion?.find((dp) => Number.isNaN(parseInt(dp)))) return null // The last number after the last dash is the hour const timePortion = datePortion.pop()