From 6de6390888468274363c2930dfbdffdc6176cbd4 Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Thu, 22 Feb 2024 13:16:55 -0500 Subject: [PATCH 01/12] simplify config --- handler.ts | 179 ++++++++++++++----------------------------------- serverless.yml | 15 +---- utils.ts | 24 ------- 3 files changed, 53 insertions(+), 165 deletions(-) diff --git a/handler.ts b/handler.ts index 7ebae0b..bb45de1 100644 --- a/handler.ts +++ b/handler.ts @@ -1,6 +1,6 @@ /** - * This script contains the AWS Lambda handler code for merging two Pelias instances - * together. + * This script contains the AWS Lambda handler code for merging some number of geocoder instances + * together * Dependencies are listed in package.json in the same folder. * Notes: * - Most of the folder contents is uploaded to AWS Lambda (see README.md for deploying). @@ -8,7 +8,7 @@ import { URLSearchParams } from 'url' import Bugsnag from '@bugsnag/js' -import getGeocoder from '@opentripplanner/geocoder' +import getGeocoder, { GeocoderConfig } from '@opentripplanner/geocoder' import { createCluster } from 'redis' import type { FeatureCollection } from 'geojson' @@ -16,7 +16,6 @@ import { cachedGeocoderRequest, checkIfResultsAreSatisfactory, convertQSPToGeocoderArgs, - fetchPelias, makeQueryPeliasCompatible, mergeResponses, ServerlessCallbackFunction, @@ -28,54 +27,41 @@ import { const BugsnagPluginAwsLambda = require('@bugsnag/plugin-aws-lambda') const { BUGSNAG_NOTIFIER_KEY, - CSV_ENABLED, - GEOCODE_EARTH_URL, - GEOCODER, - GEOCODER_API_KEY, REDIS_HOST, REDIS_KEY, - SECONDARY_GEOCODE_EARTH_URL, - SECONDARY_GEOCODER, - SECONDARY_GEOCODER_API_KEY, - TRANSIT_BASE_URL, - TRANSIT_GEOCODER + GEOCODERS, + BACKUP_GEOCODERS } = process.env // Severless... why! const redis = !!REDIS_HOST && REDIS_HOST !== 'null' ? createCluster({ - rootNodes: [ - { - password: REDIS_KEY, - url: 'redis://' + REDIS_HOST - } - ], - useReplicas: true - }) + rootNodes: [ + { + password: REDIS_KEY, + url: 'redis://' + REDIS_HOST + } + ], + useReplicas: true + }) : null if (redis) redis.on('error', (err) => console.log('Redis Client Error', err)) -// Ensure env variables have been set -if ( - typeof TRANSIT_BASE_URL !== 'string' || - typeof GEOCODER_API_KEY !== 'string' || - typeof BUGSNAG_NOTIFIER_KEY !== 'string' || - typeof GEOCODER !== 'string' -) { +if (!GEOCODERS) { throw new Error( - 'Error: required configuration variables not found! Ensure env.yml has been decrypted.' + 'Error: required configuration variable GEOCODERS not found! Ensure env.yml has been decrypted.' ) } +const geocoders = JSON.parse(GEOCODERS) +const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS) -if (CSV_ENABLED === 'true' && TRANSIT_GEOCODER === 'OTP') { - throw new Error( - 'Error: Invalid configuration. OTP Geocoder does not support CSV_ENABLED.' - ) +if (geocoders.length !== backupGeocoders.length) { + throw new Error('Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS') } Bugsnag.start({ - apiKey: BUGSNAG_NOTIFIER_KEY, + apiKey: BUGSNAG_NOTIFIER_KEY || '', appType: 'pelias-stitcher-lambda-function', appVersion: require('./package.json').version, plugins: [BugsnagPluginAwsLambda], @@ -84,53 +70,9 @@ Bugsnag.start({ // This handler will wrap around the handler code // and will report exceptions to Bugsnag automatically. // For reference, see https://docs.bugsnag.com/platforms/javascript/aws-lambda/#usage +// @ts-expect-error bugsnag plugin has weird types const bugsnagHandler = Bugsnag.getPlugin('awsLambda').createHandler() -const getPrimaryGeocoder = () => { - if (GEOCODER === 'PELIAS' && typeof GEOCODE_EARTH_URL !== 'string') { - throw new Error('Error: Geocode earth URL not set.') - } - return getGeocoder({ - apiKey: GEOCODER_API_KEY, - baseUrl: GEOCODE_EARTH_URL, - reverseUseFeatureCollection: true, - type: GEOCODER - }) -} - -const getTransitGeocoder = () => { - if (TRANSIT_GEOCODER === 'OTP') { - return getGeocoder({ - baseUrl: TRANSIT_BASE_URL, - type: TRANSIT_GEOCODER - }) - } - - return null -} - -const getSecondaryGeocoder = () => { - if (!SECONDARY_GEOCODER || !SECONDARY_GEOCODER_API_KEY) { - console.warn('Not using secondary Geocoder') - return false - } - - if ( - SECONDARY_GEOCODER === 'PELIAS' && - typeof SECONDARY_GEOCODE_EARTH_URL !== 'string' - ) { - throw new Error( - 'Secondary geocoder configured incorrectly: Geocode.earth URL not set.' - ) - } - return getGeocoder({ - apiKey: SECONDARY_GEOCODER_API_KEY, - baseUrl: SECONDARY_GEOCODE_EARTH_URL, - reverseUseFeatureCollection: true, - type: SECONDARY_GEOCODER - }) -} - /** * Makes a call to a Pelias Instance using secrets from the config file. * Includes special query parameters needed for each type of server. @@ -157,56 +99,31 @@ export const makeGeocoderRequests = async ( delete peliasQSP.layers // Run both requests in parallel - let [primaryResponse, customResponse]: [ - FeatureCollection, - FeatureCollection - ] = await Promise.all([ - cachedGeocoderRequest( - getPrimaryGeocoder(), - apiMethod, - convertQSPToGeocoderArgs(event.queryStringParameters), - // @ts-expect-error Redis Typescript types are not friendly - redis - ), - // Custom request is either through geocoder package or "old" pelias method - getTransitGeocoder() - ? cachedGeocoderRequest( - getTransitGeocoder(), - 'autocomplete', - convertQSPToGeocoderArgs(event.queryStringParameters), - null - ) - : fetchPelias( - TRANSIT_BASE_URL, - apiMethod, - `${new URLSearchParams(peliasQSP).toString()}&sources=transit${ - CSV_ENABLED && CSV_ENABLED === 'true' ? ',pelias' : '' - }` - ) - ]) + let responses: FeatureCollection[] = await Promise.all(geocoders.map(geocoder => cachedGeocoderRequest( + getGeocoder(geocoder), apiMethod, convertQSPToGeocoderArgs(event.queryStringParameters), + // TODO: add redis here + null + + ))) + + responses = await Promise.all(responses.map(async (response, index) => { + // If backup geocoder is present, and the returned results are garbage, use the backup geocoder + if (backupGeocoders[index] && !checkIfResultsAreSatisfactory(response, event.queryStringParameters.text)) { + const backupGeocoder = getGeocoder(backupGeocoders[index]) + console.log("backup geocoder used!") + return await backupGeocoder[apiMethod](convertQSPToGeocoderArgs(event.queryStringParameters)) + } - // If the primary response doesn't contain responses or the responses are not satisfactory, - // run a second (non-cached) request with the secondary geocoder, but only if one is configured. - const secondaryGeocoder = getSecondaryGeocoder() - if ( - secondaryGeocoder && - !checkIfResultsAreSatisfactory( - primaryResponse, - event.queryStringParameters.text - ) - ) { - console.log('Results not satisfactory, falling back on secondary geocoder') - primaryResponse = await secondaryGeocoder[apiMethod]( - convertQSPToGeocoderArgs(event.queryStringParameters) - ) - } + return response + })) + + const merged = responses.reduce((prev, cur, idx) => { + if (idx === 0) return cur + if (prev) return mergeResponses({ primaryResponse: prev, customResponse: cur }) + }, null) - const mergedResponse = mergeResponses({ - customResponse, - primaryResponse - }) return { - body: JSON.stringify(mergedResponse), + body: JSON.stringify(merged), /* The third "standard" CORS header, Access-Control-Allow-Methods is not included here following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/ @@ -274,12 +191,18 @@ module.exports.reverse = bugsnagHandler( context: null, callback: ServerlessCallbackFunction ): Promise => { - const geocoderResponse = await getPrimaryGeocoder().reverse( + let geocoderResponse = await getGeocoder(geocoders[0]).reverse( convertQSPToGeocoderArgs(event.queryStringParameters) ) + if (!geocoderResponse && backupGeocoders[0]) { + geocoderResponse = await getGeocoder(backupGeocoders[0]).reverse(convertQSPToGeocoderArgs(event.queryStringParameters)) + } + + geocoderResponse.label = geocoderResponse.name + callback(null, { - body: JSON.stringify(geocoderResponse), + body: JSON.stringify([geocoderResponse]), /* The third "standard" CORS header, Access-Control-Allow-Methods is not included here following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/ diff --git a/serverless.yml b/serverless.yml index 515650c..cd3fcf4 100644 --- a/serverless.yml +++ b/serverless.yml @@ -9,22 +9,11 @@ provider: subnetIds: - ${self:custom.secrets.LAMBDA_EXEC_SUBNET} environment: - GEOCODER: ${self:custom.secrets.GEOCODER} - TRANSIT_GEOCODER: ${self:custom.secrets.TRANSIT_GEOCODER, null} - TRANSIT_BASE_URL: ${self:custom.secrets.TRANSIT_BASE_URL, null} - # Pelias instance of Geocode.Earth, with street and landmarks - GEOCODE_EARTH_URL: ${self:custom.secrets.GEOCODE_EARTH_URL, null} - GEOCODER_API_KEY: ${self:custom.secrets.GEOCODER_API_KEY, null} - # Used to logging to Bugsnag + GEOCODERS: ${self:custom.secrets.GEOCODERS} + BACKUP_GEOCODERS: ${self:custom.secrets.BACKUP_GEOCODERS} BUGSNAG_NOTIFIER_KEY: ${self:custom.secrets.BUGSNAG_NOTIFIER_KEY} REDIS_HOST: ${self:custom.secrets.REDIS_HOST, null} REDIS_KEY: ${self:custom.secrets.REDIS_KEY, null} - # Used to enable CSV source - CSV_ENABLED: ${self:custom.secrets.CSV_ENABLED, false} - # Secondary Geocoder config - SECONDARY_GEOCODER: ${self:custom.secrets.SECONDARY_GEOCODER, null} - SECONDARY_GEOCODER_API_KEY: ${self:custom.secrets.SECONDARY_GEOCODER_API_KEY, null} - SECONDARY_GEOCODE_EARTH_URL: ${self:custom.secrets.SECONDARY_GEOCODE_EARTH_URL, null} custom: secrets: ${file(env.yml)} functions: diff --git a/utils.ts b/utils.ts index 3a9a6dd..32169a0 100644 --- a/utils.ts +++ b/utils.ts @@ -96,30 +96,6 @@ export const convertQSPToGeocoderArgs = ( return geocoderArgs } -/** - * Executes a request on a Pelias instance - * This is used for 'manually' querying a Pelias instance outside outside of the geocoder package. - * @param baseUrl URL of the Pelias instance, without a trailing slash - * @param service The endpoint to make the query to (generally search or autocomplete) - * @param query The rest of the Pelias query (any GET paremeters) - * @returns Pelias response decoded from JSON - */ -export const fetchPelias = async ( - baseUrl?: string, - service?: string, - query?: string -): Promise => { - if (!baseUrl) return { features: [], type: 'FeatureCollection' } - try { - const response = await fetch(`${baseUrl}/${service}?${query}`, {}) - return await response.json() - } catch (e) { - bugsnag.notify(e) - console.warn(`${baseUrl} failed to return valid Pelias response`) - return { features: [], type: 'FeatureCollection' } - } -} - /** * Compares two GeoJSON positions and returns if they are equal within 10m accuracy * @param a One GeoJSON Position object From 8564364948d81faf2c825ff4c3bc3925d8dce75c Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Fri, 23 Feb 2024 09:53:39 -0500 Subject: [PATCH 02/12] clean up --- env.example.yml | 13 ++----------- handler.ts | 4 +--- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/env.example.yml b/env.example.yml index 1dd770d..2a7c917 100644 --- a/env.example.yml +++ b/env.example.yml @@ -1,17 +1,8 @@ LAMBDA_EXEC_SG: Insert AWS Security Group ID Here (it must be in the same VPC as the subnet) LAMBDA_EXEC_SUBNET: Insert AWS Subnet ID Here (it must be in the same VPC as the security group) BUGSNAG_NOTIFIER_KEY: INSERT BUGSNAG NOTIFIER KEY HERE -GEOCODER_API_KEY: INSERT API KEY HERE -GEOCODE_EARTH_URL: https://api.geocode.earth/v1 # Not needed if geocode.earth is not used -CSV_ENABLED: true -CUSTOM_PELIAS_URL: http:///v1 -GEOCODER: HERE # Options: HERE, PELIAS -TRANSIT_GEOCODER: (optional) OTP/PELIAS -TRANSIT_BASE_URL: (conditionally required) OTP instance when TRANSIT_GEOCODER=OTP (/otp/routers/{routerId}) - -SECONDARY_GEOCODER: (optional) HERE/PELIAS -SECONDARY_GEOCODER_API_KEY: (optional) INSERT SECONDARY API KEY HERE -SECONDARY_GEOCODE_EARTH_URL: (optional) https://api.geocode.earth/v1 # Not needed if geocode.earth is not used +GEOCODERS: +BACKUP_GEOCODERS: REDIS_HOST: (optional) REDIS_KEY: (optional) diff --git a/handler.ts b/handler.ts index bb45de1..93d581e 100644 --- a/handler.ts +++ b/handler.ts @@ -5,10 +5,8 @@ * Notes: * - Most of the folder contents is uploaded to AWS Lambda (see README.md for deploying). */ -import { URLSearchParams } from 'url' - import Bugsnag from '@bugsnag/js' -import getGeocoder, { GeocoderConfig } from '@opentripplanner/geocoder' +import getGeocoder from '@opentripplanner/geocoder' import { createCluster } from 'redis' import type { FeatureCollection } from 'geojson' From b54c97fed3bbf330efcda37ad04b2b4b2318051f Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Tue, 27 Feb 2024 12:41:17 -0500 Subject: [PATCH 03/12] add POI support using otp-ui offline geocoder --- handler.ts | 7 +++++-- serverless.yml | 1 + yarn.lock | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/handler.ts b/handler.ts index 93d581e..ce5d0b4 100644 --- a/handler.ts +++ b/handler.ts @@ -28,6 +28,7 @@ const { REDIS_HOST, REDIS_KEY, GEOCODERS, + POIS, BACKUP_GEOCODERS } = process.env @@ -53,6 +54,8 @@ if (!GEOCODERS) { } const geocoders = JSON.parse(GEOCODERS) const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS) +// Serverless is not great about null +const pois = POIS && POIS !== "null" ? JSON.parse(POIS) : [] if (geocoders.length !== backupGeocoders.length) { throw new Error('Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS') @@ -68,7 +71,6 @@ Bugsnag.start({ // This handler will wrap around the handler code // and will report exceptions to Bugsnag automatically. // For reference, see https://docs.bugsnag.com/platforms/javascript/aws-lambda/#usage -// @ts-expect-error bugsnag plugin has weird types const bugsnagHandler = Bugsnag.getPlugin('awsLambda').createHandler() /** @@ -98,7 +100,7 @@ export const makeGeocoderRequests = async ( // Run both requests in parallel let responses: FeatureCollection[] = await Promise.all(geocoders.map(geocoder => cachedGeocoderRequest( - getGeocoder(geocoder), apiMethod, convertQSPToGeocoderArgs(event.queryStringParameters), + getGeocoder(geocoder), apiMethod, { ...convertQSPToGeocoderArgs(event.queryStringParameters), items: pois }, // TODO: add redis here null @@ -117,6 +119,7 @@ export const makeGeocoderRequests = async ( const merged = responses.reduce((prev, cur, idx) => { if (idx === 0) return cur + // @ts-expect-error Typechecking is broken here for some reason if (prev) return mergeResponses({ primaryResponse: prev, customResponse: cur }) }, null) diff --git a/serverless.yml b/serverless.yml index cd3fcf4..f917b80 100644 --- a/serverless.yml +++ b/serverless.yml @@ -11,6 +11,7 @@ provider: environment: GEOCODERS: ${self:custom.secrets.GEOCODERS} BACKUP_GEOCODERS: ${self:custom.secrets.BACKUP_GEOCODERS} + POIS: ${self:custom.secrets.POIS, null} BUGSNAG_NOTIFIER_KEY: ${self:custom.secrets.BUGSNAG_NOTIFIER_KEY} REDIS_HOST: ${self:custom.secrets.REDIS_HOST, null} REDIS_KEY: ${self:custom.secrets.REDIS_KEY, null} diff --git a/yarn.lock b/yarn.lock index b66bf22..2dcead2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2222,6 +2222,11 @@ resolved "https://registry.yarnpkg.com/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz#8ace5259254426ccef57f3175bc64ed7095ed919" integrity sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw== +"@leeoniya/ufuzzy@^1.0.14": + version "1.0.14" + resolved "https://registry.yarnpkg.com/@leeoniya/ufuzzy/-/ufuzzy-1.0.14.tgz#01572c0de9cfa1420cf6ecac76dd59db5ebd1337" + integrity sha512-/xF4baYuCQMo+L/fMSUrZnibcu0BquEGnbxfVPiZhs/NbJeKj4c/UmFpQzW9Us0w45ui/yYW3vyaqawhNYsTzA== + "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde" From c75fbabe7b1089815a957c2a2d1a5c9a447cca5d Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Tue, 27 Feb 2024 14:40:39 -0500 Subject: [PATCH 04/12] looser config parsing --- handler.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/handler.ts b/handler.ts index ce5d0b4..43646af 100644 --- a/handler.ts +++ b/handler.ts @@ -20,6 +20,7 @@ import { ServerlessEvent, ServerlessResponse } from './utils' +import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline' // This plugin must be imported via cjs to ensure its existence (typescript recommendation) const BugsnagPluginAwsLambda = require('@bugsnag/plugin-aws-lambda') @@ -55,7 +56,18 @@ if (!GEOCODERS) { const geocoders = JSON.parse(GEOCODERS) const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS) // Serverless is not great about null -const pois = POIS && POIS !== "null" ? JSON.parse(POIS) : [] +const pois = POIS && POIS !== "null" ? (JSON.parse(POIS) as OfflineResponse).map((poi) => { + if (typeof poi.lat === 'string') { + poi.lat = parseFloat(poi.lat) + } + if (typeof poi.lon === 'string') { + poi.lon = parseFloat(poi.lon) + } + return poi +}) : [] + +console.log(pois) + if (geocoders.length !== backupGeocoders.length) { throw new Error('Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS') From 477b3dc6d37d538ece6a6da0995f887968d70f62 Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Thu, 7 Mar 2024 14:17:53 -0500 Subject: [PATCH 05/12] chore(deps): upgrade geocoder --- package.json | 2 +- yarn.lock | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index fe083fc..9195391 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "@bugsnag/js": "^7.11.0", "@bugsnag/plugin-aws-lambda": "^7.11.0", "@conveyal/lonlat": "^1.4.1", - "@opentripplanner/geocoder": "^2.0.1", + "@opentripplanner/geocoder": "^2.2.0", "geolib": "^3.3.1", "node-fetch": "^2.6.1", "redis": "^4.1.0", diff --git a/yarn.lock b/yarn.lock index 2dcead2..557afcb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2521,13 +2521,14 @@ dependencies: "@octokit/openapi-types" "^12.11.0" -"@opentripplanner/geocoder@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@opentripplanner/geocoder/-/geocoder-2.0.1.tgz#ffb701a1e1b293b54bd368e782dd5ed6b42bdefa" - integrity sha512-hUKGCwAEc7DvwoS/3Kj+PLCUExY+LhZX61rv1R5E0KOGfsbXv1CY10g2aPGCt7ZcnozbKKo8p+U9gzwCetfQKQ== +"@opentripplanner/geocoder@^2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@opentripplanner/geocoder/-/geocoder-2.2.0.tgz#071f91664e898b06705c781fddd4f75dfcb3cc3f" + integrity sha512-V41WOCIpvwLRHEchBg3vRzGOOxoI+SdxczLGgLGPJ/Q/XiqaTqUgsK8WUv+hAUAGYNxI7jZbx7zdC2pXCH0m4w== dependencies: "@conveyal/geocoder-arcgis-geojson" "^0.0.3" "@conveyal/lonlat" "^1.4.1" + "@leeoniya/ufuzzy" "^1.0.14" isomorphic-mapzen-search "^1.6.1" lodash.memoize "^4.1.2" From 07f98a99de7457d4a097cf89ae17f87e1cd5a230 Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Tue, 19 Mar 2024 14:30:23 -0400 Subject: [PATCH 06/12] feat: remove redis support --- env.example.yml | 3 - handler.ts | 116 ++++++++++++++------------------- package.json | 1 - serverless.yml | 2 - utils.ts | 28 +------- yarn.lock | 168 ++++++------------------------------------------ 6 files changed, 71 insertions(+), 247 deletions(-) diff --git a/env.example.yml b/env.example.yml index 2a7c917..304ed92 100644 --- a/env.example.yml +++ b/env.example.yml @@ -3,6 +3,3 @@ LAMBDA_EXEC_SUBNET: Insert AWS Subnet ID Here (it must be in the same VPC as the BUGSNAG_NOTIFIER_KEY: INSERT BUGSNAG NOTIFIER KEY HERE GEOCODERS: BACKUP_GEOCODERS: - -REDIS_HOST: (optional) -REDIS_KEY: (optional) diff --git a/handler.ts b/handler.ts index 43646af..bbed626 100644 --- a/handler.ts +++ b/handler.ts @@ -7,8 +7,8 @@ */ import Bugsnag from '@bugsnag/js' import getGeocoder from '@opentripplanner/geocoder' -import { createCluster } from 'redis' import type { FeatureCollection } from 'geojson' +import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline' import { cachedGeocoderRequest, @@ -20,33 +20,10 @@ import { ServerlessEvent, ServerlessResponse } from './utils' -import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline' // This plugin must be imported via cjs to ensure its existence (typescript recommendation) const BugsnagPluginAwsLambda = require('@bugsnag/plugin-aws-lambda') -const { - BUGSNAG_NOTIFIER_KEY, - REDIS_HOST, - REDIS_KEY, - GEOCODERS, - POIS, - BACKUP_GEOCODERS -} = process.env - -// Severless... why! -const redis = - !!REDIS_HOST && REDIS_HOST !== 'null' - ? createCluster({ - rootNodes: [ - { - password: REDIS_KEY, - url: 'redis://' + REDIS_HOST - } - ], - useReplicas: true - }) - : null -if (redis) redis.on('error', (err) => console.log('Redis Client Error', err)) +const { BACKUP_GEOCODERS, BUGSNAG_NOTIFIER_KEY, GEOCODERS, POIS } = process.env if (!GEOCODERS) { throw new Error( @@ -56,21 +33,24 @@ if (!GEOCODERS) { const geocoders = JSON.parse(GEOCODERS) const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS) // Serverless is not great about null -const pois = POIS && POIS !== "null" ? (JSON.parse(POIS) as OfflineResponse).map((poi) => { - if (typeof poi.lat === 'string') { - poi.lat = parseFloat(poi.lat) - } - if (typeof poi.lon === 'string') { - poi.lon = parseFloat(poi.lon) - } - return poi -}) : [] - -console.log(pois) +const pois = + POIS && POIS !== 'null' + ? (JSON.parse(POIS) as OfflineResponse).map((poi) => { + if (typeof poi.lat === 'string') { + poi.lat = parseFloat(poi.lat) + } + if (typeof poi.lon === 'string') { + poi.lon = parseFloat(poi.lon) + } + return poi + }) + : [] if (geocoders.length !== backupGeocoders.length) { - throw new Error('Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS') + throw new Error( + 'Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS' + ) } Bugsnag.start({ @@ -111,28 +91,41 @@ export const makeGeocoderRequests = async ( delete peliasQSP.layers // Run both requests in parallel - let responses: FeatureCollection[] = await Promise.all(geocoders.map(geocoder => cachedGeocoderRequest( - getGeocoder(geocoder), apiMethod, { ...convertQSPToGeocoderArgs(event.queryStringParameters), items: pois }, - // TODO: add redis here - null - - ))) - - responses = await Promise.all(responses.map(async (response, index) => { - // If backup geocoder is present, and the returned results are garbage, use the backup geocoder - if (backupGeocoders[index] && !checkIfResultsAreSatisfactory(response, event.queryStringParameters.text)) { - const backupGeocoder = getGeocoder(backupGeocoders[index]) - console.log("backup geocoder used!") - return await backupGeocoder[apiMethod](convertQSPToGeocoderArgs(event.queryStringParameters)) - } + let responses: FeatureCollection[] = await Promise.all( + geocoders.map((geocoder) => + cachedGeocoderRequest(getGeocoder(geocoder), apiMethod, { + ...convertQSPToGeocoderArgs(event.queryStringParameters), + items: pois + }) + ) + ) - return response - })) + responses = await Promise.all( + responses.map(async (response, index) => { + // If backup geocoder is present, and the returned results are garbage, use the backup geocoder + if ( + backupGeocoders[index] && + !checkIfResultsAreSatisfactory( + response, + event.queryStringParameters.text + ) + ) { + const backupGeocoder = getGeocoder(backupGeocoders[index]) + console.log('backup geocoder used!') + return await backupGeocoder[apiMethod]( + convertQSPToGeocoderArgs(event.queryStringParameters) + ) + } + + return response + }) + ) const merged = responses.reduce((prev, cur, idx) => { if (idx === 0) return cur // @ts-expect-error Typechecking is broken here for some reason - if (prev) return mergeResponses({ primaryResponse: prev, customResponse: cur }) + if (prev) + return mergeResponses({ customResponse: cur, primaryResponse: prev }) }, null) return { @@ -162,18 +155,7 @@ module.exports.autocomplete = bugsnagHandler( context: null, callback: ServerlessCallbackFunction ): Promise => { - if (redis) { - // Only autocomplete needs redis - try { - await redis.connect() - } catch (e) { - console.warn('Redis connection failed. Likely already connected') - console.log(e) - } - } - const response = await makeGeocoderRequests(event, 'autocomplete') - callback(null, response) } ) @@ -209,7 +191,9 @@ module.exports.reverse = bugsnagHandler( ) if (!geocoderResponse && backupGeocoders[0]) { - geocoderResponse = await getGeocoder(backupGeocoders[0]).reverse(convertQSPToGeocoderArgs(event.queryStringParameters)) + geocoderResponse = await getGeocoder(backupGeocoders[0]).reverse( + convertQSPToGeocoderArgs(event.queryStringParameters) + ) } geocoderResponse.label = geocoderResponse.name diff --git a/package.json b/package.json index 9195391..2f36e15 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ "@opentripplanner/geocoder": "^2.2.0", "geolib": "^3.3.1", "node-fetch": "^2.6.1", - "redis": "^4.1.0", "serverless-api-gateway-caching": "^1.8.1", "serverless-plugin-typescript": "^1.1.9" }, diff --git a/serverless.yml b/serverless.yml index f917b80..4e1c80c 100644 --- a/serverless.yml +++ b/serverless.yml @@ -13,8 +13,6 @@ provider: BACKUP_GEOCODERS: ${self:custom.secrets.BACKUP_GEOCODERS} POIS: ${self:custom.secrets.POIS, null} BUGSNAG_NOTIFIER_KEY: ${self:custom.secrets.BUGSNAG_NOTIFIER_KEY} - REDIS_HOST: ${self:custom.secrets.REDIS_HOST, null} - REDIS_KEY: ${self:custom.secrets.REDIS_KEY, null} custom: secrets: ${file(env.yml)} functions: diff --git a/utils.ts b/utils.ts index 32169a0..60790ed 100644 --- a/utils.ts +++ b/utils.ts @@ -6,7 +6,6 @@ import { getDistance } from 'geolib' import fetch from 'node-fetch' import type { LonLatOutput } from '@conveyal/lonlat' import type { Feature, FeatureCollection, Position } from 'geojson' -import type { RedisClientType } from 'redis' import { AnyGeocoderQuery } from '@opentripplanner/geocoder/lib/geocoders/types' // Types @@ -237,42 +236,21 @@ export const mergeResponses = ( } /** - * Makes a geocoder request by first checking a potential redis store for a cached - * response. If a response is fetched, stores response in redis store. + * Formerly allowed caching requests in Redis store. This method is kept around now + * in case another caching solution is attempted again in the future * @param geocoder geocoder object returned from geocoder package * @param requestMethod Geocoder Request Method * @param args Args for Geocoder request method - * @param redisClient Redis client already connected * @returns FeatureCollection either from cache or live */ export const cachedGeocoderRequest = async ( geocoder: Record Promise>, requestMethod: string, - args: AnyGeocoderQuery, - redisClient: RedisClientType | null + args: AnyGeocoderQuery ): Promise => { const { focusPoint, text } = args if (!text) return { features: [], type: 'FeatureCollection' } - const redisKey = `${text}:${focusPoint?.lat}:${focusPoint?.lon}` - - if (redisClient) { - const cachedResponse = await redisClient.get(redisKey) - if (cachedResponse) { - return JSON.parse(cachedResponse) - } - } const onlineResponse = await geocoder[requestMethod](args) - // If we are at this point and have a redis object we know there - // was no entry in the cache - if (redisClient) { - try { - redisClient.set(redisKey, JSON.stringify(onlineResponse)) - // 30 day expiry - redisClient.expire(redisKey, 30 * 24 * 60 * 60) - } catch (e) { - console.warn(`Could not add response to redis cache: ${e}`) - } - } return onlineResponse } diff --git a/yarn.lock b/yarn.lock index 557afcb..109ca56 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2532,40 +2532,6 @@ isomorphic-mapzen-search "^1.6.1" lodash.memoize "^4.1.2" -"@redis/bloom@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" - integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== - -"@redis/client@1.5.13": - version "1.5.13" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.13.tgz#78786218fc1632ee4b5f7d73b8d456c95880e086" - integrity sha512-epkUM9D0Sdmt93/8Ozk43PNjLi36RZzG+d/T1Gdu5AI8jvghonTeLYV69WVWdilvFo+PYxbP0TZ0saMvr6nscQ== - dependencies: - cluster-key-slot "1.1.2" - generic-pool "3.9.0" - yallist "4.0.0" - -"@redis/graph@1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea" - integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw== - -"@redis/json@1.0.6": - version "1.0.6" - resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e" - integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw== - -"@redis/search@1.1.6": - version "1.1.6" - resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.6.tgz#33bcdd791d9ed88ab6910243a355d85a7fedf756" - integrity sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw== - -"@redis/time-series@1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" - integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== - "@rollup/plugin-babel@^5.1.0": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -3309,12 +3275,7 @@ "@types/node" "*" "@types/responselike" "^1.0.0" -"@types/estree@*": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" - integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== - -"@types/estree@0.0.39": +"@types/estree@*", "@types/estree@0.0.39": version "0.0.39" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.39.tgz#e177e699ee1b8c22d23174caaa7422644389509f" integrity sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw== @@ -4514,16 +4475,11 @@ cacheable-request@^7.0.2: normalize-url "^6.0.1" responselike "^2.0.0" -cachedir@2.3.0: +cachedir@2.3.0, cachedir@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.3.0.tgz#0c75892a052198f0b21c7c1804d8331edfcae0e8" integrity sha512-A+Fezp4zxnit6FanDmv9EqXNAi3vt9DWp51/71UEhXukb7QUuvtv9344h91dyAxuTLoSYJFU299qzR3tzwPAhw== -cachedir@^2.3.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.4.0.tgz#7fef9cf7367233d7c88068fe6e34ed0d355a610d" - integrity sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ== - call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513" @@ -4810,11 +4766,6 @@ clone@^1.0.2: resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" integrity sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg== -cluster-key-slot@1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" - integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== - cmd-shim@^4.0.1: version "4.1.0" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -5050,16 +5001,11 @@ core-js-compat@^3.31.0, core-js-compat@^3.33.1: dependencies: browserslist "^4.22.2" -core-util-is@1.0.2: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== -core-util-is@~1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" - integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== - cosmiconfig-typescript-loader@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-5.0.0.tgz#0d3becfe022a871f7275ceb2397d692e06045dc8" @@ -6325,16 +6271,11 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" -extsprintf@1.3.0: +extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" integrity sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g== -extsprintf@^1.2.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" - integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== - fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -6613,10 +6554,10 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== -form-data@^2.3.1: - version "2.5.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" - integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA== +form-data@^2.3.1, form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== dependencies: asynckit "^0.4.0" combined-stream "^1.0.6" @@ -6640,15 +6581,6 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" - integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.6" - mime-types "^2.1.12" - formidable@^2.0.1: version "2.1.2" resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.1.2.tgz#fa973a2bec150e4ce7cac15589d7a25fc30ebd89" @@ -6814,11 +6746,6 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" -generic-pool@3.9.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" - integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== - gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -7417,7 +7344,7 @@ init-package-json@^2.0.5: validate-npm-package-license "^3.0.4" validate-npm-package-name "^3.0.0" -inquirer@8.2.5: +inquirer@8.2.5, inquirer@^8.2.5: version "8.2.5" resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-8.2.5.tgz#d8654a7542c35a9b9e069d27e2df4858784d54f8" integrity sha512-QAgPDQMEgrDssk1XiwwHoOGYF9BAbUcc1+j+FhEvaOt8/cKRqyLn0U5qA6F74fGhTMGxf92pOvPBeh29jQJDTQ== @@ -7438,27 +7365,6 @@ inquirer@8.2.5: through "^2.3.6" wrap-ansi "^7.0.0" -inquirer@^8.2.5: - version "8.2.6" - resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-8.2.6.tgz#733b74888195d8d400a67ac332011b5fae5ea562" - integrity sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg== - dependencies: - ansi-escapes "^4.2.1" - chalk "^4.1.1" - cli-cursor "^3.1.0" - cli-width "^3.0.0" - external-editor "^3.0.3" - figures "^3.0.0" - lodash "^4.17.21" - mute-stream "0.0.8" - ora "^5.4.1" - run-async "^2.4.0" - rxjs "^7.5.5" - string-width "^4.1.0" - strip-ansi "^6.0.0" - through "^2.3.6" - wrap-ansi "^6.0.1" - internal-slot@^1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.6.tgz#37e756098c4911c5e912b8edbf71ed3aa116f930" @@ -9478,16 +9384,11 @@ minimist-options@4.1.0: is-plain-obj "^1.1.0" kind-of "^6.0.3" -minimist@1.2.7: +minimist@1.2.7, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: version "1.2.7" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== -minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: - version "1.2.8" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" - integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== - minipass-collect@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617" @@ -10912,18 +10813,6 @@ redeyed@~2.1.0: dependencies: esprima "~4.0.0" -redis@^4.1.0: - version "4.6.12" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.12.tgz#b607c93e9b0dd09e641f837092e9a68cc6ac6570" - integrity sha512-41Xuuko6P4uH4VPe5nE3BqXHB7a9lkFL0J29AlxKaIfD6eWO8VO/5PDF9ad2oS+mswMsfFxaM5DlE3tnXT+P8Q== - dependencies: - "@redis/bloom" "1.2.0" - "@redis/client" "1.5.13" - "@redis/graph" "1.1.1" - "@redis/json" "1.0.6" - "@redis/search" "1.1.6" - "@redis/time-series" "1.0.5" - reflect.getprototypeof@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz#aaccbf41aca3821b87bb71d9dcbc7ad0ba50a3f3" @@ -11280,7 +11169,7 @@ safe-array-concat@^1.0.1: has-symbols "^1.0.3" isarray "^2.0.5" -safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -11311,16 +11200,11 @@ safe-regex@^1.1.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -sax@1.2.1: +sax@1.2.1, sax@>=0.6.0: version "1.2.1" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.6.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" - integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA== - saxes@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" @@ -12018,14 +11902,7 @@ string.prototype.trimstart@^1.0.7: define-properties "^1.2.0" es-abstract "^1.22.1" -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - -string_decoder@~1.1.1: +string_decoder@^1.1.1, string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== @@ -13131,15 +13008,6 @@ wrap-ansi@^3.0.1: string-width "^2.1.1" strip-ansi "^4.0.0" -wrap-ansi@^6.0.1: - version "6.2.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" - integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -13224,16 +13092,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yallist@4.0.0, yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yallist@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yaml-ast-parser@0.0.43: version "0.0.43" resolved "https://registry.yarnpkg.com/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb" From 0a5518c05f84a30ea9ed060c02c9c4499b114386 Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Tue, 19 Mar 2024 14:33:25 -0400 Subject: [PATCH 07/12] chore: lint --- handler.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/handler.ts b/handler.ts index bbed626..b8e2041 100644 --- a/handler.ts +++ b/handler.ts @@ -46,7 +46,6 @@ const pois = }) : [] - if (geocoders.length !== backupGeocoders.length) { throw new Error( 'Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS' From 154231952916972ec6ad060ba49e4c8289690ba7 Mon Sep 17 00:00:00 2001 From: Daniel Heppner Date: Wed, 20 Mar 2024 14:11:43 +0800 Subject: [PATCH 08/12] fix types in handler --- handler.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/handler.ts b/handler.ts index b8e2041..6c1b33d 100644 --- a/handler.ts +++ b/handler.ts @@ -7,7 +7,7 @@ */ import Bugsnag from '@bugsnag/js' import getGeocoder from '@opentripplanner/geocoder' -import type { FeatureCollection } from 'geojson' +import { Feature, Geometry, FeatureCollection, GeoJsonProperties } from 'geojson' import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline' import { @@ -62,7 +62,7 @@ Bugsnag.start({ // This handler will wrap around the handler code // and will report exceptions to Bugsnag automatically. // For reference, see https://docs.bugsnag.com/platforms/javascript/aws-lambda/#usage -const bugsnagHandler = Bugsnag.getPlugin('awsLambda').createHandler() +const bugsnagHandler = Bugsnag?.getPlugin('awsLambda')?.createHandler() /** * Makes a call to a Pelias Instance using secrets from the config file. @@ -120,12 +120,11 @@ export const makeGeocoderRequests = async ( }) ) - const merged = responses.reduce((prev, cur, idx) => { - if (idx === 0) return cur - // @ts-expect-error Typechecking is broken here for some reason - if (prev) + const merged = responses.reduce>( + (prev, cur, idx) => { + if (idx === 0) return cur return mergeResponses({ customResponse: cur, primaryResponse: prev }) - }, null) + }, {type: "FeatureCollection", features: []}) return { body: JSON.stringify(merged), From 6d74c69af21050e39f9472f75f94aa1c484f025f Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Wed, 20 Mar 2024 13:14:25 -0400 Subject: [PATCH 09/12] chore: lint --- handler.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/handler.ts b/handler.ts index 6c1b33d..3bc8739 100644 --- a/handler.ts +++ b/handler.ts @@ -7,7 +7,11 @@ */ import Bugsnag from '@bugsnag/js' import getGeocoder from '@opentripplanner/geocoder' -import { Feature, Geometry, FeatureCollection, GeoJsonProperties } from 'geojson' +import { + Geometry, + FeatureCollection, + GeoJsonProperties +} from 'geojson' import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline' import { @@ -120,11 +124,15 @@ export const makeGeocoderRequests = async ( }) ) - const merged = responses.reduce>( + const merged = responses.reduce< + FeatureCollection + >( (prev, cur, idx) => { if (idx === 0) return cur return mergeResponses({ customResponse: cur, primaryResponse: prev }) - }, {type: "FeatureCollection", features: []}) + }, + { features: [], type: 'FeatureCollection' } + ) return { body: JSON.stringify(merged), From b816cc085078f6a4ebba089c97b685c69675a21f Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Wed, 20 Mar 2024 13:39:50 -0400 Subject: [PATCH 10/12] chore: lint --- handler.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/handler.ts b/handler.ts index 3bc8739..4cca040 100644 --- a/handler.ts +++ b/handler.ts @@ -7,11 +7,7 @@ */ import Bugsnag from '@bugsnag/js' import getGeocoder from '@opentripplanner/geocoder' -import { - Geometry, - FeatureCollection, - GeoJsonProperties -} from 'geojson' +import { Geometry, FeatureCollection, GeoJsonProperties } from 'geojson' import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline' import { From 1794cf4c9674f4e743543acfce040482be76958a Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Mon, 25 Mar 2024 13:14:34 -0400 Subject: [PATCH 11/12] refactor: address pr feedback --- env.example.yml | 4 ++-- handler.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/env.example.yml b/env.example.yml index 304ed92..9408353 100644 --- a/env.example.yml +++ b/env.example.yml @@ -1,5 +1,5 @@ LAMBDA_EXEC_SG: Insert AWS Security Group ID Here (it must be in the same VPC as the subnet) LAMBDA_EXEC_SUBNET: Insert AWS Subnet ID Here (it must be in the same VPC as the security group) BUGSNAG_NOTIFIER_KEY: INSERT BUGSNAG NOTIFIER KEY HERE -GEOCODERS: -BACKUP_GEOCODERS: +GEOCODERS: +BACKUP_GEOCODERS: diff --git a/handler.ts b/handler.ts index 4cca040..b5259b7 100644 --- a/handler.ts +++ b/handler.ts @@ -102,6 +102,7 @@ export const makeGeocoderRequests = async ( responses = await Promise.all( responses.map(async (response, index) => { // If backup geocoder is present, and the returned results are garbage, use the backup geocoder + // if one is configured. This request will not be cached if ( backupGeocoders[index] && !checkIfResultsAreSatisfactory( From 8bff9d93184fb8a7f1cb4bb83041edc8e4c0ee4c Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Tue, 2 Apr 2024 10:51:19 -0400 Subject: [PATCH 12/12] refactor: address pr feedback --- handler.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/handler.ts b/handler.ts index b5259b7..21ae889 100644 --- a/handler.ts +++ b/handler.ts @@ -89,8 +89,8 @@ export const makeGeocoderRequests = async ( const peliasQSP = { ...event.queryStringParameters } delete peliasQSP.layers - // Run both requests in parallel - let responses: FeatureCollection[] = await Promise.all( + // Run all requests in parallel + const uncheckedResponses: FeatureCollection[] = await Promise.all( geocoders.map((geocoder) => cachedGeocoderRequest(getGeocoder(geocoder), apiMethod, { ...convertQSPToGeocoderArgs(event.queryStringParameters), @@ -99,8 +99,9 @@ export const makeGeocoderRequests = async ( ) ) - responses = await Promise.all( - responses.map(async (response, index) => { + // Check if responses are satisfactory, and re-do them if needed + const responses = await Promise.all( + uncheckedResponses.map(async (response, index) => { // If backup geocoder is present, and the returned results are garbage, use the backup geocoder // if one is configured. This request will not be cached if ( @@ -111,7 +112,6 @@ export const makeGeocoderRequests = async ( ) ) { const backupGeocoder = getGeocoder(backupGeocoders[index]) - console.log('backup geocoder used!') return await backupGeocoder[apiMethod]( convertQSPToGeocoderArgs(event.queryStringParameters) ) @@ -128,6 +128,7 @@ export const makeGeocoderRequests = async ( if (idx === 0) return cur return mergeResponses({ customResponse: cur, primaryResponse: prev }) }, + // TODO: clean this reducer up. See https://github.com/ibi-group/pelias-stitch/pull/28#discussion_r1547582739 { features: [], type: 'FeatureCollection' } )