Merge pull request #28 from ibi-group/rebuild

Support Infinite Geocoders, Support Offline Geocoder
ibi-group · Apr 2, 2024 · 4623ba9 · 4623ba9
2 parents 942ff03 + 8bff9d9
commit 4623ba9
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 386 deletions.
diff --git a/env.example.yml b/env.example.yml
@@ -1,17 +1,5 @@
 LAMBDA_EXEC_SG: Insert AWS Security Group ID Here (it must be in the same VPC as the subnet)
 LAMBDA_EXEC_SUBNET: Insert AWS Subnet ID Here (it must be in the same VPC as the security group)
 BUGSNAG_NOTIFIER_KEY: INSERT BUGSNAG NOTIFIER KEY HERE
-GEOCODER_API_KEY: INSERT API KEY HERE
-GEOCODE_EARTH_URL: https://api.geocode.earth/v1 # Not needed if geocode.earth is not used
-CSV_ENABLED: true
-CUSTOM_PELIAS_URL: http://<insert your Pelias endpoint here>/v1
-GEOCODER: HERE # Options: HERE, PELIAS
-TRANSIT_GEOCODER: (optional) OTP/PELIAS
-TRANSIT_BASE_URL: (conditionally required) OTP instance when TRANSIT_GEOCODER=OTP (/otp/routers/{routerId})
-
-SECONDARY_GEOCODER: (optional) HERE/PELIAS
-SECONDARY_GEOCODER_API_KEY: (optional) INSERT SECONDARY API KEY HERE
-SECONDARY_GEOCODE_EARTH_URL: (optional) https://api.geocode.earth/v1 # Not needed if geocode.earth is not used
-
-REDIS_HOST: (optional) <insert IP of redis host here>
-REDIS_KEY: (optional) <insert redis password here>
+GEOCODERS: <Stringified JSON Array of OTP-UI `GeocoderConfig`s>
+BACKUP_GEOCODERS: <Stringified JSON Array of OTP-UI `GeocoderConfig`'s. Same length and order as GEOCODERS> 
diff --git a/handler.ts b/handler.ts
@@ -1,22 +1,19 @@
 /**
- * This script contains the AWS Lambda handler code for merging two Pelias instances
- * together.
+ * This script contains the AWS Lambda handler code for merging some number of geocoder instances
+ * together
  * Dependencies are listed in package.json in the same folder.
  * Notes:
  * - Most of the folder contents is uploaded to AWS Lambda (see README.md for deploying).
  */
-import { URLSearchParams } from 'url'
-
 import Bugsnag from '@bugsnag/js'
 import getGeocoder from '@opentripplanner/geocoder'
-import { createCluster } from 'redis'
-import type { FeatureCollection } from 'geojson'
+import { Geometry, FeatureCollection, GeoJsonProperties } from 'geojson'
+import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline'
 
 import {
   cachedGeocoderRequest,
   checkIfResultsAreSatisfactory,
   convertQSPToGeocoderArgs,
-  fetchPelias,
   makeQueryPeliasCompatible,
   mergeResponses,
   ServerlessCallbackFunction,
@@ -26,56 +23,37 @@ import {
 
 // This plugin must be imported via cjs to ensure its existence (typescript recommendation)
 const BugsnagPluginAwsLambda = require('@bugsnag/plugin-aws-lambda')
-const {
-  BUGSNAG_NOTIFIER_KEY,
-  CSV_ENABLED,
-  GEOCODE_EARTH_URL,
-  GEOCODER,
-  GEOCODER_API_KEY,
-  REDIS_HOST,
-  REDIS_KEY,
-  SECONDARY_GEOCODE_EARTH_URL,
-  SECONDARY_GEOCODER,
-  SECONDARY_GEOCODER_API_KEY,
-  TRANSIT_BASE_URL,
-  TRANSIT_GEOCODER
-} = process.env
-
-// Severless... why!
-const redis =
-  !!REDIS_HOST && REDIS_HOST !== 'null'
-    ? createCluster({
-        rootNodes: [
-          {
-            password: REDIS_KEY,
-            url: 'redis://' + REDIS_HOST
-          }
-        ],
-        useReplicas: true
-      })
-    : null
-if (redis) redis.on('error', (err) => console.log('Redis Client Error', err))
+const { BACKUP_GEOCODERS, BUGSNAG_NOTIFIER_KEY, GEOCODERS, POIS } = process.env
 
-// Ensure env variables have been set
-if (
-  typeof TRANSIT_BASE_URL !== 'string' ||
-  typeof GEOCODER_API_KEY !== 'string' ||
-  typeof BUGSNAG_NOTIFIER_KEY !== 'string' ||
-  typeof GEOCODER !== 'string'
-) {
+if (!GEOCODERS) {
   throw new Error(
-    'Error: required configuration variables not found! Ensure env.yml has been decrypted.'
+    'Error: required configuration variable GEOCODERS not found! Ensure env.yml has been decrypted.'
   )
 }
+const geocoders = JSON.parse(GEOCODERS)
+const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS)
+// Serverless is not great about null
+const pois =
+  POIS && POIS !== 'null'
+    ? (JSON.parse(POIS) as OfflineResponse).map((poi) => {
+        if (typeof poi.lat === 'string') {
+          poi.lat = parseFloat(poi.lat)
+        }
+        if (typeof poi.lon === 'string') {
+          poi.lon = parseFloat(poi.lon)
+        }
+        return poi
+      })
+    : []
 
-if (CSV_ENABLED === 'true' && TRANSIT_GEOCODER === 'OTP') {
+if (geocoders.length !== backupGeocoders.length) {
   throw new Error(
-    'Error: Invalid configuration. OTP Geocoder does not support CSV_ENABLED.'
+    'Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS'
   )
 }
 
 Bugsnag.start({
-  apiKey: BUGSNAG_NOTIFIER_KEY,
+  apiKey: BUGSNAG_NOTIFIER_KEY || '',
   appType: 'pelias-stitcher-lambda-function',
   appVersion: require('./package.json').version,
   plugins: [BugsnagPluginAwsLambda],
@@ -84,52 +62,7 @@ Bugsnag.start({
 // This handler will wrap around the handler code
 // and will report exceptions to Bugsnag automatically.
 // For reference, see https://docs.bugsnag.com/platforms/javascript/aws-lambda/#usage
-const bugsnagHandler = Bugsnag.getPlugin('awsLambda').createHandler()
-
-const getPrimaryGeocoder = () => {
-  if (GEOCODER === 'PELIAS' && typeof GEOCODE_EARTH_URL !== 'string') {
-    throw new Error('Error: Geocode earth URL not set.')
-  }
-  return getGeocoder({
-    apiKey: GEOCODER_API_KEY,
-    baseUrl: GEOCODE_EARTH_URL,
-    reverseUseFeatureCollection: true,
-    type: GEOCODER
-  })
-}
-
-const getTransitGeocoder = () => {
-  if (TRANSIT_GEOCODER === 'OTP') {
-    return getGeocoder({
-      baseUrl: TRANSIT_BASE_URL,
-      type: TRANSIT_GEOCODER
-    })
-  }
-
-  return null
-}
-
-const getSecondaryGeocoder = () => {
-  if (!SECONDARY_GEOCODER || !SECONDARY_GEOCODER_API_KEY) {
-    console.warn('Not using secondary Geocoder')
-    return false
-  }
-
-  if (
-    SECONDARY_GEOCODER === 'PELIAS' &&
-    typeof SECONDARY_GEOCODE_EARTH_URL !== 'string'
-  ) {
-    throw new Error(
-      'Secondary geocoder configured incorrectly: Geocode.earth URL not set.'
-    )
-  }
-  return getGeocoder({
-    apiKey: SECONDARY_GEOCODER_API_KEY,
-    baseUrl: SECONDARY_GEOCODE_EARTH_URL,
-    reverseUseFeatureCollection: true,
-    type: SECONDARY_GEOCODER
-  })
-}
+const bugsnagHandler = Bugsnag?.getPlugin('awsLambda')?.createHandler()
 
 /**
  * Makes a call to a Pelias Instance using secrets from the config file.
@@ -156,57 +89,51 @@ export const makeGeocoderRequests = async (
   const peliasQSP = { ...event.queryStringParameters }
   delete peliasQSP.layers
 
-  // Run both requests in parallel
-  let [primaryResponse, customResponse]: [
-    FeatureCollection,
-    FeatureCollection
-  ] = await Promise.all([
-    cachedGeocoderRequest(
-      getPrimaryGeocoder(),
-      apiMethod,
-      convertQSPToGeocoderArgs(event.queryStringParameters),
-      // @ts-expect-error Redis Typescript types are not friendly
-      redis
-    ),
-    // Custom request is either through geocoder package or "old" pelias method
-    getTransitGeocoder()
-      ? cachedGeocoderRequest(
-          getTransitGeocoder(),
-          'autocomplete',
-          convertQSPToGeocoderArgs(event.queryStringParameters),
-          null
+  // Run all requests in parallel
+  const uncheckedResponses: FeatureCollection[] = await Promise.all(
+    geocoders.map((geocoder) =>
+      cachedGeocoderRequest(getGeocoder(geocoder), apiMethod, {
+        ...convertQSPToGeocoderArgs(event.queryStringParameters),
+        items: pois
+      })
+    )
+  )
+
+  // Check if responses are satisfactory, and re-do them if needed
+  const responses = await Promise.all(
+    uncheckedResponses.map(async (response, index) => {
+      // If backup geocoder is present, and the returned results are garbage, use the backup geocoder
+      // if one is configured. This request will not be cached
+      if (
+        backupGeocoders[index] &&
+        !checkIfResultsAreSatisfactory(
+          response,
+          event.queryStringParameters.text
         )
-      : fetchPelias(
-          TRANSIT_BASE_URL,
-          apiMethod,
-          `${new URLSearchParams(peliasQSP).toString()}&sources=transit${
-            CSV_ENABLED && CSV_ENABLED === 'true' ? ',pelias' : ''
-          }`
+      ) {
+        const backupGeocoder = getGeocoder(backupGeocoders[index])
+        return await backupGeocoder[apiMethod](
+          convertQSPToGeocoderArgs(event.queryStringParameters)
         )
-  ])
+      }
 
-  // If the primary response doesn't contain responses or the responses are not satisfactory,
-  // run a second (non-cached) request with the secondary geocoder, but only if one is configured.
-  const secondaryGeocoder = getSecondaryGeocoder()
-  if (
-    secondaryGeocoder &&
-    !checkIfResultsAreSatisfactory(
-      primaryResponse,
-      event.queryStringParameters.text
-    )
-  ) {
-    console.log('Results not satisfactory, falling back on secondary geocoder')
-    primaryResponse = await secondaryGeocoder[apiMethod](
-      convertQSPToGeocoderArgs(event.queryStringParameters)
-    )
-  }
+      return response
+    })
+  )
+
+  const merged = responses.reduce<
+    FeatureCollection<Geometry, GeoJsonProperties>
+  >(
+    (prev, cur, idx) => {
+      if (idx === 0) return cur
+      return mergeResponses({ customResponse: cur, primaryResponse: prev })
+    },
+    // TODO: clean this reducer up. See https://github.com/ibi-group/pelias-stitch/pull/28#discussion_r1547582739
+    { features: [], type: 'FeatureCollection' }
+  )
 
-  const mergedResponse = mergeResponses({
-    customResponse,
-    primaryResponse
-  })
   return {
-    body: JSON.stringify(mergedResponse),
+    body: JSON.stringify(merged),
     /*
     The third "standard" CORS header, Access-Control-Allow-Methods is not included here
     following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/
@@ -232,18 +159,7 @@ module.exports.autocomplete = bugsnagHandler(
     context: null,
     callback: ServerlessCallbackFunction
   ): Promise<void> => {
-    if (redis) {
-      // Only autocomplete needs redis
-      try {
-        await redis.connect()
-      } catch (e) {
-        console.warn('Redis connection failed. Likely already connected')
-        console.log(e)
-      }
-    }
-
     const response = await makeGeocoderRequests(event, 'autocomplete')
-
     callback(null, response)
   }
 )
@@ -274,12 +190,20 @@ module.exports.reverse = bugsnagHandler(
     context: null,
     callback: ServerlessCallbackFunction
   ): Promise<void> => {
-    const geocoderResponse = await getPrimaryGeocoder().reverse(
+    let geocoderResponse = await getGeocoder(geocoders[0]).reverse(
       convertQSPToGeocoderArgs(event.queryStringParameters)
     )
 
+    if (!geocoderResponse && backupGeocoders[0]) {
+      geocoderResponse = await getGeocoder(backupGeocoders[0]).reverse(
+        convertQSPToGeocoderArgs(event.queryStringParameters)
+      )
+    }
+
+    geocoderResponse.label = geocoderResponse.name
+
     callback(null, {
-      body: JSON.stringify(geocoderResponse),
+      body: JSON.stringify([geocoderResponse]),
       /*
         The third "standard" CORS header, Access-Control-Allow-Methods is not included here
         following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/

diff --git a/package.json b/package.json
@@ -27,10 +27,9 @@
     "@bugsnag/js": "^7.11.0",
     "@bugsnag/plugin-aws-lambda": "^7.11.0",
     "@conveyal/lonlat": "^1.4.1",
-    "@opentripplanner/geocoder": "^2.0.1",
+    "@opentripplanner/geocoder": "^2.2.0",
     "geolib": "^3.3.1",
     "node-fetch": "^2.6.1",
-    "redis": "^4.1.0",
     "serverless-api-gateway-caching": "^1.8.1",
     "serverless-plugin-typescript": "^1.1.9"
   },

diff --git a/serverless.yml b/serverless.yml
@@ -9,22 +9,10 @@ provider:
     subnetIds:
       - ${self:custom.secrets.LAMBDA_EXEC_SUBNET}
   environment:
-    GEOCODER: ${self:custom.secrets.GEOCODER}
-    TRANSIT_GEOCODER: ${self:custom.secrets.TRANSIT_GEOCODER, null}
-    TRANSIT_BASE_URL: ${self:custom.secrets.TRANSIT_BASE_URL, null}
-    # Pelias instance of Geocode.Earth, with street and landmarks
-    GEOCODE_EARTH_URL: ${self:custom.secrets.GEOCODE_EARTH_URL, null}
-    GEOCODER_API_KEY: ${self:custom.secrets.GEOCODER_API_KEY, null}
-    # Used to logging to Bugsnag
+    GEOCODERS: ${self:custom.secrets.GEOCODERS}
+    BACKUP_GEOCODERS: ${self:custom.secrets.BACKUP_GEOCODERS}
+    POIS: ${self:custom.secrets.POIS, null}
     BUGSNAG_NOTIFIER_KEY: ${self:custom.secrets.BUGSNAG_NOTIFIER_KEY}
-    REDIS_HOST: ${self:custom.secrets.REDIS_HOST, null}
-    REDIS_KEY: ${self:custom.secrets.REDIS_KEY, null}
-    # Used to enable CSV source
-    CSV_ENABLED: ${self:custom.secrets.CSV_ENABLED, false}
-    # Secondary Geocoder config
-    SECONDARY_GEOCODER: ${self:custom.secrets.SECONDARY_GEOCODER, null}
-    SECONDARY_GEOCODER_API_KEY: ${self:custom.secrets.SECONDARY_GEOCODER_API_KEY, null}
-    SECONDARY_GEOCODE_EARTH_URL: ${self:custom.secrets.SECONDARY_GEOCODE_EARTH_URL, null}
 custom:
   secrets: ${file(env.yml)}
 functions: