Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Infinite Geocoders, Support Offline Geocoder #28

Merged
merged 12 commits into from
Apr 2, 2024
16 changes: 2 additions & 14 deletions env.example.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,5 @@
LAMBDA_EXEC_SG: Insert AWS Security Group ID Here (it must be in the same VPC as the subnet)
LAMBDA_EXEC_SUBNET: Insert AWS Subnet ID Here (it must be in the same VPC as the security group)
BUGSNAG_NOTIFIER_KEY: INSERT BUGSNAG NOTIFIER KEY HERE
GEOCODER_API_KEY: INSERT API KEY HERE
GEOCODE_EARTH_URL: https://api.geocode.earth/v1 # Not needed if geocode.earth is not used
CSV_ENABLED: true
CUSTOM_PELIAS_URL: http://<insert your Pelias endpoint here>/v1
GEOCODER: HERE # Options: HERE, PELIAS
TRANSIT_GEOCODER: (optional) OTP/PELIAS
TRANSIT_BASE_URL: (conditionally required) OTP instance when TRANSIT_GEOCODER=OTP (/otp/routers/{routerId})

SECONDARY_GEOCODER: (optional) HERE/PELIAS
SECONDARY_GEOCODER_API_KEY: (optional) INSERT SECONDARY API KEY HERE
SECONDARY_GEOCODE_EARTH_URL: (optional) https://api.geocode.earth/v1 # Not needed if geocode.earth is not used

REDIS_HOST: (optional) <insert IP of redis host here>
REDIS_KEY: (optional) <insert redis password here>
GEOCODERS: <Stringified JSON Array of OTP-UI `GeocoderConfig`s>
BACKUP_GEOCODERS: <Stringified JSON Array of OTP-UI `GeocoderConfig`'s. Same length and order as GEOCODERS>
225 changes: 74 additions & 151 deletions handler.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
/**
* This script contains the AWS Lambda handler code for merging two Pelias instances
* together.
* This script contains the AWS Lambda handler code for merging some number of geocoder instances
* together
* Dependencies are listed in package.json in the same folder.
* Notes:
* - Most of the folder contents is uploaded to AWS Lambda (see README.md for deploying).
*/
import { URLSearchParams } from 'url'

import Bugsnag from '@bugsnag/js'
import getGeocoder from '@opentripplanner/geocoder'
import { createCluster } from 'redis'
import type { FeatureCollection } from 'geojson'
import { Geometry, FeatureCollection, GeoJsonProperties } from 'geojson'
import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline'

import {
cachedGeocoderRequest,
checkIfResultsAreSatisfactory,
convertQSPToGeocoderArgs,
fetchPelias,
makeQueryPeliasCompatible,
mergeResponses,
ServerlessCallbackFunction,
Expand All @@ -26,56 +23,37 @@ import {

// This plugin must be imported via cjs to ensure its existence (typescript recommendation)
const BugsnagPluginAwsLambda = require('@bugsnag/plugin-aws-lambda')
const {
BUGSNAG_NOTIFIER_KEY,
CSV_ENABLED,
GEOCODE_EARTH_URL,
GEOCODER,
GEOCODER_API_KEY,
REDIS_HOST,
REDIS_KEY,
SECONDARY_GEOCODE_EARTH_URL,
SECONDARY_GEOCODER,
SECONDARY_GEOCODER_API_KEY,
TRANSIT_BASE_URL,
TRANSIT_GEOCODER
} = process.env

// Severless... why!
const redis =
!!REDIS_HOST && REDIS_HOST !== 'null'
? createCluster({
rootNodes: [
{
password: REDIS_KEY,
url: 'redis://' + REDIS_HOST
}
],
useReplicas: true
})
: null
if (redis) redis.on('error', (err) => console.log('Redis Client Error', err))
const { BACKUP_GEOCODERS, BUGSNAG_NOTIFIER_KEY, GEOCODERS, POIS } = process.env

// Ensure env variables have been set
if (
typeof TRANSIT_BASE_URL !== 'string' ||
typeof GEOCODER_API_KEY !== 'string' ||
typeof BUGSNAG_NOTIFIER_KEY !== 'string' ||
typeof GEOCODER !== 'string'
) {
if (!GEOCODERS) {
throw new Error(
'Error: required configuration variables not found! Ensure env.yml has been decrypted.'
'Error: required configuration variable GEOCODERS not found! Ensure env.yml has been decrypted.'
)
}
const geocoders = JSON.parse(GEOCODERS)
const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS)
// Serverless is not great about null
const pois =
POIS && POIS !== 'null'
? (JSON.parse(POIS) as OfflineResponse).map((poi) => {
if (typeof poi.lat === 'string') {
poi.lat = parseFloat(poi.lat)
}
if (typeof poi.lon === 'string') {
poi.lon = parseFloat(poi.lon)
}
return poi
})
: []

if (CSV_ENABLED === 'true' && TRANSIT_GEOCODER === 'OTP') {
if (geocoders.length !== backupGeocoders.length) {
throw new Error(
'Error: Invalid configuration. OTP Geocoder does not support CSV_ENABLED.'
'Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS'
)
}

Bugsnag.start({
apiKey: BUGSNAG_NOTIFIER_KEY,
apiKey: BUGSNAG_NOTIFIER_KEY || '',
appType: 'pelias-stitcher-lambda-function',
appVersion: require('./package.json').version,
plugins: [BugsnagPluginAwsLambda],
Expand All @@ -84,52 +62,7 @@ Bugsnag.start({
// This handler will wrap around the handler code
// and will report exceptions to Bugsnag automatically.
// For reference, see https://docs.bugsnag.com/platforms/javascript/aws-lambda/#usage
const bugsnagHandler = Bugsnag.getPlugin('awsLambda').createHandler()

const getPrimaryGeocoder = () => {
if (GEOCODER === 'PELIAS' && typeof GEOCODE_EARTH_URL !== 'string') {
throw new Error('Error: Geocode earth URL not set.')
}
return getGeocoder({
apiKey: GEOCODER_API_KEY,
baseUrl: GEOCODE_EARTH_URL,
reverseUseFeatureCollection: true,
type: GEOCODER
})
}

const getTransitGeocoder = () => {
if (TRANSIT_GEOCODER === 'OTP') {
return getGeocoder({
baseUrl: TRANSIT_BASE_URL,
type: TRANSIT_GEOCODER
})
}

return null
}

const getSecondaryGeocoder = () => {
if (!SECONDARY_GEOCODER || !SECONDARY_GEOCODER_API_KEY) {
console.warn('Not using secondary Geocoder')
return false
}

if (
SECONDARY_GEOCODER === 'PELIAS' &&
typeof SECONDARY_GEOCODE_EARTH_URL !== 'string'
) {
throw new Error(
'Secondary geocoder configured incorrectly: Geocode.earth URL not set.'
)
}
return getGeocoder({
apiKey: SECONDARY_GEOCODER_API_KEY,
baseUrl: SECONDARY_GEOCODE_EARTH_URL,
reverseUseFeatureCollection: true,
type: SECONDARY_GEOCODER
})
}
const bugsnagHandler = Bugsnag?.getPlugin('awsLambda')?.createHandler()

/**
* Makes a call to a Pelias Instance using secrets from the config file.
Expand Down Expand Up @@ -157,56 +90,49 @@ export const makeGeocoderRequests = async (
delete peliasQSP.layers

// Run both requests in parallel
miles-grant-ibigroup marked this conversation as resolved.
Show resolved Hide resolved
let [primaryResponse, customResponse]: [
FeatureCollection,
FeatureCollection
] = await Promise.all([
cachedGeocoderRequest(
getPrimaryGeocoder(),
apiMethod,
convertQSPToGeocoderArgs(event.queryStringParameters),
// @ts-expect-error Redis Typescript types are not friendly
redis
),
// Custom request is either through geocoder package or "old" pelias method
getTransitGeocoder()
? cachedGeocoderRequest(
getTransitGeocoder(),
'autocomplete',
convertQSPToGeocoderArgs(event.queryStringParameters),
null
let responses: FeatureCollection[] = await Promise.all(
geocoders.map((geocoder) =>
cachedGeocoderRequest(getGeocoder(geocoder), apiMethod, {
...convertQSPToGeocoderArgs(event.queryStringParameters),
items: pois
})
)
)

responses = await Promise.all(
miles-grant-ibigroup marked this conversation as resolved.
Show resolved Hide resolved
responses.map(async (response, index) => {
// If backup geocoder is present, and the returned results are garbage, use the backup geocoder
// if one is configured. This request will not be cached
if (
backupGeocoders[index] &&
!checkIfResultsAreSatisfactory(
response,
event.queryStringParameters.text
)
: fetchPelias(
TRANSIT_BASE_URL,
apiMethod,
`${new URLSearchParams(peliasQSP).toString()}&sources=transit${
CSV_ENABLED && CSV_ENABLED === 'true' ? ',pelias' : ''
}`
) {
const backupGeocoder = getGeocoder(backupGeocoders[index])
console.log('backup geocoder used!')
miles-grant-ibigroup marked this conversation as resolved.
Show resolved Hide resolved
return await backupGeocoder[apiMethod](
miles-grant-ibigroup marked this conversation as resolved.
Show resolved Hide resolved
convertQSPToGeocoderArgs(event.queryStringParameters)
)
])
}

// If the primary response doesn't contain responses or the responses are not satisfactory,
// run a second (non-cached) request with the secondary geocoder, but only if one is configured.
const secondaryGeocoder = getSecondaryGeocoder()
if (
secondaryGeocoder &&
!checkIfResultsAreSatisfactory(
primaryResponse,
event.queryStringParameters.text
)
) {
console.log('Results not satisfactory, falling back on secondary geocoder')
primaryResponse = await secondaryGeocoder[apiMethod](
convertQSPToGeocoderArgs(event.queryStringParameters)
)
}
return response
})
)

const merged = responses.reduce<
FeatureCollection<Geometry, GeoJsonProperties>
>(
(prev, cur, idx) => {
if (idx === 0) return cur
return mergeResponses({ customResponse: cur, primaryResponse: prev })
},
{ features: [], type: 'FeatureCollection' }
miles-grant-ibigroup marked this conversation as resolved.
Show resolved Hide resolved
)

const mergedResponse = mergeResponses({
customResponse,
primaryResponse
})
return {
body: JSON.stringify(mergedResponse),
body: JSON.stringify(merged),
/*
The third "standard" CORS header, Access-Control-Allow-Methods is not included here
following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/
Expand All @@ -232,18 +158,7 @@ module.exports.autocomplete = bugsnagHandler(
context: null,
callback: ServerlessCallbackFunction
): Promise<void> => {
if (redis) {
// Only autocomplete needs redis
try {
await redis.connect()
} catch (e) {
console.warn('Redis connection failed. Likely already connected')
console.log(e)
}
}

const response = await makeGeocoderRequests(event, 'autocomplete')

callback(null, response)
}
)
Expand Down Expand Up @@ -274,12 +189,20 @@ module.exports.reverse = bugsnagHandler(
context: null,
callback: ServerlessCallbackFunction
): Promise<void> => {
const geocoderResponse = await getPrimaryGeocoder().reverse(
let geocoderResponse = await getGeocoder(geocoders[0]).reverse(
convertQSPToGeocoderArgs(event.queryStringParameters)
)

if (!geocoderResponse && backupGeocoders[0]) {
geocoderResponse = await getGeocoder(backupGeocoders[0]).reverse(
convertQSPToGeocoderArgs(event.queryStringParameters)
)
}

geocoderResponse.label = geocoderResponse.name

callback(null, {
body: JSON.stringify(geocoderResponse),
body: JSON.stringify([geocoderResponse]),
/*
The third "standard" CORS header, Access-Control-Allow-Methods is not included here
following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/
Expand Down
3 changes: 1 addition & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,9 @@
"@bugsnag/js": "^7.11.0",
"@bugsnag/plugin-aws-lambda": "^7.11.0",
"@conveyal/lonlat": "^1.4.1",
"@opentripplanner/geocoder": "^2.0.1",
"@opentripplanner/geocoder": "^2.2.0",
"geolib": "^3.3.1",
"node-fetch": "^2.6.1",
"redis": "^4.1.0",
"serverless-api-gateway-caching": "^1.8.1",
"serverless-plugin-typescript": "^1.1.9"
},
Expand Down
18 changes: 3 additions & 15 deletions serverless.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,10 @@ provider:
subnetIds:
- ${self:custom.secrets.LAMBDA_EXEC_SUBNET}
environment:
GEOCODER: ${self:custom.secrets.GEOCODER}
TRANSIT_GEOCODER: ${self:custom.secrets.TRANSIT_GEOCODER, null}
TRANSIT_BASE_URL: ${self:custom.secrets.TRANSIT_BASE_URL, null}
# Pelias instance of Geocode.Earth, with street and landmarks
GEOCODE_EARTH_URL: ${self:custom.secrets.GEOCODE_EARTH_URL, null}
GEOCODER_API_KEY: ${self:custom.secrets.GEOCODER_API_KEY, null}
# Used to logging to Bugsnag
GEOCODERS: ${self:custom.secrets.GEOCODERS}
BACKUP_GEOCODERS: ${self:custom.secrets.BACKUP_GEOCODERS}
POIS: ${self:custom.secrets.POIS, null}
BUGSNAG_NOTIFIER_KEY: ${self:custom.secrets.BUGSNAG_NOTIFIER_KEY}
REDIS_HOST: ${self:custom.secrets.REDIS_HOST, null}
REDIS_KEY: ${self:custom.secrets.REDIS_KEY, null}
# Used to enable CSV source
CSV_ENABLED: ${self:custom.secrets.CSV_ENABLED, false}
# Secondary Geocoder config
SECONDARY_GEOCODER: ${self:custom.secrets.SECONDARY_GEOCODER, null}
SECONDARY_GEOCODER_API_KEY: ${self:custom.secrets.SECONDARY_GEOCODER_API_KEY, null}
SECONDARY_GEOCODE_EARTH_URL: ${self:custom.secrets.SECONDARY_GEOCODE_EARTH_URL, null}
custom:
secrets: ${file(env.yml)}
functions:
Expand Down
Loading
Loading