From a27f4b5f79da6e996063b29908cbc3008386c14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ot=C3=A1vio=20Jacobi?= Date: Mon, 28 Aug 2023 09:00:05 -0300 Subject: [PATCH] Add S3 webresource handler Change-type: minor --- config.ts | 2 + config/confd/conf.d/cloudront-pk.pem.toml | 7 ++ config/confd/conf.d/env.toml | 11 +- config/confd/templates/cloudfront-pk.pem.tmpl | 1 + config/confd/templates/env.tmpl | 9 ++ package-lock.json | 68 +++++++----- package.json | 3 +- src/fileupload-handler.ts | 102 ++++++++++++++++++ src/index.ts | 1 + src/lib/config.ts | 23 ++++ 10 files changed, 199 insertions(+), 28 deletions(-) create mode 100644 config/confd/conf.d/cloudront-pk.pem.toml create mode 100644 config/confd/templates/cloudfront-pk.pem.tmpl create mode 100644 src/fileupload-handler.ts diff --git a/config.ts b/config.ts index 17cb4db8b..211a295e5 100644 --- a/config.ts +++ b/config.ts @@ -1,5 +1,6 @@ import type { ConfigLoader } from '@balena/pinejs'; import * as balenaModel from './src/balena'; +import { getFileUploadHandler } from './src/fileupload-handler'; export = { models: [balenaModel], @@ -25,4 +26,5 @@ export = { ], }, ], + webResourceHandler: getFileUploadHandler(), } as ConfigLoader.Config; diff --git a/config/confd/conf.d/cloudront-pk.pem.toml b/config/confd/conf.d/cloudront-pk.pem.toml new file mode 100644 index 000000000..8127f2759 --- /dev/null +++ b/config/confd/conf.d/cloudront-pk.pem.toml @@ -0,0 +1,7 @@ +[template] +src = "cloudfront-pk.pem.tmpl" +dest = "/etc/ssl/private/cloudfront-pk.pem" +keys = [ + "WEBRESOURCES_CLOUDFRONT_PRIVATEKEY", +] +mode = "0400" diff --git a/config/confd/conf.d/env.toml b/config/confd/conf.d/env.toml index e7d28b0fc..44d08be61 100644 --- a/config/confd/conf.d/env.toml +++ b/config/confd/conf.d/env.toml @@ -53,5 +53,14 @@ keys = [ "VPN_PORT", "VPN_SERVICE_API_KEY", "VPN_GUEST_API_KEY", - "VPN_CONNECT_PROXY_PORT" + "VPN_CONNECT_PROXY_PORT", + "WEBRESOURCES_S3_ACCESS_KEY", + "WEBRESOURCES_S3_SECRET_KEY", + "WEBRESOURCES_S3_REGION", + "WEBRESOURCES_S3_HOST", + "WEBRESOURCES_S3_BUCKET", + "WEBRESOURCES_S3_MAX_FILESIZE", + "WEBRESOURCES_CLOUDFRONT_PUBLICKEY", + "WEBRESOURCES_CLOUDFRONT_HOST", + "WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH" ] diff --git a/config/confd/templates/cloudfront-pk.pem.tmpl b/config/confd/templates/cloudfront-pk.pem.tmpl new file mode 100644 index 000000000..068ad8bd9 --- /dev/null +++ b/config/confd/templates/cloudfront-pk.pem.tmpl @@ -0,0 +1 @@ +{{base64Decode (getenv "WEBRESOURCES_CLOUDFRONT_PRIVATEKEY" "")}} diff --git a/config/confd/templates/env.tmpl b/config/confd/templates/env.tmpl index f6284dba7..123bbdbdb 100644 --- a/config/confd/templates/env.tmpl +++ b/config/confd/templates/env.tmpl @@ -79,3 +79,12 @@ VPN_SERVICE_API_KEY={{getenv "VPN_SERVICE_API_KEY"}} {{if getenv "VPN_GUEST_API_KEY"}}VPN_GUEST_API_KEY={{getenv "VPN_GUEST_API_KEY"}}{{end}} {{if getenv "AUTH_RESINOS_REGISTRY_CODE"}}AUTH_RESINOS_REGISTRY_CODE={{getenv "AUTH_RESINOS_REGISTRY_CODE"}}{{end}} {{if getenv "BROTLI_COMPRESSION_QUALITY"}}BROTLI_COMPRESSION_QUALITY={{getenv "BROTLI_COMPRESSION_QUALITY"}}{{end}} +{{if getenv "WEBRESOURCES_S3_ACCESS_KEY"}}WEBRESOURCES_S3_ACCESS_KEY={{getenv "WEBRESOURCES_S3_ACCESS_KEY"}}{{end}} +{{if getenv "WEBRESOURCES_S3_SECRET_KEY"}}WEBRESOURCES_S3_SECRET_KEY={{getenv "WEBRESOURCES_S3_SECRET_KEY"}}{{end}} +{{if getenv "WEBRESOURCES_S3_REGION"}}WEBRESOURCES_S3_REGION={{getenv "WEBRESOURCES_S3_REGION"}}{{end}} +{{if getenv "WEBRESOURCES_S3_HOST"}}WEBRESOURCES_S3_HOST={{getenv "WEBRESOURCES_S3_HOST"}}{{end}} +{{if getenv "WEBRESOURCES_S3_BUCKET"}}WEBRESOURCES_S3_BUCKET={{getenv "WEBRESOURCES_S3_BUCKET"}}{{end}} +{{if getenv "WEBRESOURCES_S3_MAX_FILESIZE"}}WEBRESOURCES_S3_MAX_FILESIZE={{getenv "WEBRESOURCES_S3_MAX_FILESIZE"}}{{end}} +{{if getenv "WEBRESOURCES_CLOUDFRONT_PUBLICKEY"}}WEBRESOURCES_CLOUDFRONT_PUBLICKEY={{getenv "WEBRESOURCES_CLOUDFRONT_PUBLICKEY"}}{{end}} +{{if getenv "WEBRESOURCES_CLOUDFRONT_HOST"}}WEBRESOURCES_CLOUDFRONT_HOST={{getenv "WEBRESOURCES_CLOUDFRONT_HOST"}}{{end}} +WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH=/etc/ssl/private/cloudfront-pk.pem diff --git a/package-lock.json b/package-lock.json index 9c751b82c..c7bc0c265 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "@balena/env-parsing": "^1.1.5", "@balena/es-version": "^1.0.2", "@balena/node-metrics-gatherer": "^6.0.3", - "@balena/pinejs": "^15.3.4", + "@balena/pinejs": "^15.3.8", + "@balena/pinejs-webresource-cloudfront": "^0.0.4", "@sentry/node": "^7.49.0", "@types/basic-auth": "^1.1.3", "@types/bluebird": "^3.5.38", @@ -484,6 +485,17 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", "optional": true }, + "node_modules/@aws-sdk/cloudfront-signer": { + "version": "3.398.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/cloudfront-signer/-/cloudfront-signer-3.398.0.tgz", + "integrity": "sha512-fyUty9SNI3oiOSvgVcK0S2OmihawzqWCR5TdcZ2EWbpiLk0V94U5BaKoIKu6jYY+57OgGjr/vUDrDideT/0cMw==", + "dependencies": { + "@smithy/url-parser": "^2.0.5" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@aws-sdk/config-resolver": { "version": "3.200.0", "resolved": "https://registry.npmjs.org/@aws-sdk/config-resolver/-/config-resolver-3.200.0.tgz", @@ -2907,12 +2919,12 @@ } }, "node_modules/@balena/pinejs": { - "version": "15.3.4", - "resolved": "https://registry.npmjs.org/@balena/pinejs/-/pinejs-15.3.4.tgz", - "integrity": "sha512-kFEUeg3DduSR6vIPO/kkuUs/T4LkklVreXLKKDjHuQ6IOqIxHmJcej4S5PptmAjNVVeHD6HfMr4Ke07uY6TbLg==", + "version": "15.3.8", + "resolved": "https://registry.npmjs.org/@balena/pinejs/-/pinejs-15.3.8.tgz", + "integrity": "sha512-scNqPhb0ZfrC9m/Si4V3hhzHMPn/N/iE9rHh1mEp40WJW1KxR2n9NS5e/PlYA/thblA2CGakQLwuqR+0ZTqlbQ==", "dependencies": { "@balena/abstract-sql-compiler": "^9.0.3", - "@balena/abstract-sql-to-typescript": "^2.1.0", + "@balena/abstract-sql-to-typescript": "^2.1.1", "@balena/env-parsing": "^1.1.5", "@balena/lf-to-abstract-sql": "^5.0.0", "@balena/odata-parser": "^3.0.0", @@ -2976,6 +2988,16 @@ "serve-static": "^1.15.0" } }, + "node_modules/@balena/pinejs-webresource-cloudfront": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@balena/pinejs-webresource-cloudfront/-/pinejs-webresource-cloudfront-0.0.4.tgz", + "integrity": "sha512-pACee791NLsxih/4nwZe6dIyWy4UP+F41ZoyO9udIIUlHJ0pZin7mviWAuRoAqyfT3iC0Nhm0r/0CcT4nA08kg==", + "dependencies": { + "@aws-sdk/cloudfront-signer": "^3.398.0", + "@balena/pinejs": "^15.3.3", + "memoizee": "^0.4.15" + } + }, "node_modules/@balena/pinejs/node_modules/@types/node": { "version": "18.17.15", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.15.tgz", @@ -3981,12 +4003,11 @@ "optional": true }, "node_modules/@smithy/querystring-parser": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.1.tgz", - "integrity": "sha512-h+e7k1z+IvI2sSbUBG9Aq46JsgLl4UqIUl6aigAlRBj+P6ocNXpM6Yn1vMBw5ijtXeZbYpd1YvCxwDgdw3jhmg==", - "optional": true, + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.5.tgz", + "integrity": "sha512-C2stCULH0r54KBksv3AWcN8CLS3u9+WsEW8nBrvctrJ5rQTNa1waHkffpVaiKvcW2nP0aIMBPCobD/kYf/q9mA==", "dependencies": { - "@smithy/types": "^2.0.2", + "@smithy/types": "^2.2.2", "tslib": "^2.5.0" }, "engines": { @@ -3996,8 +4017,7 @@ "node_modules/@smithy/querystring-parser/node_modules/tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", - "optional": true + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/@smithy/signature-v4": { "version": "2.0.1", @@ -4046,10 +4066,9 @@ "optional": true }, "node_modules/@smithy/types": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.0.2.tgz", - "integrity": "sha512-wcymEjIXQ9+NEfE5Yt5TInAqe1o4n+Nh+rh00AwoazppmUt8tdo6URhc5gkDcOYrcvlDVAZE7uG69nDpEGUKxw==", - "optional": true, + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.2.2.tgz", + "integrity": "sha512-4PS0y1VxDnELGHGgBWlDksB2LJK8TG8lcvlWxIsgR+8vROI7Ms8h1P4FQUx+ftAX2QZv5g1CJCdhdRmQKyonyw==", "dependencies": { "tslib": "^2.5.0" }, @@ -4060,25 +4079,22 @@ "node_modules/@smithy/types/node_modules/tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", - "optional": true + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/@smithy/url-parser": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.1.tgz", - "integrity": "sha512-NpHVOAwddo+OyyIoujDL9zGL96piHWrTNXqltWmBvlUoWgt1HPyBuKs6oHjioyFnNZXUqveTOkEEq0U5w6Uv8A==", - "optional": true, + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.5.tgz", + "integrity": "sha512-OdMBvZhpckQSkugCXNJQCvqJ71wE7Ftxce92UOQLQ9pwF6hoS5PLL7wEfpnuEXtStzBqJYkzu1C1ZfjuFGOXAA==", "dependencies": { - "@smithy/querystring-parser": "^2.0.1", - "@smithy/types": "^2.0.2", + "@smithy/querystring-parser": "^2.0.5", + "@smithy/types": "^2.2.2", "tslib": "^2.5.0" } }, "node_modules/@smithy/url-parser/node_modules/tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", - "optional": true + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/@smithy/util-base64": { "version": "2.0.0", diff --git a/package.json b/package.json index 1657d4be9..0062c4d42 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "@balena/env-parsing": "^1.1.5", "@balena/es-version": "^1.0.2", "@balena/node-metrics-gatherer": "^6.0.3", - "@balena/pinejs": "^15.3.4", + "@balena/pinejs": "^15.3.8", + "@balena/pinejs-webresource-cloudfront": "^0.0.4", "@sentry/node": "^7.49.0", "@types/basic-auth": "^1.1.3", "@types/bluebird": "^3.5.38", diff --git a/src/fileupload-handler.ts b/src/fileupload-handler.ts new file mode 100644 index 000000000..dda78416e --- /dev/null +++ b/src/fileupload-handler.ts @@ -0,0 +1,102 @@ +import { webResourceHandler } from '@balena/pinejs'; +import { + CloudFrontHandler, + CloudFrontHandlerProps, +} from '@balena/pinejs-webresource-cloudfront'; +import * as fs from 'fs'; + +import { + WEBRESOURCES_S3_ACCESS_KEY, + WEBRESOURCES_S3_SECRET_KEY, + WEBRESOURCES_S3_REGION, + WEBRESOURCES_S3_HOST, + WEBRESOURCES_S3_BUCKET, + WEBRESOURCES_S3_MAX_FILESIZE, + WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH, + WEBRESOURCES_CLOUDFRONT_PUBLICKEY, + WEBRESOURCES_CLOUDFRONT_HOST, +} from './lib/config'; + +const getEndpointFromHost = (host: string): string => { + return host.startsWith('http') ? host : `https://${host}`; +}; + +const getS3Config = (): webResourceHandler.S3HandlerProps | undefined => { + if ( + WEBRESOURCES_S3_ACCESS_KEY != null && + WEBRESOURCES_S3_SECRET_KEY != null && + WEBRESOURCES_S3_REGION != null && + WEBRESOURCES_S3_HOST != null && + WEBRESOURCES_S3_BUCKET != null + ) { + return { + endpoint: getEndpointFromHost(WEBRESOURCES_S3_HOST), + accessKey: WEBRESOURCES_S3_ACCESS_KEY, + secretKey: WEBRESOURCES_S3_SECRET_KEY, + region: WEBRESOURCES_S3_REGION, + bucket: WEBRESOURCES_S3_BUCKET, + maxSize: WEBRESOURCES_S3_MAX_FILESIZE, + }; + } +}; + +const getCloudfrontConfig = (): CloudFrontHandlerProps | undefined => { + const s3Config = getS3Config(); + if ( + s3Config != null && + WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH != null && + WEBRESOURCES_CLOUDFRONT_PUBLICKEY != null && + WEBRESOURCES_CLOUDFRONT_HOST != null + ) { + let cfSecretKey: string; + try { + cfSecretKey = fs.readFileSync( + WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH, + 'utf-8', + ); + } catch (e) { + console.error('Failed to start cloudfront with error', e); + return; + } + + return { + cfDistributionDomain: getEndpointFromHost(WEBRESOURCES_CLOUDFRONT_HOST), + cfPublicKeyId: WEBRESOURCES_CLOUDFRONT_PUBLICKEY, + cfSecretKey, + ...s3Config, + }; + } +}; + +let handler: webResourceHandler.WebResourceHandler | undefined; +export const getFileUploadHandler = () => { + if (handler == null) { + const cfConfig = getCloudfrontConfig(); + if (cfConfig != null) { + handler = new CloudFrontHandler(cfConfig); + console.log('Successfully initialised webresource CloudFront handler.'); + console.log({ + region: cfConfig.region, + endpoint: cfConfig.endpoint, + bucket: cfConfig.bucket, + cfHost: cfConfig.cfDistributionDomain, + }); + return handler; + } + + const s3Config = getS3Config(); + if (s3Config != null) { + handler = new webResourceHandler.S3Handler(s3Config); + console.log('Successfully initialised webresource S3 handler.'); + console.log({ + region: s3Config.region, + endpoint: s3Config.endpoint, + bucket: s3Config.bucket, + }); + return handler; + } + + console.log('No webresource handler loaded.'); + } + return handler; +}; diff --git a/src/index.ts b/src/index.ts index 1f62777e5..3617e1e52 100644 --- a/src/index.ts +++ b/src/index.ts @@ -164,6 +164,7 @@ export * as scheduler from './infra/scheduler'; export * as cache from './infra/cache'; export * as config from './lib/config'; export * as abstractSql from './abstract-sql-utils'; +export { getFileUploadHandler } from './fileupload-handler'; export * as deviceState from './features/device-state'; export const errors = { diff --git a/src/lib/config.ts b/src/lib/config.ts index def2a5009..25ce84ea2 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -416,6 +416,29 @@ export const IGNORE_FROZEN_DEVICE_PERMISSIONS = boolVar( false, ); +export const WEBRESOURCES_S3_HOST = optionalVar('WEBRESOURCES_S3_HOST'); +export const WEBRESOURCES_S3_REGION = optionalVar('WEBRESOURCES_S3_REGION'); +export const WEBRESOURCES_S3_ACCESS_KEY = optionalVar( + 'WEBRESOURCES_S3_ACCESS_KEY', +); +export const WEBRESOURCES_S3_SECRET_KEY = optionalVar( + 'WEBRESOURCES_S3_SECRET_KEY', +); +export const WEBRESOURCES_S3_BUCKET = optionalVar('WEBRESOURCES_S3_BUCKET'); +export const WEBRESOURCES_S3_MAX_FILESIZE = intVar( + 'WEBRESOURCES_S3_MAX_FILESIZE', + 10000000, +); +export const WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH = optionalVar( + 'WEBRESOURCES_CLOUDFRONT_PRIVATEKEY_PATH', +); +export const WEBRESOURCES_CLOUDFRONT_PUBLICKEY = optionalVar( + 'WEBRESOURCES_CLOUDFRONT_PUBLICKEY', +); +export const WEBRESOURCES_CLOUDFRONT_HOST = optionalVar( + 'WEBRESOURCES_CLOUDFRONT_HOST', +); + /** * Splits an env var in the format of `${username}:${password}` * into a RedisAuth object. Auth is optional, so this can return