From bd48ccfda189d4e45f8a5e0f2653160986c11411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 11:32:22 +0100 Subject: [PATCH 01/21] feat: add CDN proxy function --- proxy/{ => handlers}/agent.ts | 40 ++++++------- proxy/{ => handlers}/ingress.ts | 4 +- proxy/headers.ts | 58 ------------------ proxy/index.ts | 12 +++- proxy/utils/cacheControl.ts | 26 ++++++++ proxy/{ => utils}/config.ts | 0 proxy/utils/headers.ts | 101 ++++++++++++++++++++++++++++++++ proxy/utils/traffic.ts | 14 +++++ 8 files changed, 170 insertions(+), 85 deletions(-) rename proxy/{ => handlers}/agent.ts (69%) rename proxy/{ => handlers}/ingress.ts (95%) delete mode 100644 proxy/headers.ts create mode 100644 proxy/utils/cacheControl.ts rename proxy/{ => utils}/config.ts (100%) create mode 100644 proxy/utils/headers.ts create mode 100644 proxy/utils/traffic.ts diff --git a/proxy/agent.ts b/proxy/handlers/agent.ts similarity index 69% rename from proxy/agent.ts rename to proxy/handlers/agent.ts index 61f28e17..f4cc5294 100644 --- a/proxy/agent.ts +++ b/proxy/handlers/agent.ts @@ -1,15 +1,15 @@ -import { HttpRequest } from '@azure/functions' -import { config } from './config' +import { HttpRequest, Logger } from '@azure/functions' +import { config } from '../utils/config' import * as https from 'https' -import { updateResponseHeaders } from './headers' +import { filterRequestHeaders, updateResponseHeaders } from '../utils/headers' import { HttpResponseSimple } from '@azure/functions/types/http' -function getEndpoint(apiKey: string | undefined, version: string, loaderVersion: string | undefined): string { - const lv: string = loaderVersion !== undefined && loaderVersion !== '' ? `/loader_v${loaderVersion}.js` : '' - return `/v${version}/${apiKey}${lv}` +export interface DownloadAgentParams { + httpRequest: HttpRequest + logger: Logger } -export async function downloadAgent(httpRequest: HttpRequest) { +export async function downloadAgent({ httpRequest, logger }: DownloadAgentParams) { const apiKey = httpRequest.query.apiKey const version = httpRequest.query.version const loaderVersion = httpRequest.query.loaderVersion @@ -19,26 +19,19 @@ export async function downloadAgent(httpRequest: HttpRequest) { const url = new URL(`https://${config.fpdcdn}`) url.pathname = getEndpoint(apiKey, version, loaderVersion) - const headers = { - ...httpRequest.headers, - } + logger.verbose('Downloading agent from', url.toString()) - // TODO - Extract this into separate function - delete headers['host'] - delete headers['content-length'] - delete headers['transfer-encoding'] - delete headers['via'] + const headers = filterRequestHeaders(httpRequest.headers) return new Promise((resolve) => { const data: any[] = [] - console.debug('Downloading agent from', url.toString()) + logger.verbose('Downloading agent from', url.toString()) const request = https.request( url, { method: 'GET', - // TODO Filter headers headers, }, (response) => { @@ -56,7 +49,6 @@ export async function downloadAgent(httpRequest: HttpRequest) { resolve({ status: response.statusCode ? response.statusCode.toString() : '500', - // TODO Filter headers headers: updateResponseHeaders(response.headers, domain), body: new Uint8Array(body), isRaw: true, @@ -66,18 +58,22 @@ export async function downloadAgent(httpRequest: HttpRequest) { ) request.on('error', (error) => { - console.error('unable to download agent', { error }) + logger.error('unable to download agent', { error }) resolve({ status: '500', headers: { - 'Content-Type': 'application/json', + 'Content-Type': 'text/plain', }, - // TODO Generate error response with our integrations format - body: error, + body: 'error', }) }) request.end() }) } + +function getEndpoint(apiKey: string | undefined, version: string, loaderVersion: string | undefined): string { + const lv: string = loaderVersion !== undefined && loaderVersion !== '' ? `/loader_v${loaderVersion}.js` : '' + return `/v${version}/${apiKey}${lv}` +} diff --git a/proxy/ingress.ts b/proxy/handlers/ingress.ts similarity index 95% rename from proxy/ingress.ts rename to proxy/handlers/ingress.ts index 0116ea9f..7ace9f1f 100644 --- a/proxy/ingress.ts +++ b/proxy/handlers/ingress.ts @@ -1,7 +1,7 @@ import { HttpRequest } from '@azure/functions' -import { config } from './config' +import { config } from '../utils/config' import * as https from 'https' -import { updateResponseHeaders } from './headers' +import { updateResponseHeaders } from '../utils/headers' import { HttpResponseSimple } from '@azure/functions/types/http' export function handleIngress(httpRequest: HttpRequest) { diff --git a/proxy/headers.ts b/proxy/headers.ts deleted file mode 100644 index 211b37b7..00000000 --- a/proxy/headers.ts +++ /dev/null @@ -1,58 +0,0 @@ -import * as http from 'http' -import { HttpResponseHeaders } from '@azure/functions/types/http' - -const COOKIE_HEADER_NAME = 'set-cookie' -const ALLOWED_RESPONSE_HEADERS = [ - 'access-control-allow-credentials', - 'access-control-allow-origin', - 'access-control-expose-headers', - 'content-encoding', - 'content-type', - 'cross-origin-resource-policy', - 'etag', - 'vary', -] - -export function updateResponseHeaders(headers: http.IncomingHttpHeaders, domain: string): HttpResponseHeaders { - const result: HttpResponseHeaders = {} - - for (const key in headers) { - if (ALLOWED_RESPONSE_HEADERS.includes(key)) { - const headerValue = headers[key] - - if (headerValue) { - result[key] = Array.isArray(headerValue) ? headerValue.join(' ') : headerValue - } - } - } - - if (headers[COOKIE_HEADER_NAME] !== undefined) { - result[COOKIE_HEADER_NAME] = adjustCookies(headers[COOKIE_HEADER_NAME], domain) - } - - return result -} - -export function adjustCookies(cookies: string[], domainName: string): string { - const newCookies: string[] = [] - cookies.forEach((it) => { - const parts: string[] = it.split(';') - - parts.map((v: string) => { - const s = v.trim() - const ind = s.indexOf('=') - if (ind !== -1) { - const key = s.substring(0, ind) - let value = s.substring(ind + 1) - if (key.toLowerCase() === 'domain') { - value = domainName - } - newCookies.push(`${key}=${value}`) - } else { - newCookies.push(s) - } - }) - }) - - return newCookies.join('; ').trim() -} diff --git a/proxy/index.ts b/proxy/index.ts index e4afd165..ed61d76a 100644 --- a/proxy/index.ts +++ b/proxy/index.ts @@ -1,13 +1,19 @@ import { AzureFunction, Context, HttpRequest } from '@azure/functions' -import { downloadAgent } from './agent' -import { handleIngress } from './ingress' +import { downloadAgent } from './handlers/agent' +import { handleIngress } from './handlers/ingress' const httpTrigger: AzureFunction = async (context: Context, req: HttpRequest): Promise => { + context.log.verbose('Handling request', { + req, + context, + }) + const path = req.params?.restOfPath + // TODO Resolve paths using customer variables switch (path) { case 'client': { - context.res = await downloadAgent(req) + context.res = await downloadAgent({ httpRequest: req, logger: context.log }) break } diff --git a/proxy/utils/cacheControl.ts b/proxy/utils/cacheControl.ts new file mode 100644 index 00000000..c746ecc4 --- /dev/null +++ b/proxy/utils/cacheControl.ts @@ -0,0 +1,26 @@ +const CACHE_MAX_AGE = 3600 +const SHARED_CACHE_MAX_AGE = 60 + +export function updateCacheControlHeader(headerValue: string): string { + let result = updateCacheControlAge(headerValue, 'max-age', CACHE_MAX_AGE) + result = updateCacheControlAge(result, 's-maxage', SHARED_CACHE_MAX_AGE) + + return result +} + +function updateCacheControlAge(headerValue: string, type: 'max-age' | 's-maxage', cacheMaxAge: number): string { + const cacheControlDirectives = headerValue.split(', ') + const maxAgeIndex = cacheControlDirectives.findIndex( + (directive) => directive.split('=')[0].trim().toLowerCase() === type, + ) + + if (maxAgeIndex === -1) { + cacheControlDirectives.push(`${type}=${cacheMaxAge}`) + } else { + const oldMaxAge = Number(cacheControlDirectives[maxAgeIndex].split('=')[1]) + const newMaxAge = Math.min(cacheMaxAge, oldMaxAge) + cacheControlDirectives[maxAgeIndex] = `${type}=${newMaxAge}` + } + + return cacheControlDirectives.join(', ') +} diff --git a/proxy/config.ts b/proxy/utils/config.ts similarity index 100% rename from proxy/config.ts rename to proxy/utils/config.ts diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts new file mode 100644 index 00000000..6cc21826 --- /dev/null +++ b/proxy/utils/headers.ts @@ -0,0 +1,101 @@ +import * as http from 'http' +import { HttpRequestHeaders, HttpResponseHeaders } from '@azure/functions' +import { updateCacheControlHeader } from './cacheControl' + +const COOKIE_HEADER_NAME = 'set-cookie' +const CACHE_CONTROL_HEADER_NAME = 'cache-control' +const ALLOWED_RESPONSE_HEADERS = [ + 'access-control-allow-credentials', + 'access-control-allow-origin', + 'access-control-expose-headers', + 'content-encoding', + 'content-type', + 'cross-origin-resource-policy', + 'etag', + 'vary', +] +const BLACKLISTED_REQUEST_HEADERS = ['content-length', 'host', 'transfer-encoding', 'via'] + +export function filterRequestHeaders(headers: HttpRequestHeaders) { + return Object.entries(headers).reduce((result: { [key: string]: string }, [name, value]) => { + const headerName = name.toLowerCase() + + if (!BLACKLISTED_REQUEST_HEADERS.includes(headerName)) { + let headerValue = value[0] + + if (headerName === 'cookie') { + headerValue = headerValue.split(/; */).join('; ') + headerValue = filterCookie(headerValue, (key) => key === '_iidt') + } + + result[headerName] = headerValue + } + + return result + }, {}) +} + +export function updateResponseHeaders(headers: http.IncomingHttpHeaders, domain: string): HttpResponseHeaders { + const result: HttpResponseHeaders = {} + + for (const name of ALLOWED_RESPONSE_HEADERS) { + const value = headers[name] + + if (value) { + result[name] = value.toString() + } + } + + if (headers[COOKIE_HEADER_NAME]) { + result[COOKIE_HEADER_NAME] = adjustCookies(headers[COOKIE_HEADER_NAME], domain) + } + + if (headers[CACHE_CONTROL_HEADER_NAME]) { + result[CACHE_CONTROL_HEADER_NAME] = updateCacheControlHeader(headers[CACHE_CONTROL_HEADER_NAME]) + } + + return result +} + +export function filterCookie(cookie: string, filterPredicate: (key: string) => boolean): string { + const newCookie: string[] = [] + const parts = cookie.split(';') + + parts.forEach((it) => { + const s = it.trim() + const ind = s.indexOf('=') + if (ind !== -1) { + const key = s.substring(0, ind) + const value = s.substring(ind + 1) + if (filterPredicate(key)) { + newCookie.push(`${key}=${value}`) + } + } + }) + + return newCookie.join('; ').trim() +} + +export function adjustCookies(cookies: string[], domainName: string): string { + const newCookies: string[] = [] + cookies.forEach((it) => { + const parts: string[] = it.split(';') + + parts.map((v: string) => { + const s = v.trim() + const ind = s.indexOf('=') + if (ind !== -1) { + const key = s.substring(0, ind) + let value = s.substring(ind + 1) + if (key.toLowerCase() === 'domain') { + value = domainName + } + newCookies.push(`${key}=${value}`) + } else { + newCookies.push(s) + } + }) + }) + + return newCookies.join('; ').trim() +} diff --git a/proxy/utils/traffic.ts b/proxy/utils/traffic.ts new file mode 100644 index 00000000..05f3d027 --- /dev/null +++ b/proxy/utils/traffic.ts @@ -0,0 +1,14 @@ +const LAMBDA_FUNC_VERSION = '__lambda_func_version__' +const PARAM_NAME = 'ii' + +export function addTrafficMonitoringSearchParamsForProCDN(url: URL) { + url.searchParams.append(PARAM_NAME, getTrafficMonitoringValue('procdn')) +} + +export function addTrafficMonitoringSearchParamsForVisitorIdRequest(url: URL) { + url.searchParams.append(PARAM_NAME, getTrafficMonitoringValue('ingress')) +} + +function getTrafficMonitoringValue(type: 'procdn' | 'ingress'): string { + return `fingerprintjs-pro-cloudfront/${LAMBDA_FUNC_VERSION}/${type}` +} From 96c2a09b3a89ea84c88c8ba6535474414126c468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 11:32:28 +0100 Subject: [PATCH 02/21] test: add tests setup --- jest.config.js | 10 +++++++++- package.json | 1 + proxy/utils/cacheControl.test.ts | 15 +++++++++++++++ rollup.config.mjs | 4 +++- tsconfig.app.json | 14 ++++++++++++++ tsconfig.json | 15 ++++++++++----- tsconfig.test.json | 14 ++++++++++++++ yarn.lock | 20 ++++++++++---------- 8 files changed, 76 insertions(+), 17 deletions(-) create mode 100644 proxy/utils/cacheControl.test.ts create mode 100644 tsconfig.app.json create mode 100644 tsconfig.test.json diff --git a/jest.config.js b/jest.config.js index f073e589..53eb2770 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,8 +2,16 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', - testRegex: '/functions/.+test.tsx?$', + testRegex: '/proxy/.+test.tsx?$', passWithNoTests: true, collectCoverageFrom: ['./functions/**/**.ts', '!**/index.ts'], coverageReporters: ['lcov', 'json-summary', ['text', { file: 'coverage.txt', path: './' }]], + transform: { + '^.+\\.[tj]sx?$': [ + 'ts-jest', + { + tsconfig: '/tsconfig.test.json', + }, + ], + }, } diff --git a/package.json b/package.json index d0123dad..b1151a75 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@rollup/plugin-json": "^4.1.0", "@rollup/plugin-node-resolve": "^15.0.1", "@rollup/plugin-replace": "^5.0.1", + "@types/jest": "^29.4.1", "@types/node": "^16.x", "@typescript-eslint/eslint-plugin": "^5.44.0", "@typescript-eslint/parser": "^5.44.0", diff --git a/proxy/utils/cacheControl.test.ts b/proxy/utils/cacheControl.test.ts new file mode 100644 index 00000000..4ac97898 --- /dev/null +++ b/proxy/utils/cacheControl.test.ts @@ -0,0 +1,15 @@ +import { updateCacheControlHeader } from './cacheControl' + +describe('updateCacheControlHeader', () => { + test('adjust max-age to lower value', () => { + expect(updateCacheControlHeader('public, max-age=36000, s-maxage=36000')).toBe('public, max-age=3600, s-maxage=60') + }) + + test('keep existing smaller value', () => { + expect(updateCacheControlHeader('public, max-age=600, s-maxage=600')).toBe('public, max-age=600, s-maxage=60') + }) + + test('add max age if not exist', () => { + expect(updateCacheControlHeader('no-cache')).toBe('no-cache, max-age=3600, s-maxage=60') + }) +}) diff --git a/rollup.config.mjs b/rollup.config.mjs index e712eb8b..d219ac9e 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -45,7 +45,9 @@ function makeConfig(entryFile, artifactName, functionJsonPath) { ], }), jsonPlugin(), - typescript(), + typescript({ + tsconfig: 'tsconfig.app.json', + }), commonjs(), nodeResolve({ preferBuiltins: false, modulesOnly: true }), replace({ diff --git a/tsconfig.app.json b/tsconfig.app.json new file mode 100644 index 00000000..88becbc7 --- /dev/null +++ b/tsconfig.app.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "exclude": [ + "dist", + "node_modules", + "**/*.test.ts" + ], + "include": [ + "**/*.ts" + ] +} diff --git a/tsconfig.json b/tsconfig.json index 9432b61a..c3b77f0d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,11 +8,16 @@ "noImplicitReturns": true, "noImplicitAny": true, "resolveJsonModule": true, - "moduleResolution": "node" + "moduleResolution": "node", + "esModuleInterop": true }, - "exclude": [ - "dist", - "node_modules", - "**/*.test.ts" + "include": ["proxy", "management"], + "references": [ + { + "path": "./tsconfig.app.json" + }, + { + "path": "./tsconfig.test.json" + } ] } diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 00000000..eda9ef31 --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "types": ["node", "jest"] + }, + "extends": "./tsconfig.json", + "exclude": [ + "dist", + "node_modules", + "**/*.test.ts" + ], + "include": [ + "**/*.test.ts" + ] +} diff --git a/yarn.lock b/yarn.lock index dc2b53d1..4be17dbe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1022,14 +1022,6 @@ "@rollup/pluginutils" "^5.0.1" magic-string "^0.27.0" -"@rollup/plugin-typescript@^9.0.2": - version "9.0.2" - resolved "https://registry.yarnpkg.com/@rollup/plugin-typescript/-/plugin-typescript-9.0.2.tgz#c0cdfa39e267f306ff7316405a35406d5821eaa7" - integrity sha512-/sS93vmHUMjzDUsl5scNQr1mUlNE1QjBBvOhmRwJCH8k2RRhDIm3c977B3wdu3t3Ap17W6dDeXP3hj1P1Un1bA== - dependencies: - "@rollup/pluginutils" "^5.0.1" - resolve "^1.22.1" - "@rollup/pluginutils@^3.0.8": version "3.1.0" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b" @@ -1171,6 +1163,14 @@ dependencies: "@types/istanbul-lib-report" "*" +"@types/jest@^29.4.1": + version "29.4.1" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.4.1.tgz#d76cbd07e5a24c0f4e86c2c0f9ed9e4ed33af3c8" + integrity sha512-zDQSWXG+ZkEvs2zFFMszePhx4euKz+Yt3Gg1P+RHjfJBinTTr6L2DEyovO4V/WrKXuF0Dgn56GWGZPDa6TW9eQ== + dependencies: + expect "^29.0.0" + pretty-format "^29.0.0" + "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" @@ -2298,7 +2298,7 @@ exit@^0.1.2: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" integrity sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ== -expect@^29.5.0: +expect@^29.0.0, expect@^29.5.0: version "29.5.0" resolved "https://registry.yarnpkg.com/expect/-/expect-29.5.0.tgz#68c0509156cb2a0adb8865d413b137eeaae682f7" integrity sha512-yM7xqUrCO2JdpFo4XpM82t+PJBFybdqoQuJLDGeDX2ij8NZzqRHyu3Hp188/JX7SWqud+7t4MUdvcgGBICMHZg== @@ -4204,7 +4204,7 @@ prettier@^2.8.0: resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.4.tgz#34dd2595629bfbb79d344ac4a91ff948694463c3" integrity sha512-vIS4Rlc2FNh0BySk3Wkd6xmwxB0FpOndW5fisM5H8hsZSxU2VWVB5CWIkIjWvrHjIhxk2g3bfMKM87zNTrZddw== -pretty-format@^29.5.0: +pretty-format@^29.0.0, pretty-format@^29.5.0: version "29.5.0" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.5.0.tgz#283134e74f70e2e3e7229336de0e4fce94ccde5a" integrity sha512-V2mGkI31qdttvTFX7Mt4efOqHXqJWMu4/r66Xh3Z3BwZaPfPJgp6/gbwoujRpPUtfEF6AUUWx3Jim3GCw5g/Qw== From d9f2002e443bcd517b0e0b83ab32663c3903110c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 11:51:31 +0100 Subject: [PATCH 03/21] chore: improve headers logic --- proxy/utils/headers.test.ts | 98 +++++++++++++++++++++++++++++++++ proxy/utils/headers.ts | 104 +++++++++++++++++++++++++++--------- 2 files changed, 178 insertions(+), 24 deletions(-) create mode 100644 proxy/utils/headers.test.ts diff --git a/proxy/utils/headers.test.ts b/proxy/utils/headers.test.ts new file mode 100644 index 00000000..cb74e886 --- /dev/null +++ b/proxy/utils/headers.test.ts @@ -0,0 +1,98 @@ +import { HttpRequest } from '@azure/functions' +import { filterRequestHeaders, updateResponseHeaders } from './headers' +import { IncomingHttpHeaders } from 'http' + +const mockReq: HttpRequest = { + method: 'GET', + url: 'https://example.org/fpjs/client', + query: { + apiKey: 'ujKG34hUYKLJKJ1F', + version: '3', + loaderVersion: '3.6.2', + }, + headers: { + 'content-type': 'application/json', + 'content-length': '24354', + host: 'fpjs.sh', + 'transfer-encoding': 'br', + via: 'azure.com', + cookie: '_iidt=7A03Gwg; _vid_t=gEFRuIQlzYmv692/UL4GLA==', + 'x-custom-header': 'value123899', + 'x-edge-qqq': 'x-edge-qqq', + 'strict-transport-security': 'max-age=600', + }, + user: null, + params: {}, + get: jest.fn(), + parseFormBody: jest.fn(), +} + +describe('filterRequestHeaders', () => { + test('test filtering blackilisted headers', () => { + const headers = filterRequestHeaders(mockReq.headers) + + expect(headers.hasOwnProperty('content-length')).toBe(false) + expect(headers.hasOwnProperty('host')).toBe(false) + expect(headers.hasOwnProperty('transfer-encoding')).toBe(false) + expect(headers.hasOwnProperty('via')).toBe(false) + expect(headers['content-type']).toBe('application/json') + expect(headers['cookie']).toBe('_iidt=7A03Gwg') + expect(headers['x-custom-header']).toBe('value123899') + expect(headers.hasOwnProperty('x-edge-qqq')).toBe(false) + expect(headers.hasOwnProperty('strict-transport-security')).toBe(false) + }) +}) + +describe('updateResponseHeaders', () => { + test('correctly updates response headers', () => { + const headers: IncomingHttpHeaders = { + 'access-control-allow-credentials': 'true', + 'access-control-allow-origin': 'true', + 'access-control-expose-headers': 'true', + 'cache-control': 'public, max-age=40000, s-maxage=40000', + 'content-encoding': 'br', + 'content-length': '73892', + 'content-type': 'application/json', + 'cross-origin-resource-policy': 'cross-origin', + etag: 'dskjhfadsjk', + 'set-cookie': ['_iidf', 'HttpOnly', 'Domain=azure.net'], + vary: 'Accept-Encoding', + 'custom-header-1': 'gdfddfd', + 'x-edge-xxx': 'ery8u', + 'strict-transport-security': 'max-age=1000', + } + + const resultHeaders = updateResponseHeaders(headers, 'fpjs.sh') + + expect(resultHeaders.hasOwnProperty('custom-header-1')).toBe(true) + expect(resultHeaders.hasOwnProperty('content-length')).toBe(false) + expect(resultHeaders.hasOwnProperty('x-edge-xxx')).toBe(false) + expect(resultHeaders['cache-control']).toBe('public, max-age=3600, s-maxage=60') + expect(resultHeaders['set-cookie']).toBe('_iidf; HttpOnly; Domain=fpjs.sh') + expect(resultHeaders.hasOwnProperty('strict-transport-security')).toBe(false) + }) + + test('updates cache policy', () => { + const headers: IncomingHttpHeaders = { + 'access-control-allow-credentials': 'true', + 'access-control-allow-origin': 'true', + 'access-control-expose-headers': 'true', + 'cache-control': 'no-cache', + 'content-encoding': 'br', + 'content-length': '73892', + 'content-type': 'application/json', + 'cross-origin-resource-policy': 'cross-origin', + etag: 'dskjhfadsjk', + 'set-cookie': ['_iidf', 'HttpOnly', 'Domain=cloudfront.net'], + vary: 'Accept-Encoding', + 'custom-header-1': 'gdfddfd', + } + + const resultHeaders = updateResponseHeaders(headers, 'fpjs.sh') + + expect(resultHeaders.hasOwnProperty('custom-header-1')).toBe(true) + expect(resultHeaders.hasOwnProperty('content-length')).toBe(false) + expect(resultHeaders['cache-control']).toBe('no-cache, max-age=3600, s-maxage=60') + expect(resultHeaders['set-cookie']).toBe('_iidf; HttpOnly; Domain=fpjs.sh') + }) +}) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index 6cc21826..a5db0017 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -4,24 +4,47 @@ import { updateCacheControlHeader } from './cacheControl' const COOKIE_HEADER_NAME = 'set-cookie' const CACHE_CONTROL_HEADER_NAME = 'cache-control' -const ALLOWED_RESPONSE_HEADERS = [ - 'access-control-allow-credentials', - 'access-control-allow-origin', - 'access-control-expose-headers', - 'content-encoding', - 'content-type', - 'cross-origin-resource-policy', - 'etag', - 'vary', -] -const BLACKLISTED_REQUEST_HEADERS = ['content-length', 'host', 'transfer-encoding', 'via'] +// TODO Include azure headers +const BLACKLISTED_HEADERS_PREFIXES = ['x-edge-', 'x-amz-cf-'] + +const READ_ONLY_REQUEST_HEADERS = new Set(['content-length', 'host', 'transfer-encoding', 'via']) +const READ_ONLY_RESPONSE_HEADERS = new Set([ + 'accept-encoding', + 'content-length', + 'if-modified-since', + 'if-none-match', + 'if-range', + 'if-unmodified-since', + 'transfer-encoding', + 'via', +]) + +// TODO Include azure headers +const BLACKLISTED_HEADERS = new Set([ + 'connection', + 'expect', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'proxy-connection', + 'trailer', + 'upgrade', + 'x-accel-buffering', + 'x-accel-charset', + 'x-accel-limit-rate', + 'x-accel-redirect', + 'x-cache', + 'x-forwarded-proto', + 'x-real-ip', + 'strict-transport-security', +]) export function filterRequestHeaders(headers: HttpRequestHeaders) { return Object.entries(headers).reduce((result: { [key: string]: string }, [name, value]) => { const headerName = name.toLowerCase() - if (!BLACKLISTED_REQUEST_HEADERS.includes(headerName)) { - let headerValue = value[0] + if (isHeaderAllowedForRequest(headerName)) { + let headerValue = value if (headerName === 'cookie') { headerValue = headerValue.split(/; */).join('; ') @@ -38,20 +61,27 @@ export function filterRequestHeaders(headers: HttpRequestHeaders) { export function updateResponseHeaders(headers: http.IncomingHttpHeaders, domain: string): HttpResponseHeaders { const result: HttpResponseHeaders = {} - for (const name of ALLOWED_RESPONSE_HEADERS) { - const value = headers[name] - - if (value) { - result[name] = value.toString() + for (const [key, value] of Object.entries(headers)) { + if (!isHeaderAllowedForResponse(key) || !value) { + continue } - } - if (headers[COOKIE_HEADER_NAME]) { - result[COOKIE_HEADER_NAME] = adjustCookies(headers[COOKIE_HEADER_NAME], domain) - } + switch (key) { + case COOKIE_HEADER_NAME: { + result[COOKIE_HEADER_NAME] = adjustCookies(Array.isArray(value) ? value : [value], domain) + + break + } - if (headers[CACHE_CONTROL_HEADER_NAME]) { - result[CACHE_CONTROL_HEADER_NAME] = updateCacheControlHeader(headers[CACHE_CONTROL_HEADER_NAME]) + case CACHE_CONTROL_HEADER_NAME: { + result[CACHE_CONTROL_HEADER_NAME] = updateCacheControlHeader(value.toString()) + + break + } + + default: + result[key] = value.toString() + } } return result @@ -99,3 +129,29 @@ export function adjustCookies(cookies: string[], domainName: string): string { return newCookies.join('; ').trim() } + +function isHeaderAllowedForResponse(headerName: string) { + return ( + !READ_ONLY_RESPONSE_HEADERS.has(headerName) && + !BLACKLISTED_HEADERS.has(headerName) && + !matchesBlacklistedHeaderPrefix(headerName) + ) +} + +function isHeaderAllowedForRequest(headerName: string) { + return ( + !READ_ONLY_REQUEST_HEADERS.has(headerName) && + !BLACKLISTED_HEADERS.has(headerName) && + !matchesBlacklistedHeaderPrefix(headerName) + ) +} + +function matchesBlacklistedHeaderPrefix(headerName: string) { + for (const blacklistedHeaderPrefix of BLACKLISTED_HEADERS_PREFIXES) { + if (headerName.startsWith(blacklistedHeaderPrefix)) { + return true + } + } + + return false +} From 427cf1a7a287ffa48fb1a33dc21e6a41569f43e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 11:51:38 +0100 Subject: [PATCH 04/21] chore: use env vars for config --- proxy/utils/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/utils/config.ts b/proxy/utils/config.ts index e91e68c3..a66dafc6 100644 --- a/proxy/utils/config.ts +++ b/proxy/utils/config.ts @@ -1,5 +1,5 @@ // TODO - Extract these into ENV variables export const config = { - fpdcdn: 'fpcdn.io', - ingressApi: 'api.fpjs.io', + fpdcdn: '__FPCDN__', + ingressApi: '__INGRESS_API__', } From 25a94db8d6b9782ec4a79a9de1eb9880ed6e3551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:37:26 +0100 Subject: [PATCH 05/21] chore: add more tests, move cookies utils to separate file --- proxy/utils/config.ts | 1 - proxy/utils/cookies.test.ts | 73 +++++++++++++++++++++++++++++++++++++ proxy/utils/cookies.ts | 45 +++++++++++++++++++++++ proxy/utils/headers.test.ts | 25 ++++++++++++- proxy/utils/headers.ts | 46 ++--------------------- 5 files changed, 146 insertions(+), 44 deletions(-) create mode 100644 proxy/utils/cookies.test.ts create mode 100644 proxy/utils/cookies.ts diff --git a/proxy/utils/config.ts b/proxy/utils/config.ts index a66dafc6..50ed0a95 100644 --- a/proxy/utils/config.ts +++ b/proxy/utils/config.ts @@ -1,4 +1,3 @@ -// TODO - Extract these into ENV variables export const config = { fpdcdn: '__FPCDN__', ingressApi: '__INGRESS_API__', diff --git a/proxy/utils/cookies.test.ts b/proxy/utils/cookies.test.ts new file mode 100644 index 00000000..422aafd9 --- /dev/null +++ b/proxy/utils/cookies.test.ts @@ -0,0 +1,73 @@ +import { adjustCookies, filterCookie } from './cookies' + +const domain = 'fingerprint.com' + +describe('updateCookie', () => { + test('empty cookie', () => { + expect(adjustCookies([''], domain)).toBe('') + }) + + test('simple', () => { + const value = '_iidf' + expect(adjustCookies([value], domain)).toBe(value) + }) + + test('some non domain with =', () => { + const value = '_iidf; Value=x;' + expect(adjustCookies([value], domain)).toBe(value) + }) + + test('key=value=with=equal=sign', () => { + const value = 'key=value=with=equal=sign' + expect(adjustCookies([value], domain)).toBe(value) + }) + + test('with equal signs', () => { + const value = ['_iidt=7A03Gwg==', '_vid_t=gEFRuIQlzYmv692/UL4GLA=='] + expect(adjustCookies(value, domain)).toBe('_iidt=7A03Gwg==; _vid_t=gEFRuIQlzYmv692/UL4GLA==') + }) + + test('without domain', () => { + const initialValue = + 'iidt,dEbSJkkvz8Yiv4eFAGbOJWPL69y0Z8Z8vnpEk/mJkaZ4hXmM+zb+8iWRy1j6IuqK5Fq1BnLRi2BC/Q,,; ' + + 'Path,/; Expires,Fri, 27 Oct 2023 19:17:51 GMT; HttpOnly; Secure; SameSite,None' + expect(adjustCookies([initialValue], domain)).toBe(initialValue) + }) + + test('update domain', () => { + const initialValue = + 'iidt,dEbSJkkvz8Yiv4eFAGbOJWPL69y0Z8Z8vnpEk/mJkaZ4hXmM+zb+8iWRy1j6IuqK5Fq1BnLRi2BC/Q,,; ' + + 'Path,/; Domain=hfdgjkjds.cloudfront.net; Expires,Fri, 27 Oct 2023 19:17:51 GMT; HttpOnly; Secure; SameSite,None' + const expectedValue = + 'iidt,dEbSJkkvz8Yiv4eFAGbOJWPL69y0Z8Z8vnpEk/mJkaZ4hXmM+zb+8iWRy1j6IuqK5Fq1BnLRi2BC/Q,,; ' + + 'Path,/; Domain=fingerprint.com; Expires,Fri, 27 Oct 2023 19:17:51 GMT; HttpOnly; Secure; SameSite,None' + expect(adjustCookies([initialValue], domain)).toBe(expectedValue) + }) +}) + +describe('filterCookies', () => { + const predicate = (key: string) => key === '_iidt' + + test('the same result', () => { + const value = '_iidt=sfdsafdasf' + expect(filterCookie(value, predicate)).toBe(value) + }) + + test('reduce', () => { + const value = '_iidt=aass; vid_t=xcvbnm' + expect(filterCookie(value, predicate)).toBe('_iidt=aass') + }) + + test('empty', () => { + expect(filterCookie('', predicate)).toBe('') + }) + + test('no value', () => { + expect(filterCookie('_iidt', predicate)).toBe('') + }) + + test('with equal signs', () => { + const value = '_iidt=7A03Gwg==; _vid_t=gEFRuIQlzYmv692/UL4GLA==' + expect(filterCookie(value, predicate)).toBe('_iidt=7A03Gwg==') + }) +}) diff --git a/proxy/utils/cookies.ts b/proxy/utils/cookies.ts new file mode 100644 index 00000000..2fdf4455 --- /dev/null +++ b/proxy/utils/cookies.ts @@ -0,0 +1,45 @@ +export function filterCookie(cookie: string, filterPredicate: (key: string) => boolean): string { + const newCookie: string[] = [] + const parts = cookie.split(';') + + parts.forEach((cookie) => { + const trimmedCookie = cookie.trim() + const index = trimmedCookie.indexOf('=') + + if (index !== -1) { + const key = trimmedCookie.substring(0, index) + const value = trimmedCookie.substring(index + 1) + if (filterPredicate(key)) { + newCookie.push(`${key}=${value}`) + } + } + }) + + return newCookie.join('; ').trim() +} + +export function adjustCookies(cookies: string[], domainName: string): string { + const newCookies: string[] = [] + + cookies.forEach((cookie) => { + const parts: string[] = cookie.split(';') + + parts.map((rawValue: string) => { + const trimmedValue = rawValue.trim() + const index = trimmedValue.indexOf('=') + + if (index !== -1) { + const key = trimmedValue.substring(0, index) + let value = trimmedValue.substring(index + 1) + if (key.toLowerCase() === 'domain') { + value = domainName + } + newCookies.push(`${key}=${value}`) + } else { + newCookies.push(trimmedValue) + } + }) + }) + + return newCookies.join('; ').trim() +} diff --git a/proxy/utils/headers.test.ts b/proxy/utils/headers.test.ts index cb74e886..c76078b5 100644 --- a/proxy/utils/headers.test.ts +++ b/proxy/utils/headers.test.ts @@ -1,5 +1,5 @@ import { HttpRequest } from '@azure/functions' -import { filterRequestHeaders, updateResponseHeaders } from './headers' +import { filterRequestHeaders, getHost, updateResponseHeaders } from './headers' import { IncomingHttpHeaders } from 'http' const mockReq: HttpRequest = { @@ -96,3 +96,26 @@ describe('updateResponseHeaders', () => { expect(resultHeaders['set-cookie']).toBe('_iidf; HttpOnly; Domain=fpjs.sh') }) }) + +describe('getHost', () => { + it.each([ + [ + { + headers: { + 'x-forwarded-host': 'fpjs.sh', + }, + }, + 'fpjs.sh', + ], + [ + { + headers: { + host: 'fpjs.sh', + }, + }, + 'fpjs.sh', + ], + ])('returns correct host', (request, expectedHost) => { + expect(getHost(request)).toBe(expectedHost) + }) +}) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index a5db0017..fc81bf6b 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -1,6 +1,7 @@ import * as http from 'http' -import { HttpRequestHeaders, HttpResponseHeaders } from '@azure/functions' +import { HttpRequest, HttpRequestHeaders, HttpResponseHeaders } from '@azure/functions' import { updateCacheControlHeader } from './cacheControl' +import { adjustCookies, filterCookie } from './cookies' const COOKIE_HEADER_NAME = 'set-cookie' const CACHE_CONTROL_HEADER_NAME = 'cache-control' @@ -87,47 +88,8 @@ export function updateResponseHeaders(headers: http.IncomingHttpHeaders, domain: return result } -export function filterCookie(cookie: string, filterPredicate: (key: string) => boolean): string { - const newCookie: string[] = [] - const parts = cookie.split(';') - - parts.forEach((it) => { - const s = it.trim() - const ind = s.indexOf('=') - if (ind !== -1) { - const key = s.substring(0, ind) - const value = s.substring(ind + 1) - if (filterPredicate(key)) { - newCookie.push(`${key}=${value}`) - } - } - }) - - return newCookie.join('; ').trim() -} - -export function adjustCookies(cookies: string[], domainName: string): string { - const newCookies: string[] = [] - cookies.forEach((it) => { - const parts: string[] = it.split(';') - - parts.map((v: string) => { - const s = v.trim() - const ind = s.indexOf('=') - if (ind !== -1) { - const key = s.substring(0, ind) - let value = s.substring(ind + 1) - if (key.toLowerCase() === 'domain') { - value = domainName - } - newCookies.push(`${key}=${value}`) - } else { - newCookies.push(s) - } - }) - }) - - return newCookies.join('; ').trim() +export function getHost(request: Pick) { + return request.headers['x-forwarded-host'] || request.headers.host } function isHeaderAllowedForResponse(headerName: string) { From 3ef10708a4f955667864545b95fa69ada2d580ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:37:32 +0100 Subject: [PATCH 06/21] build: add local settings --- .gitignore | 3 +-- local.settings.json | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 local.settings.json diff --git a/.gitignore b/.gitignore index 34882af7..bc0dd092 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ typings/ .yarn-integrity # dotenv environment variables file -.env.test +.env # parcel-bundler cache (https://parceljs.org/) .cache @@ -113,6 +113,5 @@ stack.json bin obj appsettings.json -local.settings.json storage/* !storage/.gitkeep diff --git a/local.settings.json b/local.settings.json new file mode 100644 index 00000000..beebb98f --- /dev/null +++ b/local.settings.json @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "node", + "StorageConnectionString": "UseDevelopmentStorage=true" + } +} From af56a4fcf846a8cb7b0f2f13aad732c107c5cf07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:40:02 +0100 Subject: [PATCH 07/21] chore: add trafic tests, fix coverage --- jest.config.js | 2 +- proxy/utils/traffic.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 proxy/utils/traffic.test.ts diff --git a/jest.config.js b/jest.config.js index 53eb2770..6c732556 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,7 +4,7 @@ module.exports = { testEnvironment: 'node', testRegex: '/proxy/.+test.tsx?$', passWithNoTests: true, - collectCoverageFrom: ['./functions/**/**.ts', '!**/index.ts'], + collectCoverageFrom: ['./proxy/**/**.ts', '!**/index.ts', '!**/config.ts', './management/**/**.ts'], coverageReporters: ['lcov', 'json-summary', ['text', { file: 'coverage.txt', path: './' }]], transform: { '^.+\\.[tj]sx?$': [ diff --git a/proxy/utils/traffic.test.ts b/proxy/utils/traffic.test.ts new file mode 100644 index 00000000..cbbbafc1 --- /dev/null +++ b/proxy/utils/traffic.test.ts @@ -0,0 +1,24 @@ +import { + addTrafficMonitoringSearchParamsForProCDN, + addTrafficMonitoringSearchParamsForVisitorIdRequest, +} from './traffic' + +describe('test procdn call', () => { + test('test', () => { + const url = new URL('https://fpjs.sh/agent?smth') + addTrafficMonitoringSearchParamsForProCDN(url) + + const param = url.searchParams.get('ii') + expect(param).toBe('fingerprintjs-pro-cloudfront/__lambda_func_version__/procdn') + }) +}) + +describe('test visitor call', () => { + test('test', () => { + const url = new URL('https://fpjs.sh/visitorId?smth') + addTrafficMonitoringSearchParamsForVisitorIdRequest(url) + + const param = url.searchParams.get('ii') + expect(param).toBe('fingerprintjs-pro-cloudfront/__lambda_func_version__/ingress') + }) +}) From 5cfff4b2fd97e5d1607c3e8dd5dd8fbcb2bc5e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:40:34 +0100 Subject: [PATCH 08/21] ci: add coverage actions --- .github/workflows/coverage-diff.yml | 31 ++++++++++++++++++ .github/workflows/coverage-report.yml | 46 +++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 .github/workflows/coverage-diff.yml create mode 100644 .github/workflows/coverage-report.yml diff --git a/.github/workflows/coverage-diff.yml b/.github/workflows/coverage-diff.yml new file mode 100644 index 00000000..20e91f8c --- /dev/null +++ b/.github/workflows/coverage-diff.yml @@ -0,0 +1,31 @@ +name: Check coverage for PR + +on: + pull_request: + +jobs: + run-tests-check-coverage: + runs-on: ubuntu-20.04 + name: Run tests & check coverage + steps: + - uses: actions/checkout@v3 + - name: Jest coverage comment + id: coverage + uses: ArtiomTr/jest-coverage-report-action@9f733792c44d05327cb371766bf78a5261e43936 + with: + package-manager: yarn + output: report-markdown + - name: Read coverage text report + uses: fingerprintjs/action-coverage-report-md@v1 + id: coverage-md + with: + srcBasePath: './src' + - uses: marocchino/sticky-pull-request-comment@adca94abcaf73c10466a71cc83ae561fd66d1a56 + with: + message: | + ${{ steps.coverage.outputs.report }} +
+ Show full coverage report + + ${{ steps.coverage-md.outputs.markdownReport }} +
diff --git a/.github/workflows/coverage-report.yml b/.github/workflows/coverage-report.yml new file mode 100644 index 00000000..1745fcd5 --- /dev/null +++ b/.github/workflows/coverage-report.yml @@ -0,0 +1,46 @@ +name: Coverage + +on: + push: + branches: + - main + +jobs: + build-and-run-tests: + runs-on: ubuntu-20.04 + name: Build & run tests & publish coverage + steps: + - uses: actions/checkout@v3 + + - name: Install node + uses: actions/setup-node@v3 + with: + node-version-file: '.node-version' + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT + - uses: actions/cache@v3 + id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: Install Dependencies and prepare packages + run: yarn install + env: + CI: true + - name: Run test + run: yarn test + + - name: Create Coverage Badges + uses: jaywcjlove/coverage-badges-cli@e07f25709cd25486855c1ba1b26da53576ff3620 + with: + source: coverage/coverage-summary.json + output: coverage/lcov-report/badges.svg + + - name: Deploy + uses: JamesIves/github-pages-deploy-action@8817a56e5bfec6e2b08345c81f4d422db53a2cdc + with: + branch: gh-pages + folder: ./coverage/lcov-report/ From f99151a955074cab350a5aaabed50c940ec08c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:42:06 +0100 Subject: [PATCH 09/21] chore: remove local settings --- .gitignore | 1 + local.settings.json | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 local.settings.json diff --git a/.gitignore b/.gitignore index bc0dd092..3c7283c3 100644 --- a/.gitignore +++ b/.gitignore @@ -113,5 +113,6 @@ stack.json bin obj appsettings.json +local.settings.json storage/* !storage/.gitkeep diff --git a/local.settings.json b/local.settings.json deleted file mode 100644 index beebb98f..00000000 --- a/local.settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "node", - "StorageConnectionString": "UseDevelopmentStorage=true" - } -} From e30a24f546ad53f8092e6a6f586c197bde57b41f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:46:32 +0100 Subject: [PATCH 10/21] chore: remove duplicate logs --- proxy/handlers/agent.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/proxy/handlers/agent.ts b/proxy/handlers/agent.ts index f4cc5294..4b3a9dfd 100644 --- a/proxy/handlers/agent.ts +++ b/proxy/handlers/agent.ts @@ -26,8 +26,6 @@ export async function downloadAgent({ httpRequest, logger }: DownloadAgentParams return new Promise((resolve) => { const data: any[] = [] - logger.verbose('Downloading agent from', url.toString()) - const request = https.request( url, { From 3c278779ae57bc3f814a3317b64669bb8d42fe35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 12:58:30 +0100 Subject: [PATCH 11/21] chore: update filtered headers --- proxy/utils/headers.test.ts | 4 ++++ proxy/utils/headers.ts | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/proxy/utils/headers.test.ts b/proxy/utils/headers.test.ts index c76078b5..462c7a9c 100644 --- a/proxy/utils/headers.test.ts +++ b/proxy/utils/headers.test.ts @@ -20,6 +20,8 @@ const mockReq: HttpRequest = { 'x-custom-header': 'value123899', 'x-edge-qqq': 'x-edge-qqq', 'strict-transport-security': 'max-age=600', + 'x-azure-requestchain': 'hops=1', + 'x-azure-socketip': '46.204.4.119', }, user: null, params: {}, @@ -40,6 +42,8 @@ describe('filterRequestHeaders', () => { expect(headers['x-custom-header']).toBe('value123899') expect(headers.hasOwnProperty('x-edge-qqq')).toBe(false) expect(headers.hasOwnProperty('strict-transport-security')).toBe(false) + expect(headers.hasOwnProperty('x-azure-requestchain')).toBe(false) + expect(headers.hasOwnProperty('x-azure-socketip')).toBe(false) }) }) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index fc81bf6b..3ce0d707 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -5,10 +5,11 @@ import { adjustCookies, filterCookie } from './cookies' const COOKIE_HEADER_NAME = 'set-cookie' const CACHE_CONTROL_HEADER_NAME = 'cache-control' -// TODO Include azure headers -const BLACKLISTED_HEADERS_PREFIXES = ['x-edge-', 'x-amz-cf-'] -const READ_ONLY_REQUEST_HEADERS = new Set(['content-length', 'host', 'transfer-encoding', 'via']) +// Azure specific headers +const BLACKLISTED_HEADERS_PREFIXES = ['x-edge-', 'x-arr-', 'x-site', 'x-azure-'] + +const READ_ONLY_REQUEST_HEADERS = new Set(['content-length', 'host', 'transfer-encoding', 'via', 'disguised-host']) const READ_ONLY_RESPONSE_HEADERS = new Set([ 'accept-encoding', 'content-length', @@ -20,7 +21,6 @@ const READ_ONLY_RESPONSE_HEADERS = new Set([ 'via', ]) -// TODO Include azure headers const BLACKLISTED_HEADERS = new Set([ 'connection', 'expect', From b5df2c232a64b9bb7cf43dc9d1dd29c414abe76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 13:14:27 +0100 Subject: [PATCH 12/21] chore: don't filter most headers --- proxy/utils/headers.test.ts | 14 ++++++------ proxy/utils/headers.ts | 44 ++++--------------------------------- 2 files changed, 11 insertions(+), 47 deletions(-) diff --git a/proxy/utils/headers.test.ts b/proxy/utils/headers.test.ts index 462c7a9c..f2580569 100644 --- a/proxy/utils/headers.test.ts +++ b/proxy/utils/headers.test.ts @@ -2,7 +2,7 @@ import { HttpRequest } from '@azure/functions' import { filterRequestHeaders, getHost, updateResponseHeaders } from './headers' import { IncomingHttpHeaders } from 'http' -const mockReq: HttpRequest = { +const mockReq = { method: 'GET', url: 'https://example.org/fpjs/client', query: { @@ -27,16 +27,16 @@ const mockReq: HttpRequest = { params: {}, get: jest.fn(), parseFormBody: jest.fn(), -} +} satisfies HttpRequest describe('filterRequestHeaders', () => { test('test filtering blackilisted headers', () => { const headers = filterRequestHeaders(mockReq.headers) - expect(headers.hasOwnProperty('content-length')).toBe(false) + expect(headers.hasOwnProperty('content-length')).toBe(true) expect(headers.hasOwnProperty('host')).toBe(false) - expect(headers.hasOwnProperty('transfer-encoding')).toBe(false) - expect(headers.hasOwnProperty('via')).toBe(false) + expect(headers.hasOwnProperty('transfer-encoding')).toBe(true) + expect(headers.hasOwnProperty('via')).toBe(true) expect(headers['content-type']).toBe('application/json') expect(headers['cookie']).toBe('_iidt=7A03Gwg') expect(headers['x-custom-header']).toBe('value123899') @@ -69,7 +69,7 @@ describe('updateResponseHeaders', () => { const resultHeaders = updateResponseHeaders(headers, 'fpjs.sh') expect(resultHeaders.hasOwnProperty('custom-header-1')).toBe(true) - expect(resultHeaders.hasOwnProperty('content-length')).toBe(false) + expect(resultHeaders.hasOwnProperty('content-length')).toBe(true) expect(resultHeaders.hasOwnProperty('x-edge-xxx')).toBe(false) expect(resultHeaders['cache-control']).toBe('public, max-age=3600, s-maxage=60') expect(resultHeaders['set-cookie']).toBe('_iidf; HttpOnly; Domain=fpjs.sh') @@ -95,7 +95,7 @@ describe('updateResponseHeaders', () => { const resultHeaders = updateResponseHeaders(headers, 'fpjs.sh') expect(resultHeaders.hasOwnProperty('custom-header-1')).toBe(true) - expect(resultHeaders.hasOwnProperty('content-length')).toBe(false) + expect(resultHeaders.hasOwnProperty('content-length')).toBe(true) expect(resultHeaders['cache-control']).toBe('no-cache, max-age=3600, s-maxage=60') expect(resultHeaders['set-cookie']).toBe('_iidf; HttpOnly; Domain=fpjs.sh') }) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index 3ce0d707..e82dd7d0 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -9,36 +9,8 @@ const CACHE_CONTROL_HEADER_NAME = 'cache-control' // Azure specific headers const BLACKLISTED_HEADERS_PREFIXES = ['x-edge-', 'x-arr-', 'x-site', 'x-azure-'] -const READ_ONLY_REQUEST_HEADERS = new Set(['content-length', 'host', 'transfer-encoding', 'via', 'disguised-host']) -const READ_ONLY_RESPONSE_HEADERS = new Set([ - 'accept-encoding', - 'content-length', - 'if-modified-since', - 'if-none-match', - 'if-range', - 'if-unmodified-since', - 'transfer-encoding', - 'via', -]) - -const BLACKLISTED_HEADERS = new Set([ - 'connection', - 'expect', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'proxy-connection', - 'trailer', - 'upgrade', - 'x-accel-buffering', - 'x-accel-charset', - 'x-accel-limit-rate', - 'x-accel-redirect', - 'x-cache', - 'x-forwarded-proto', - 'x-real-ip', - 'strict-transport-security', -]) +const BLACKLISTED_REQUEST_HEADERS = new Set(['host', 'strict-transport-security']) +const BLACKLISTED_RESPONSE_HEADERS = new Set(['strict-transport-security']) export function filterRequestHeaders(headers: HttpRequestHeaders) { return Object.entries(headers).reduce((result: { [key: string]: string }, [name, value]) => { @@ -93,19 +65,11 @@ export function getHost(request: Pick) { } function isHeaderAllowedForResponse(headerName: string) { - return ( - !READ_ONLY_RESPONSE_HEADERS.has(headerName) && - !BLACKLISTED_HEADERS.has(headerName) && - !matchesBlacklistedHeaderPrefix(headerName) - ) + return !BLACKLISTED_RESPONSE_HEADERS.has(headerName) && !matchesBlacklistedHeaderPrefix(headerName) } function isHeaderAllowedForRequest(headerName: string) { - return ( - !READ_ONLY_REQUEST_HEADERS.has(headerName) && - !BLACKLISTED_HEADERS.has(headerName) && - !matchesBlacklistedHeaderPrefix(headerName) - ) + return !BLACKLISTED_REQUEST_HEADERS.has(headerName) && !matchesBlacklistedHeaderPrefix(headerName) } function matchesBlacklistedHeaderPrefix(headerName: string) { From c2c08ac11845ae4f6129d34770ba485e5c2d1b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 13:48:46 +0100 Subject: [PATCH 13/21] chore: update domain to azure --- proxy/utils/cookies.test.ts | 2 +- proxy/utils/headers.test.ts | 2 +- proxy/utils/traffic.test.ts | 4 ++-- proxy/utils/traffic.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/proxy/utils/cookies.test.ts b/proxy/utils/cookies.test.ts index 422aafd9..d36bf848 100644 --- a/proxy/utils/cookies.test.ts +++ b/proxy/utils/cookies.test.ts @@ -37,7 +37,7 @@ describe('updateCookie', () => { test('update domain', () => { const initialValue = 'iidt,dEbSJkkvz8Yiv4eFAGbOJWPL69y0Z8Z8vnpEk/mJkaZ4hXmM+zb+8iWRy1j6IuqK5Fq1BnLRi2BC/Q,,; ' + - 'Path,/; Domain=hfdgjkjds.cloudfront.net; Expires,Fri, 27 Oct 2023 19:17:51 GMT; HttpOnly; Secure; SameSite,None' + 'Path,/; Domain=hfdgjkjds.azure.net; Expires,Fri, 27 Oct 2023 19:17:51 GMT; HttpOnly; Secure; SameSite,None' const expectedValue = 'iidt,dEbSJkkvz8Yiv4eFAGbOJWPL69y0Z8Z8vnpEk/mJkaZ4hXmM+zb+8iWRy1j6IuqK5Fq1BnLRi2BC/Q,,; ' + 'Path,/; Domain=fingerprint.com; Expires,Fri, 27 Oct 2023 19:17:51 GMT; HttpOnly; Secure; SameSite,None' diff --git a/proxy/utils/headers.test.ts b/proxy/utils/headers.test.ts index f2580569..81946ec2 100644 --- a/proxy/utils/headers.test.ts +++ b/proxy/utils/headers.test.ts @@ -87,7 +87,7 @@ describe('updateResponseHeaders', () => { 'content-type': 'application/json', 'cross-origin-resource-policy': 'cross-origin', etag: 'dskjhfadsjk', - 'set-cookie': ['_iidf', 'HttpOnly', 'Domain=cloudfront.net'], + 'set-cookie': ['_iidf', 'HttpOnly', 'Domain=azure.net'], vary: 'Accept-Encoding', 'custom-header-1': 'gdfddfd', } diff --git a/proxy/utils/traffic.test.ts b/proxy/utils/traffic.test.ts index cbbbafc1..663f4000 100644 --- a/proxy/utils/traffic.test.ts +++ b/proxy/utils/traffic.test.ts @@ -9,7 +9,7 @@ describe('test procdn call', () => { addTrafficMonitoringSearchParamsForProCDN(url) const param = url.searchParams.get('ii') - expect(param).toBe('fingerprintjs-pro-cloudfront/__lambda_func_version__/procdn') + expect(param).toBe('fingerprintjs-pro-azure/__lambda_func_version__/procdn') }) }) @@ -19,6 +19,6 @@ describe('test visitor call', () => { addTrafficMonitoringSearchParamsForVisitorIdRequest(url) const param = url.searchParams.get('ii') - expect(param).toBe('fingerprintjs-pro-cloudfront/__lambda_func_version__/ingress') + expect(param).toBe('fingerprintjs-pro-azure/__lambda_func_version__/ingress') }) }) diff --git a/proxy/utils/traffic.ts b/proxy/utils/traffic.ts index 05f3d027..35bbcc9b 100644 --- a/proxy/utils/traffic.ts +++ b/proxy/utils/traffic.ts @@ -10,5 +10,5 @@ export function addTrafficMonitoringSearchParamsForVisitorIdRequest(url: URL) { } function getTrafficMonitoringValue(type: 'procdn' | 'ingress'): string { - return `fingerprintjs-pro-cloudfront/${LAMBDA_FUNC_VERSION}/${type}` + return `fingerprintjs-pro-azure/${LAMBDA_FUNC_VERSION}/${type}` } From 95a8cb0467c8119295d053b728e77fc112723dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 13:57:15 +0100 Subject: [PATCH 14/21] build: add source maps --- rollup.config.mjs | 1 + tsconfig.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rollup.config.mjs b/rollup.config.mjs index d219ac9e..380dabca 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -62,6 +62,7 @@ function makeConfig(entryFile, artifactName, functionJsonPath) { const commonOutput = { exports: 'named', + sourcemap: true, } /** diff --git a/tsconfig.json b/tsconfig.json index c3b77f0d..165bce56 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,7 +9,8 @@ "noImplicitAny": true, "resolveJsonModule": true, "moduleResolution": "node", - "esModuleInterop": true + "esModuleInterop": true, + "sourceMap": true }, "include": ["proxy", "management"], "references": [ From 8c653f6015117e2b5bc0505b3239edffb4e285eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 13:57:24 +0100 Subject: [PATCH 15/21] chore: remove isRaw --- proxy/handlers/agent.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/handlers/agent.ts b/proxy/handlers/agent.ts index 4b3a9dfd..9997a6cf 100644 --- a/proxy/handlers/agent.ts +++ b/proxy/handlers/agent.ts @@ -23,7 +23,7 @@ export async function downloadAgent({ httpRequest, logger }: DownloadAgentParams const headers = filterRequestHeaders(httpRequest.headers) - return new Promise((resolve) => { + return new Promise((resolve) => { const data: any[] = [] const request = https.request( @@ -44,12 +44,12 @@ export async function downloadAgent({ httpRequest, logger }: DownloadAgentParams response.on('end', () => { const body = Buffer.concat(data) + const responseHeaders = updateResponseHeaders(response.headers, domain) resolve({ status: response.statusCode ? response.statusCode.toString() : '500', - headers: updateResponseHeaders(response.headers, domain), + headers: responseHeaders, body: new Uint8Array(body), - isRaw: true, }) }) }, From 6feaaad3886cc2538462a57ea98bdb6e03d79ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 13:57:34 +0100 Subject: [PATCH 16/21] chore: update blacklisted response headers --- proxy/utils/headers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index e82dd7d0..583bde85 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -10,7 +10,7 @@ const CACHE_CONTROL_HEADER_NAME = 'cache-control' const BLACKLISTED_HEADERS_PREFIXES = ['x-edge-', 'x-arr-', 'x-site', 'x-azure-'] const BLACKLISTED_REQUEST_HEADERS = new Set(['host', 'strict-transport-security']) -const BLACKLISTED_RESPONSE_HEADERS = new Set(['strict-transport-security']) +const BLACKLISTED_RESPONSE_HEADERS = new Set(['strict-transport-security', 'transfer-encoding']) export function filterRequestHeaders(headers: HttpRequestHeaders) { return Object.entries(headers).reduce((result: { [key: string]: string }, [name, value]) => { From 1fe7cad2728a2fc63200811c1035d5f18fd54732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 13:59:55 +0100 Subject: [PATCH 17/21] refactor: use .some --- proxy/utils/headers.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index 583bde85..33d7aa80 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -73,11 +73,5 @@ function isHeaderAllowedForRequest(headerName: string) { } function matchesBlacklistedHeaderPrefix(headerName: string) { - for (const blacklistedHeaderPrefix of BLACKLISTED_HEADERS_PREFIXES) { - if (headerName.startsWith(blacklistedHeaderPrefix)) { - return true - } - } - - return false + return BLACKLISTED_HEADERS_PREFIXES.some((prefix) => headerName.startsWith(prefix)) } From 1b85e526b5f809d0c189db4ffb7030af9d14cccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Tue, 14 Mar 2023 14:00:32 +0100 Subject: [PATCH 18/21] refactor: extract cookie name --- proxy/utils/headers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/proxy/utils/headers.ts b/proxy/utils/headers.ts index 33d7aa80..2e40fdae 100644 --- a/proxy/utils/headers.ts +++ b/proxy/utils/headers.ts @@ -6,6 +6,8 @@ import { adjustCookies, filterCookie } from './cookies' const COOKIE_HEADER_NAME = 'set-cookie' const CACHE_CONTROL_HEADER_NAME = 'cache-control' +const FPJS_COOKIE_NAME = '_iidt' + // Azure specific headers const BLACKLISTED_HEADERS_PREFIXES = ['x-edge-', 'x-arr-', 'x-site', 'x-azure-'] @@ -21,7 +23,8 @@ export function filterRequestHeaders(headers: HttpRequestHeaders) { if (headerName === 'cookie') { headerValue = headerValue.split(/; */).join('; ') - headerValue = filterCookie(headerValue, (key) => key === '_iidt') + + headerValue = filterCookie(headerValue, (key) => key === FPJS_COOKIE_NAME) } result[headerName] = headerValue From 549789c74d94bc74350b41b7a0e30d5529943d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Wed, 15 Mar 2023 13:13:16 +0100 Subject: [PATCH 19/21] build: fix typecheck --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b1151a75..a5877cfb 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "eslint --ext .js,.ts --ignore-path .gitignore --max-warnings 0 .", "lint:fix": "yarn lint --fix", "test": "jest --coverage", - "test:dts": "tsc --noEmit --isolatedModules dist/*.d.ts", + "test:dts": "tsc --noEmit --isolatedModules dist/**/*.d.ts", "emulate-storage": "azurite -l ./storage --silent", "start": "func start" }, From 58b97978e55175e46a6d6281f4ab5976ca5bd6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20=C5=BBydek?= Date: Wed, 15 Mar 2023 15:31:36 +0100 Subject: [PATCH 20/21] ci: update src base path Co-authored-by: Ilya Taratukhin --- .github/workflows/coverage-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage-diff.yml b/.github/workflows/coverage-diff.yml index 20e91f8c..584082f0 100644 --- a/.github/workflows/coverage-diff.yml +++ b/.github/workflows/coverage-diff.yml @@ -19,7 +19,7 @@ jobs: uses: fingerprintjs/action-coverage-report-md@v1 id: coverage-md with: - srcBasePath: './src' + srcBasePath: './proxy' - uses: marocchino/sticky-pull-request-comment@adca94abcaf73c10466a71cc83ae561fd66d1a56 with: message: | From cf3bc90554871364ab9e54a3534018d563310be1 Mon Sep 17 00:00:00 2001 From: Ilya Taratukhin Date: Thu, 16 Mar 2023 14:37:45 +0100 Subject: [PATCH 21/21] chore: update coverage src base path --- .github/workflows/coverage-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage-diff.yml b/.github/workflows/coverage-diff.yml index 584082f0..7d25c5f3 100644 --- a/.github/workflows/coverage-diff.yml +++ b/.github/workflows/coverage-diff.yml @@ -19,7 +19,7 @@ jobs: uses: fingerprintjs/action-coverage-report-md@v1 id: coverage-md with: - srcBasePath: './proxy' + srcBasePath: './' - uses: marocchino/sticky-pull-request-comment@adca94abcaf73c10466a71cc83ae561fd66d1a56 with: message: |