From fede0e7e901bcfd4e25f03d78e6f0152c1c20de9 Mon Sep 17 00:00:00 2001 From: Warren Day Date: Thu, 5 Sep 2024 10:29:16 +0100 Subject: [PATCH 1/2] handle compression of requests for deflate and gzip --- src/helpers/gzip.ts | 53 +++++++++ src/helpers/networkHelpers.ts | 85 ++++++++++++--- src/hooks/useLatestState.ts | 35 ++++++ src/hooks/useNetworkMonitor.ts | 189 ++++++++++++++++++++------------- 4 files changed, 271 insertions(+), 91 deletions(-) create mode 100644 src/helpers/gzip.ts create mode 100644 src/hooks/useLatestState.ts diff --git a/src/helpers/gzip.ts b/src/helpers/gzip.ts new file mode 100644 index 0000000..8987d49 --- /dev/null +++ b/src/helpers/gzip.ts @@ -0,0 +1,53 @@ +export type CompressionType = 'gzip' | 'deflate' + +const rawToUint8Array = (raw: chrome.webRequest.UploadData[]): Uint8Array => { + const arrays = raw + .filter((data) => data.bytes) + .map((data) => new Uint8Array(data.bytes!)) + + const totalLength = arrays.reduce((acc, arr) => acc + arr.length, 0) + const result = new Uint8Array(totalLength) + + let offset = 0 + for (const arr of arrays) { + result.set(arr, offset) + offset += arr.length + } + + return result +} + +const stringToUint8Array = (str: string): Uint8Array => { + const array = new Uint8Array(str.length) + for (let i = 0; i < str.length; i++) { + array[i] = str.charCodeAt(i) + } + return array +} + +export const decompress = async ( + raw: chrome.webRequest.UploadData[] | string, + compressionType: CompressionType +) => { + const uint8Array = + typeof raw === 'string' ? stringToUint8Array(raw) : rawToUint8Array(raw) + + const readableStream = new Response(uint8Array).body + if (!readableStream) { + throw new Error('Failed to create readable stream from Uint8Array.') + } + + // Pipe through the decompression stream + const decompressedStream = readableStream.pipeThrough( + new (window as any).DecompressionStream(compressionType) + ) + + console.log({ raw, uint8Array }) + + // Convert the decompressed stream back to a Uint8Array + const decompressedArrayBuffer = await new Response( + decompressedStream + ).arrayBuffer() + + return new Uint8Array(decompressedArrayBuffer) +} diff --git a/src/helpers/networkHelpers.ts b/src/helpers/networkHelpers.ts index 57c404a..9cd71f3 100644 --- a/src/helpers/networkHelpers.ts +++ b/src/helpers/networkHelpers.ts @@ -1,6 +1,7 @@ import { IGraphqlRequestBody, IOperationDetails } from './graphqlHelpers' import decodeQueryParam from './decodeQueryParam' import { parse } from './safeJson' +import { decompress, CompressionType } from './gzip' export interface IHeader { name: string @@ -81,12 +82,56 @@ export const isRequestComplete = ( ) } +/** + * Detect the compression type based on the content-encoding header. + * + * @param headers all request headers + * @returns the compression type if detected + */ +const detectCompressionType = ( + headers: IHeader[] +): CompressionType | undefined => { + const contentEncodingHeader = headers.find( + (header) => header.name.toLowerCase() === 'content-encoding' + ) + + if (!contentEncodingHeader) { + return + } + + if (contentEncodingHeader.value === 'gzip') { + return 'gzip' + } + + if (contentEncodingHeader.value === 'deflate') { + return 'deflate' + } +} + /** * Decode the raw request body into a string */ -const decodeRawBody = (raw: chrome.webRequest.UploadData[]) => { - const decoder = new TextDecoder('utf-8') - return raw.map((data) => decoder.decode(data.bytes)).join('') +const decodeRawBody = async ( + raw: chrome.webRequest.UploadData[] | string, + compressionType?: CompressionType +): Promise => { + // If the body is compressed, decompress it + if (compressionType) { + return decompress(raw, compressionType).then((res) => { + const decoder = new TextDecoder('utf-8') + return decoder.decode(res) + }) + } + + if (typeof raw === 'string') { + // If we have a plain string, just return it, it is already + // decoded + return raw + } else { + // Decode the raw bytes into a string + const decoder = new TextDecoder('utf-8') + return raw.map((data) => decoder.decode(data.bytes)).join('') + } } /** @@ -234,16 +279,20 @@ export const getRequestBodyFromUrl = (url: string): IGraphqlRequestBody => { throw new Error('Could not parse request body from URL') } -const getRequestBodyFromWebRequestBodyDetails = ( +const getRequestBodyFromWebRequestBodyDetails = async ( details: chrome.webRequest.WebRequestBodyDetails, headers: IHeader[] -): string | undefined => { +): Promise => { if (details.method === 'GET') { const body = getRequestBodyFromUrl(details.url) return JSON.stringify(body) } - const body = decodeRawBody(details.requestBody?.raw || []) + const compressionType = detectCompressionType(headers) + const body = await decodeRawBody( + details.requestBody?.raw || [], + compressionType + ) const boundary = getMultipartFormDataBoundary(headers) if (boundary && body) { @@ -254,15 +303,19 @@ const getRequestBodyFromWebRequestBodyDetails = ( return body } -const getRequestBodyFromNetworkRequest = ( +const getRequestBodyFromNetworkRequest = async ( details: chrome.devtools.network.Request -): string | undefined => { +): Promise => { if (details.request.method === 'GET') { const body = getRequestBodyFromUrl(details.request.url) return JSON.stringify(body) } - const body = details.request.postData?.text + const compressionType = detectCompressionType(details.request.headers) + const body = await decodeRawBody( + details.request.postData?.text || '', + compressionType + ) const boundary = getMultipartFormDataBoundary(details.request.headers) if (boundary && body) { @@ -273,7 +326,7 @@ const getRequestBodyFromNetworkRequest = ( return body } -export const getRequestBody = < +export const getRequestBody = async < T extends | chrome.devtools.network.Request | chrome.webRequest.WebRequestBodyDetails @@ -282,7 +335,7 @@ export const getRequestBody = < ...headers: T extends chrome.webRequest.WebRequestBodyDetails ? [IHeader[]] : [] -): string | undefined => { +): Promise => { try { if (isNetworkRequest(details)) { return getRequestBodyFromNetworkRequest(details) @@ -304,17 +357,19 @@ export const getRequestBody = < * no stable id is available to us. * */ -export const matchWebAndNetworkRequest = ( +export const matchWebAndNetworkRequest = async ( networkRequest: chrome.devtools.network.Request, webRequest: chrome.webRequest.WebRequestBodyDetails, webRequestHeaders: IHeader[] -): boolean => { +): Promise => { try { - const webRequestBody = getRequestBodyFromWebRequestBodyDetails( + const webRequestBody = await getRequestBodyFromWebRequestBodyDetails( webRequest, webRequestHeaders ) - const networkRequestBody = getRequestBodyFromNetworkRequest(networkRequest) + const networkRequestBody = await getRequestBodyFromNetworkRequest( + networkRequest + ) const isMethodMatch = webRequest.method === networkRequest.request.method const isBodyMatch = webRequestBody === networkRequestBody diff --git a/src/hooks/useLatestState.ts b/src/hooks/useLatestState.ts new file mode 100644 index 0000000..aa96026 --- /dev/null +++ b/src/hooks/useLatestState.ts @@ -0,0 +1,35 @@ +import { useCallback, useRef, useState } from 'react' + +/** + * Has a matching API to useState, but also provides a getter to always + * access the latest state. + * + * This is handled through a ref, so it's safe to use in callbacks or + * other places where the state might be stale. + * + */ +const useLatestState = (initialState: T) => { + const [state, setState] = useState(initialState) + const latestStateRef = useRef(state) + + // This getter can be used to always access the latest state + const getState = () => latestStateRef.current + + const setStateWrapper = useCallback( + (newState: T | ((state: T) => T)) => { + setState((prevState) => { + const updatedState = + typeof newState === 'function' + ? (newState as any)(prevState) + : newState + latestStateRef.current = updatedState + return updatedState + }) + }, + [setState] + ) + + return [state, setStateWrapper, getState] as const +} + +export default useLatestState diff --git a/src/hooks/useNetworkMonitor.ts b/src/hooks/useNetworkMonitor.ts index 59dc9cd..a378e87 100644 --- a/src/hooks/useNetworkMonitor.ts +++ b/src/hooks/useNetworkMonitor.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' import { v4 as uuid } from 'uuid' import { parseGraphqlBody, @@ -17,6 +17,7 @@ import { isRequestComplete, matchWebAndNetworkRequest, } from '../helpers/networkHelpers' +import useLatestState from './useLatestState' export interface IClearWebRequestsOptions { clearPending?: boolean @@ -28,9 +29,11 @@ export interface IClearWebRequestsOptions { * by checking the request body for a valid graphql operation * */ -const validateNetworkRequest = (details: chrome.devtools.network.Request) => { +const validateNetworkRequest = async ( + details: chrome.devtools.network.Request +) => { try { - const body = getRequestBody(details) + const body = await getRequestBody(details) if (!body) { return false } @@ -80,13 +83,37 @@ const processNetworkRequest = ( } } +/** + * Match a network request to a webRequest + * + * @param webRequests + * @param details + * @returns + */ +const findMatchingWebRequest = async ( + webRequests: IIncompleteNetworkRequest[], + details: chrome.devtools.network.Request +) => { + const res = await Promise.all( + webRequests.map(async (webRequest) => { + const isMatch = await matchWebAndNetworkRequest( + details, + webRequest.native?.webRequest, + webRequest.request?.headers || [] + ) + return isMatch ? webRequest : null + }) + ) + return res.filter((r) => r)[0] +} + export const useNetworkMonitor = (): [ INetworkRequest[], (opts?: IClearWebRequestsOptions) => void ] => { - const [webRequests, setWebRequests] = useState( - [] - ) + const [webRequests, setWebRequests, getLatestWebRequests] = useLatestState< + IIncompleteNetworkRequest[] + >([]) const handleBeforeRequest = useCallback( (details: chrome.webRequest.WebRequestBodyDetails) => { @@ -109,62 +136,73 @@ export const useNetworkMonitor = (): [ ) const handleBeforeSendHeaders = useCallback( - (details: chrome.webRequest.WebRequestHeadersDetails) => { - setWebRequests((webRequests) => { - return webRequests.flatMap((webRequest) => { - // Don't overwrite the request if it's already complete - if (webRequest.response) { - return webRequest - } + async (details: chrome.webRequest.WebRequestHeadersDetails) => { + const webRequests = getLatestWebRequests() - // We only want to update the request which matches on id. - if (webRequest.id !== details.requestId) { - return webRequest - } + const webRequest = webRequests.find( + (webRequest) => webRequest.id === details.requestId + ) + if (!webRequest) { + return + } - // Now we have both the headers and the body from the webRequest api - // we can determine if this is a graphql request. - // - // If it is not, we return an empty array so flatMap will remove it. - const body = getRequestBody( - webRequest.native.webRequest, - details.requestHeaders || [] - ) - if (!body) { - return [] - } + // Don't overwrite the request if it's already complete + if (webRequest.response) { + return webRequest + } - const graphqlRequestBody = parseGraphqlBody(body) - if (!graphqlRequestBody) { - return [] - } + // Now we have both the headers and the body from the webRequest api + // we can determine if this is a graphql request. + // + // If it is not, we return an empty array so flatMap will remove it. + const body = await getRequestBody( + webRequest.native.webRequest, + details.requestHeaders || [] + ) - const primaryOperation = getFirstGraphqlOperation(graphqlRequestBody) - if (!primaryOperation) { - return [] - } + if (!body) { + return + } - return { - ...webRequest, - request: { - primaryOperation, - body: graphqlRequestBody.map((requestBody) => ({ - ...requestBody, - id: uuid(), - })), - bodySize: body ? body.length : 0, - headers: details.requestHeaders, - headersSize: (details.requestHeaders || []).reduce( - (acc, header) => - acc + header.name.length + (header.value?.length || 0), - 0 - ), - }, + const graphqlRequestBody = parseGraphqlBody(body) + if (!graphqlRequestBody) { + return + } + + const primaryOperation = getFirstGraphqlOperation(graphqlRequestBody) + if (!primaryOperation) { + return + } + + const request = { + primaryOperation, + body: graphqlRequestBody.map((requestBody) => ({ + ...requestBody, + id: uuid(), + })), + bodySize: body ? body.length : 0, + headers: details.requestHeaders, + headersSize: (details.requestHeaders || []).reduce( + (acc, header) => + acc + header.name.length + (header.value?.length || 0), + 0 + ), + } + + setWebRequests((prevWebRequests) => { + return prevWebRequests.map((prevWebRequest) => { + if (prevWebRequest.id === webRequest.id) { + return { + ...prevWebRequest, + request, + } + } else { + return prevWebRequest } }) }) }, - [setWebRequests] + [setWebRequests, getLatestWebRequests] ) const handleRequestFinished = useCallback( @@ -173,33 +211,32 @@ export const useNetworkMonitor = (): [ return } - details.getContent((responseBody) => { - setWebRequests((webRequests) => { - return webRequests.map((webRequest) => { - // Don't overwrite the request if it's already complete - if (webRequest.response) { - return webRequest - } - - const isMatch = matchWebAndNetworkRequest( - details, - webRequest.native?.webRequest, - webRequest.request?.headers || [] - ) - if (!isMatch) { - return webRequest - } + details.getContent(async (responseBody) => { + const webRequests = getLatestWebRequests() + const matchingWebRequest = await findMatchingWebRequest( + webRequests, + details + ) + if (!matchingWebRequest) { + return + } - return { - ...webRequest, - id: webRequest.id, - ...processNetworkRequest(details, responseBody), + setWebRequests((prevWebRequests) => { + return prevWebRequests.map((prevWebRequest) => { + if (prevWebRequest.id === matchingWebRequest.id) { + return { + ...prevWebRequest, + id: prevWebRequest.id, + ...processNetworkRequest(details, responseBody), + } + } else { + return prevWebRequest } }) }) }) }, - [setWebRequests] + [setWebRequests, getLatestWebRequests] ) const handleHAREntries = useCallback( @@ -211,8 +248,8 @@ export const useNetworkMonitor = (): [ const entriesWithContent = await Promise.all( validEntries.map((details) => { return new Promise((resolve) => { - details.getContent((responseBody) => { - const body = getRequestBody(details) + details.getContent(async (responseBody) => { + const body = await getRequestBody(details) if (!body) { return } From 7ae3e7efa1522d75d9bddfc49c19b723afc1e358 Mon Sep 17 00:00:00 2001 From: Warren Day Date: Thu, 5 Sep 2024 15:57:06 +0100 Subject: [PATCH 2/2] added test for compression handler --- package.json | 3 +- src/helpers/gzip.ts | 2 - src/helpers/networkHelpers.test.ts | 67 +++++++++++++++++++++++++----- src/hooks/useNetworkMonitor.ts | 30 ++++++++----- yarn.lock | 5 +++ 5 files changed, 82 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index fc344a9..6c60133 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,8 @@ "tailwindcss": "^3.0.23", "typescript": "^4.0.3", "util": "^0.12.5", - "utility-types": "^3.11.0" + "utility-types": "^3.11.0", + "web-streams-polyfill": "^4.0.0" }, "_resolutions_comment_": "https://stackoverflow.com/a/71855781/2573621", "resolutions": { diff --git a/src/helpers/gzip.ts b/src/helpers/gzip.ts index 8987d49..5a3e006 100644 --- a/src/helpers/gzip.ts +++ b/src/helpers/gzip.ts @@ -42,8 +42,6 @@ export const decompress = async ( new (window as any).DecompressionStream(compressionType) ) - console.log({ raw, uint8Array }) - // Convert the decompressed stream back to a Uint8Array const decompressedArrayBuffer = await new Response( decompressedStream diff --git a/src/helpers/networkHelpers.test.ts b/src/helpers/networkHelpers.test.ts index bfe040a..8a85dfa 100644 --- a/src/helpers/networkHelpers.test.ts +++ b/src/helpers/networkHelpers.test.ts @@ -1,5 +1,4 @@ import { DeepPartial } from 'utility-types' -import { TextEncoder } from 'util' import { IHeader, getRequestBody, @@ -9,6 +8,17 @@ import { } from './networkHelpers' import dedent from 'dedent' +// Unable to test actual gzip decompression as the DecompressionStream is not available in JSDOM +// and the Response object from fetch is missing parts of the steam API +jest.mock('../helpers/gzip', () => ({ + __esModule: true, + decompress: (val: chrome.webRequest.UploadData[]) => { + return new Promise((resolve) => { + resolve(val[0].bytes) + }) + }, +})) + describe('networkHelpers.getRequestBodyFromUrl', () => { it('throws an error when no query is found in the URL', () => { expect(() => { @@ -116,7 +126,7 @@ describe('networkHelpers.getRequestBodyFromUrl', () => { }) describe('networkHelpers.matchWebAndNetworkRequest', () => { - it('matches a web request with a network request', () => { + it('matches a web request with a network request', async () => { const body = JSON.stringify({ query: 'query { user }', }) @@ -140,7 +150,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { }, } - const match = matchWebAndNetworkRequest( + const match = await matchWebAndNetworkRequest( networkRequest as any, webRequest as any, webRequestHeaders @@ -148,7 +158,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { expect(match).toBe(true) }) - it('does not match request with different URLs', () => { + it('does not match request with different URLs', async () => { const body = JSON.stringify({ query: 'query { user }', }) @@ -172,7 +182,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { }, } - const match = matchWebAndNetworkRequest( + const match = await matchWebAndNetworkRequest( networkRequest as any, webRequest as any, webRequestHeaders @@ -180,7 +190,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { expect(match).toBe(false) }) - it('does not match requests with different methods', () => { + it('does not match requests with different methods', async () => { const body = JSON.stringify({ query: 'query { user }', }) @@ -204,7 +214,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { }, } - const match = matchWebAndNetworkRequest( + const match = await matchWebAndNetworkRequest( networkRequest as any, webRequest as any, webRequestHeaders @@ -212,7 +222,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { expect(match).toBe(false) }) - it('does not match requests with different bodies', () => { + it('does not match requests with different bodies', async () => { const webRequest: DeepPartial = { url: 'http://example1.com', method: 'POST', @@ -242,7 +252,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => { }, } - const match = matchWebAndNetworkRequest( + const match = await matchWebAndNetworkRequest( networkRequest as any, webRequest as any, webRequestHeaders @@ -275,7 +285,7 @@ describe('networkHelpers.getRequestBodyFromMultipartFormData', () => { }) describe('networkHelpers.getRequestBody', () => { - it('returns request body from a multipart form data request', () => { + it('returns request body from a multipart form data request', async () => { const details: Partial = { requestBody: { raw: [ @@ -296,7 +306,7 @@ describe('networkHelpers.getRequestBody', () => { }, ] - const result = getRequestBody(details as any, headers) + const result = await getRequestBody(details as any, headers) expect(result).toEqual( JSON.stringify({ @@ -313,4 +323,39 @@ describe('networkHelpers.getRequestBody', () => { }) ) }) + + it('returns request body from a compressed request', async () => { + const details: Partial = { + requestBody: { + raw: [ + { + bytes: new TextEncoder().encode( + JSON.stringify({ + query: 'query { user }', + }) + ), + }, + ], + }, + } + + const headers: IHeader[] = [ + { + name: 'content-type', + value: 'application/json', + }, + { + name: 'content-encoding', + value: 'deflate', + }, + ] + + const result = await getRequestBody(details as any, headers) + + expect(result).toEqual( + JSON.stringify({ + query: 'query { user }', + }) + ) + }) }) diff --git a/src/hooks/useNetworkMonitor.ts b/src/hooks/useNetworkMonitor.ts index a378e87..984091f 100644 --- a/src/hooks/useNetworkMonitor.ts +++ b/src/hooks/useNetworkMonitor.ts @@ -94,17 +94,21 @@ const findMatchingWebRequest = async ( webRequests: IIncompleteNetworkRequest[], details: chrome.devtools.network.Request ) => { - const res = await Promise.all( - webRequests.map(async (webRequest) => { - const isMatch = await matchWebAndNetworkRequest( - details, - webRequest.native?.webRequest, - webRequest.request?.headers || [] - ) - return isMatch ? webRequest : null - }) + const match = await Promise.all( + webRequests + // Don't target requests that already have a response + .filter((webRequest) => !webRequest.response) + .map(async (webRequest) => { + const isMatch = await matchWebAndNetworkRequest( + details, + webRequest.native?.webRequest, + webRequest.request?.headers || [] + ) + return isMatch ? webRequest : null + }) + .filter((r) => r) ) - return res.filter((r) => r)[0] + return match[0] } export const useNetworkMonitor = (): [ @@ -336,9 +340,13 @@ export const useNetworkMonitor = (): [ return onRequestFinished(handleRequestFinished) }, [handleRequestFinished]) - // Only return complete networkRequests. + // Only return webRequests where the request portion is complete. + // Since we build up the data from multiple events. We only want + // to display results that have enough data to be useful. const completeWebRequests = webRequests.filter(isRequestComplete) // @ts-ignore + // Ignored as completeWebRequests is readonly. Need to update type + // across app. return [completeWebRequests, clearWebRequests] as const } diff --git a/yarn.lock b/yarn.lock index eb4c5cb..5945ebb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9565,6 +9565,11 @@ wbuf@^1.1.0, wbuf@^1.7.3: dependencies: minimalistic-assert "^1.0.0" +web-streams-polyfill@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0.tgz#74cedf168339ee6e709532f76c49313a8c7acdac" + integrity sha512-0zJXHRAYEjM2tUfZ2DiSOHAa2aw1tisnnhU3ufD57R8iefL+DcdJyRBRyJpG+NUimDgbTI/lH+gAE1PAvV3Cgw== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"