Handle compression

package.json

@@ -70,7 +70,8 @@
     "tailwindcss": "^3.0.23",
     "typescript": "^4.0.3",
     "util": "^0.12.5",
-    "utility-types": "^3.11.0"
+    "utility-types": "^3.11.0",
+    "web-streams-polyfill": "^4.0.0"
   },
   "_resolutions_comment_": "https://stackoverflow.com/a/71855781/2573621",
   "resolutions": {

src/helpers/gzip.ts

@@ -0,0 +1,51 @@
+export type CompressionType = 'gzip' | 'deflate'
+
+const rawToUint8Array = (raw: chrome.webRequest.UploadData[]): Uint8Array => {
+  const arrays = raw
+    .filter((data) => data.bytes)
+    .map((data) => new Uint8Array(data.bytes!))
+
+  const totalLength = arrays.reduce((acc, arr) => acc + arr.length, 0)
+  const result = new Uint8Array(totalLength)
+
+  let offset = 0
+  for (const arr of arrays) {
+    result.set(arr, offset)
+    offset += arr.length
+  }
+
+  return result
+}
+
+const stringToUint8Array = (str: string): Uint8Array => {
+  const array = new Uint8Array(str.length)
+  for (let i = 0; i < str.length; i++) {
+    array[i] = str.charCodeAt(i)
+  }
+  return array
+}
+
+export const decompress = async (
+  raw: chrome.webRequest.UploadData[] | string,
+  compressionType: CompressionType
+) => {
+  const uint8Array =
+    typeof raw === 'string' ? stringToUint8Array(raw) : rawToUint8Array(raw)
+
+  const readableStream = new Response(uint8Array).body
+  if (!readableStream) {
+    throw new Error('Failed to create readable stream from Uint8Array.')
+  }
+
+  // Pipe through the decompression stream
+  const decompressedStream = readableStream.pipeThrough(
+    new (window as any).DecompressionStream(compressionType)
+  )
+
+  // Convert the decompressed stream back to a Uint8Array
+  const decompressedArrayBuffer = await new Response(
+    decompressedStream
+  ).arrayBuffer()
+
+  return new Uint8Array(decompressedArrayBuffer)
+}

src/helpers/networkHelpers.test.ts

@@ -1,5 +1,4 @@
 import { DeepPartial } from 'utility-types'
-import { TextEncoder } from 'util'
 import {
   IHeader,
   getRequestBody,
@@ -9,6 +8,17 @@ import {
 } from './networkHelpers'
 import dedent from 'dedent'
 
+// Unable to test actual gzip decompression as the DecompressionStream is not available in JSDOM
+// and the Response object from fetch is missing parts of the steam API
+jest.mock('../helpers/gzip', () => ({
+  __esModule: true,
+  decompress: (val: chrome.webRequest.UploadData[]) => {
+    return new Promise((resolve) => {
+      resolve(val[0].bytes)
+    })
+  },
+}))
+
 describe('networkHelpers.getRequestBodyFromUrl', () => {
   it('throws an error when no query is found in the URL', () => {
     expect(() => {
@@ -116,7 +126,7 @@ describe('networkHelpers.getRequestBodyFromUrl', () => {
 })
 
 describe('networkHelpers.matchWebAndNetworkRequest', () => {
-  it('matches a web request with a network request', () => {
+  it('matches a web request with a network request', async () => {
     const body = JSON.stringify({
       query: 'query { user }',
     })
@@ -140,15 +150,15 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => {
       },
     }
 
-    const match = matchWebAndNetworkRequest(
+    const match = await matchWebAndNetworkRequest(
       networkRequest as any,
       webRequest as any,
       webRequestHeaders
     )
     expect(match).toBe(true)
   })
 
-  it('does not match request with different URLs', () => {
+  it('does not match request with different URLs', async () => {
     const body = JSON.stringify({
       query: 'query { user }',
     })
@@ -172,15 +182,15 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => {
       },
     }
 
-    const match = matchWebAndNetworkRequest(
+    const match = await matchWebAndNetworkRequest(
       networkRequest as any,
       webRequest as any,
       webRequestHeaders
     )
     expect(match).toBe(false)
   })
 
-  it('does not match requests with different methods', () => {
+  it('does not match requests with different methods', async () => {
     const body = JSON.stringify({
       query: 'query { user }',
     })
@@ -204,15 +214,15 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => {
       },
     }
 
-    const match = matchWebAndNetworkRequest(
+    const match = await matchWebAndNetworkRequest(
       networkRequest as any,
       webRequest as any,
       webRequestHeaders
     )
     expect(match).toBe(false)
   })
 
-  it('does not match requests with different bodies', () => {
+  it('does not match requests with different bodies', async () => {
     const webRequest: DeepPartial<chrome.webRequest.WebRequestBodyDetails> = {
       url: 'http://example1.com',
       method: 'POST',
@@ -242,7 +252,7 @@ describe('networkHelpers.matchWebAndNetworkRequest', () => {
       },
     }
 
-    const match = matchWebAndNetworkRequest(
+    const match = await matchWebAndNetworkRequest(
       networkRequest as any,
       webRequest as any,
       webRequestHeaders
@@ -275,7 +285,7 @@ describe('networkHelpers.getRequestBodyFromMultipartFormData', () => {
 })
 
 describe('networkHelpers.getRequestBody', () => {
-  it('returns request body from a multipart form data request', () => {
+  it('returns request body from a multipart form data request', async () => {
     const details: Partial<chrome.webRequest.WebRequestBodyDetails> = {
       requestBody: {
         raw: [
@@ -296,7 +306,7 @@ describe('networkHelpers.getRequestBody', () => {
       },
     ]
 
-    const result = getRequestBody(details as any, headers)
+    const result = await getRequestBody(details as any, headers)
 
     expect(result).toEqual(
       JSON.stringify({
@@ -313,4 +323,39 @@ describe('networkHelpers.getRequestBody', () => {
       })
     )
   })
+
+  it('returns request body from a compressed request', async () => {
+    const details: Partial<chrome.webRequest.WebRequestBodyDetails> = {
+      requestBody: {
+        raw: [
+          {
+            bytes: new TextEncoder().encode(
+              JSON.stringify({
+                query: 'query { user }',
+              })
+            ),
+          },
+        ],
+      },
+    }
+
+    const headers: IHeader[] = [
+      {
+        name: 'content-type',
+        value: 'application/json',
+      },
+      {
+        name: 'content-encoding',
+        value: 'deflate',
+      },
+    ]
+
+    const result = await getRequestBody(details as any, headers)
+
+    expect(result).toEqual(
+      JSON.stringify({
+        query: 'query { user }',
+      })
+    )
+  })
 })

src/helpers/networkHelpers.ts

@@ -1,6 +1,7 @@
 import { IGraphqlRequestBody, IOperationDetails } from './graphqlHelpers'
 import decodeQueryParam from './decodeQueryParam'
 import { parse } from './safeJson'
+import { decompress, CompressionType } from './gzip'
 
 export interface IHeader {
   name: string
@@ -81,12 +82,56 @@ export const isRequestComplete = (
   )
 }
 
+/**
+ * Detect the compression type based on the content-encoding header.
+ *
+ * @param headers all request headers
+ * @returns the compression type if detected
+ */
+const detectCompressionType = (
+  headers: IHeader[]
+): CompressionType | undefined => {
+  const contentEncodingHeader = headers.find(
+    (header) => header.name.toLowerCase() === 'content-encoding'
+  )
+
+  if (!contentEncodingHeader) {
+    return
+  }
+
+  if (contentEncodingHeader.value === 'gzip') {
+    return 'gzip'
+  }
+
+  if (contentEncodingHeader.value === 'deflate') {
+    return 'deflate'
+  }
+}
+
 /**
  * Decode the raw request body into a string
  */
-const decodeRawBody = (raw: chrome.webRequest.UploadData[]) => {
-  const decoder = new TextDecoder('utf-8')
-  return raw.map((data) => decoder.decode(data.bytes)).join('')
+const decodeRawBody = async (
+  raw: chrome.webRequest.UploadData[] | string,
+  compressionType?: CompressionType
+): Promise<string> => {
+  // If the body is compressed, decompress it
+  if (compressionType) {
+    return decompress(raw, compressionType).then((res) => {
+      const decoder = new TextDecoder('utf-8')
+      return decoder.decode(res)
+    })
+  }
+
+  if (typeof raw === 'string') {
+    // If we have a plain string, just return it, it is already
+    // decoded
+    return raw
+  } else {
+    // Decode the raw bytes into a string
+    const decoder = new TextDecoder('utf-8')
+    return raw.map((data) => decoder.decode(data.bytes)).join('')
+  }
 }
 
 /**
@@ -234,16 +279,20 @@ export const getRequestBodyFromUrl = (url: string): IGraphqlRequestBody => {
   throw new Error('Could not parse request body from URL')
 }
 
-const getRequestBodyFromWebRequestBodyDetails = (
+const getRequestBodyFromWebRequestBodyDetails = async (
   details: chrome.webRequest.WebRequestBodyDetails,
   headers: IHeader[]
-): string | undefined => {
+): Promise<string | undefined> => {
   if (details.method === 'GET') {
     const body = getRequestBodyFromUrl(details.url)
     return JSON.stringify(body)
   }
 
-  const body = decodeRawBody(details.requestBody?.raw || [])
+  const compressionType = detectCompressionType(headers)
+  const body = await decodeRawBody(
+    details.requestBody?.raw || [],
+    compressionType
+  )
   const boundary = getMultipartFormDataBoundary(headers)
 
   if (boundary && body) {
@@ -254,15 +303,19 @@ const getRequestBodyFromWebRequestBodyDetails = (
   return body
 }
 
-const getRequestBodyFromNetworkRequest = (
+const getRequestBodyFromNetworkRequest = async (
   details: chrome.devtools.network.Request
-): string | undefined => {
+): Promise<string | undefined> => {
   if (details.request.method === 'GET') {
     const body = getRequestBodyFromUrl(details.request.url)
     return JSON.stringify(body)
   }
 
-  const body = details.request.postData?.text
+  const compressionType = detectCompressionType(details.request.headers)
+  const body = await decodeRawBody(
+    details.request.postData?.text || '',
+    compressionType
+  )
 
   const boundary = getMultipartFormDataBoundary(details.request.headers)
   if (boundary && body) {
@@ -273,7 +326,7 @@ const getRequestBodyFromNetworkRequest = (
   return body
 }
 
-export const getRequestBody = <
+export const getRequestBody = async <
   T extends
     | chrome.devtools.network.Request
     | chrome.webRequest.WebRequestBodyDetails
@@ -282,7 +335,7 @@ export const getRequestBody = <
   ...headers: T extends chrome.webRequest.WebRequestBodyDetails
     ? [IHeader[]]
     : []
-): string | undefined => {
+): Promise<string | undefined> => {
   try {
     if (isNetworkRequest(details)) {
       return getRequestBodyFromNetworkRequest(details)
@@ -304,17 +357,19 @@ export const getRequestBody = <
  * no stable id is available to us.
  *
  */
-export const matchWebAndNetworkRequest = (
+export const matchWebAndNetworkRequest = async (
   networkRequest: chrome.devtools.network.Request,
   webRequest: chrome.webRequest.WebRequestBodyDetails,
   webRequestHeaders: IHeader[]
-): boolean => {
+): Promise<boolean> => {
   try {
-    const webRequestBody = getRequestBodyFromWebRequestBodyDetails(
+    const webRequestBody = await getRequestBodyFromWebRequestBodyDetails(
       webRequest,
       webRequestHeaders
     )
-    const networkRequestBody = getRequestBodyFromNetworkRequest(networkRequest)
+    const networkRequestBody = await getRequestBodyFromNetworkRequest(
+      networkRequest
+    )
 
     const isMethodMatch = webRequest.method === networkRequest.request.method
     const isBodyMatch = webRequestBody === networkRequestBody