diff --git a/.devcontainer/devcontainer.json b/.devcontainer.json similarity index 60% rename from .devcontainer/devcontainer.json rename to .devcontainer.json index 6ac3c9d..b10f39a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer.json @@ -5,14 +5,18 @@ "ghcr.io/guiyomh/features/just:0": {}, "ghcr.io/devcontainers-contrib/features/actionlint:1": {}, "ghcr.io/dhoeric/features/hadolint:1": {}, - "./feature-playwright": { "version": "1.31.1" }, - "./feature-novnc": {}, - "ghcr.io/lukewiwa/features/shellcheck:0": {} + "ghcr.io/lukewiwa/features/shellcheck:0": {}, + "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, + "privileged": true, // to pass through /dev/kvm for osrun (in docker-in-docker) + "mounts": [ // for osrun to store its Windows image + { + "source": "osrun-cache", + "target": "/osruncache", + "type": "volume" + } + ], "postCreateCommand": "npm i", - "containerEnv": { - "CHROME_PATH": "/ms-playwright/chromium-1048/chrome-linux/chrome" - }, "customizations": { "vscode": { "extensions": [ @@ -24,4 +28,4 @@ ] } } -} \ No newline at end of file +} diff --git a/.devcontainer/feature-novnc/devcontainer-feature.json b/.devcontainer/feature-novnc/devcontainer-feature.json deleted file mode 100644 index c94d2b1..0000000 --- a/.devcontainer/feature-novnc/devcontainer-feature.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "novnc", - "id": "novnc", - "version": "1.0.0", - "description": "Installs novnc along with xvfb, x11vnc, and fluxbox. Web interface will listen on port 8282.", - "installsAfter": [ - "ghcr.io/devcontainers/features/common-utils" - ], - "entrypoint": "/usr/local/bin/start-novnc", - "containerEnv": { - "DISPLAY": ":0.0" - } -} \ No newline at end of file diff --git a/.devcontainer/feature-novnc/install.sh b/.devcontainer/feature-novnc/install.sh deleted file mode 100644 index d81483a..0000000 --- a/.devcontainer/feature-novnc/install.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -set -e - -echo "Activating feature 'novnc'" - -###################### -## FROM https://github.com/devcontainers/features/blob/main/src/desktop-lite/install.sh - -apt_get_update() { - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y - fi -} - -# Checks if packages are installed and installs them if not -check_packages() { - if ! dpkg -s "$@" >/dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" - fi -} - -###################### - -check_packages xvfb x11vnc novnc fluxbox x11-utils - -cat > /usr/local/bin/start-novnc \ -<< EOF -#!/bin/sh -echo "Starting Xvfb on DISPLAY :0.0 (2560x1440x16)" -Xvfb :0 -screen 0 2560x1440x16 -listen tcp -ac & - -echo "Starting x11vnc on port 5900" -x11vnc -forever -shared -nopw -quiet 2>/dev/null >/dev/null & - -echo "Starting novnc on port 8282" -websockify --daemon --web /usr/share/novnc 8282 localhost:5900 2>/dev/null - -echo "Starting fluxbox" -fluxbox 2>/dev/null & -EOF - -chmod +x /usr/local/bin/start-novnc \ No newline at end of file diff --git a/.devcontainer/feature-playwright/devcontainer-feature.json b/.devcontainer/feature-playwright/devcontainer-feature.json deleted file mode 100644 index f4926f6..0000000 --- a/.devcontainer/feature-playwright/devcontainer-feature.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "Playwright", - "id": "playwright", - "version": "1.0.0", - "description": "A feature that installs Playwright and it's dependencies. Note that you need to have npm and an X server configured.", - "options": { - "version": { - "type": "string", - "default": "latest", - "description": "The version of Playwright to install (ex. '1.31.1'). Use 'latest' to get the latest version." - } - }, - "installsAfter": [ - "ghcr.io/devcontainers/features/common-utils" - ], - "containerEnv": { - "PLAYWRIGHT_BROWSERS_PATH": "/ms-playwright" - } -} \ No newline at end of file diff --git a/.devcontainer/feature-playwright/install.sh b/.devcontainer/feature-playwright/install.sh deleted file mode 100644 index b7fe2ff..0000000 --- a/.devcontainer/feature-playwright/install.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -set -e - -echo "Activating feature 'playwright' on version $VERSION" -env - -apt update -npm -g install "playwright@$VERSION" -npx playwright install-deps diff --git a/.vscode/launch.json b/.vscode/launch.json index 3836299..30b5ae5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,12 +9,14 @@ "program": "${workspaceFolder}/dist/awardwiz-scrapers/main-debug.js", "args": "${input:run-scraper-params}", "envFile": "${workspaceFolder}/.env", + "env": { + "TZ": "America/Los_Angeles", + }, "request": "launch", "skipFiles": ["/**", "**/node_modules/**"], "type": "node", "preLaunchTask": "tsc: build - tsconfig.json", "console": "integratedTerminal", - "internalConsoleOptions": "openOnSessionStart", } ], "inputs": [ diff --git a/Justfile b/Justfile index 0d23ecd..1e97b86 100644 --- a/Justfile +++ b/Justfile @@ -17,7 +17,7 @@ check: build test actionlint -color hadolint **/Dockerfile npm exec -- ajv -s config.schema.json -d config.json - shellcheck **/*.sh .devcontainer/**/*.sh + shellcheck **/*.sh NODE_NO_WARNINGS=1 npm exec -- depcheck --ignores depcheck,npm-check,typescript,devtools-protocol,@types/har-format,@iconify/json,~icons,@vitest/coverage-c8,vite-node,node-fetch,geo-tz,@types/node-fetch,@svgr/plugin-jsx,typescript-json-schema,ajv-cli @echo 'ok' diff --git a/arkalis/arkalis.ts b/arkalis/arkalis.ts index 38f471f..e5917e7 100644 --- a/arkalis/arkalis.ts +++ b/arkalis/arkalis.ts @@ -117,6 +117,7 @@ export type ArkalisCore = { pause: () => Promise, scraperMeta: Required, debugOptions: Required, + identifier: string, } type ArkalisPluginBuiltins = { @@ -149,7 +150,7 @@ async function runArkalisAttempt(code: (arkalis: Arkalis) => Promise, debu log(`Starting Arkalis run for scraper ${scraperMeta.name}`) const loadedPlugins: ArkalisPluginExports[] = [] - const arkalisCore: ArkalisCore = { client: undefined! as CDP.Client, log, warn, wait, scraperMeta, debugOptions, pause } + const arkalisCore: ArkalisCore = { client: undefined! as CDP.Client, log, warn, wait, scraperMeta, debugOptions, pause, identifier } // Loading plugins one at a time, populating the Arkalis object with their exports. Note that though we cast this // object as ArkalisCore, it can be recasted to Arkalis in the plugin, allowing access to previous plugins' exports. diff --git a/arkalis/browser.ts b/arkalis/browser.ts index 6eb4bd6..d597f0a 100644 --- a/arkalis/browser.ts +++ b/arkalis/browser.ts @@ -1,62 +1,112 @@ -import { promisify } from "util" -import { exec as execNoPromise } from "node:child_process" import url from "node:url" -import ChromeLauncher from "chrome-launcher" import { Arkalis, ArkalisCore } from "./arkalis.js" import CDP from "chrome-remote-interface" +import Dockerode from "dockerode" +import path from "node:path" +import fs from "node:fs" + +const launchChromeViaOsRunDocker = async (arkalis: ArkalisCore, switches: string[]) => { + switches.push(...[ + "user-data-dir=c:\\chrome-user-data", + arkalis.scraperMeta.useGlobalBrowserCache ? "disk-cache-dir=\"\\\\10.0.2.4\\qemu\\chrome-cache\"" : "", + ].filter(s => s !== "")) + + const command = + "netsh interface portproxy add v4tov4 listenaddress=10.0.2.15 listenport=9222 connectaddress=127.0.0.1 connectport=9222 " + + ` & "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" ${switches.map(s => s.length > 0 ? `--${s}` : "").join(" ")}` + arkalis.debugOptions.browserDebug && arkalis.log(`Launching chrome with command: ${command}`) + + const globalBrowserCacheDir = path.resolve(arkalis.debugOptions.globalBrowserCacheDir) + if (arkalis.scraperMeta.useGlobalBrowserCache && !fs.existsSync(globalBrowserCacheDir)) + fs.mkdirSync(globalBrowserCacheDir, { recursive: true }) + arkalis.debugOptions.browserDebug && arkalis.log(`Using global browser cache: ${globalBrowserCacheDir}`) + + const docker = new Dockerode() + const containerName = `arkalis-${arkalis.identifier}` + void docker.run("ghcr.io/lg/osrun", ["-f", "9222", command], process.stdout, { + name: containerName, + ExposedPorts: { "8000/tcp": {}, "9222/tcp": {} }, + Env: [ "TZ=America/Los_Angeles" ], + HostConfig: { + AutoRemove: true, + Mounts: [ + { Type: "bind", Source: "/osruncache", Target: "/cache" }, + arkalis.scraperMeta.useGlobalBrowserCache ? { Type: "bind", Source: globalBrowserCacheDir, Target: "/tmp/qemu-status/chrome-cache" } : undefined, + ].filter(m => !!m) as Dockerode.MountConfig, + Devices: [ { PathOnHost: "/dev/kvm", PathInContainer: "/dev/kvm", CGroupPermissions: "rwm" } ], + PortBindings: { "8000/tcp": [{ HostPort: "8000" }], "9222/tcp": [{ HostPort: "9222" }] }, + } + }, ) -const exec = promisify(execNoPromise) + arkalis.debugOptions.browserDebug && arkalis.log("Waiting for chrome to be ready on port 9222") + let client = undefined + while ((client = await CDP({ port: 9222 }).catch(() => undefined)) === undefined) { + await arkalis.wait(500) + } + arkalis.debugOptions.browserDebug && arkalis.log("Chrome ready on port 9222") -export const arkalisBrowser = async (arkalis: ArkalisCore) => { - async function genWindowCoords() { - const screenResolution = await exec("xdpyinfo | grep dimensions") - const rawRes = / (?\d+x\d+) /u.exec(screenResolution.stdout)?.groups?.["res"]?.trim().split("x") - if (!rawRes || rawRes.length !== 2) - throw new Error("Unable to get screen resolution") - const res = (rawRes as [string, string]).map(num => parseInt(num)) as [number, number] - const size = [Math.ceil(res[0] * (Math.random() * 0.2 + 0.8)), Math.ceil(res[1] * (Math.random() * 0.2 + 0.8))] as const - return { - size, - pos: [Math.ceil((res[0] - size[0]) * Math.random()), Math.ceil((res[1] - size[1]) * Math.random())] as const + return { + client, + closeBrowser: async () => { + await closeCDPClient(arkalis) + + arkalis.debugOptions.browserDebug && arkalis.log("Waiting for browser to close on its own") + const startTime = Date.now() + const container = docker.getContainer(containerName) + while (await container.inspect().catch(() => undefined) !== undefined && Date.now() - startTime < 3000) { + await arkalis.wait(200) + } + if (await container.inspect().catch(() => undefined) !== undefined) { + arkalis.debugOptions.browserDebug && arkalis.log("Browser did not close on its own after 3 seconds, killing it") + await container.stop({ t: 0, signal: "SIGINT" }).catch(() => {}) + } } } +} + +const closeCDPClient = async (arkalis: ArkalisCore) => { + arkalis.debugOptions.browserDebug && arkalis.log("Closing cdp client") - // generate a random window size - const window = await genWindowCoords() + for (const domain of [arkalis.client.Network, arkalis.client.Page, arkalis.client.Runtime, arkalis.client.DOM]) + await domain.disable().catch(() => {}) + await arkalis.client.Browser.close().catch(() => {}) + await arkalis.client.close().catch(() => {}) +} +export const arkalisBrowser = async (arkalis: ArkalisCore) => { // these domains are used by the browser when creating a new profile const blockDomains = [ - "accounts.google.com", "clients2.google.com", "optimizationguide-pa.googleapis.com", - "content-autofill.googleapis.com" + "accounts.google.com", "clients2.google.com", "optimizationguide-pa.googleapis.com", "edgedl.me.gvt1.com", + "content-autofill.googleapis.com", "update.googleapis.com" ] const switches = [ // these should all be undetectable, but speed things up - "disable-sync", "disable-backgrounding-occluded-windows", "disable-breakpad", - "disable-domain-reliability", "disable-background-networking", "disable-features=AutofillServerCommunication", - "disable-features=CertificateTransparencyComponentUpdater", "enable-crash-reporter-for-testing", "no-service-autorun", + "disable-features=OptimizationHints,MediaRouter,AutofillServerCommunication,CertificateTransparencyComponentUpdater,CalculateNativeWinOcclusion,InterestFeedContentSuggestions,Translate", + "disable-sync", "disable-backgrounding-occluded-windows", "disable-breakpad", "disable-renderer-backgrounding", + "disable-domain-reliability", "disable-background-networking", "disable-background-timer-throttling", + "enable-crash-reporter-for-testing", "no-service-autorun", "disable-ipc-flooding-protection", "password-store=basic", "no-first-run", "no-default-browser-check", "disable-prompt-on-repost", "disable-client-side-phishing-detection", - "disable-features=InterestFeedContentSuggestions", "disable-features=Translate", "disable-hang-monitor", - "autoplay-policy=no-user-gesture-required", "use-mock-keychain", "disable-omnibox-autocomplete-off-method", + "disable-hang-monitor", "autoplay-policy=no-user-gesture-required", "use-mock-keychain", "disable-omnibox-autocomplete-off-method", "disable-gaia-services", "disable-crash-reporter", "noerrdialogs", "disable-component-update", - "disable-features=MediaRouter", "metrics-recording-only", "disable-features=OptimizationHints", - "disable-component-update", "disable-features=CalculateNativeWinOcclusion", "enable-precise-memory-info", + "metrics-recording-only", "disable-component-update", "enable-precise-memory-info", + "force-fieldtrials=*BackgroundTracing/default/", - "no-sandbox", "disable-dev-shm-usage", // for linux docker + // "no-sandbox", "disable-dev-shm-usage", // for linux docker // "disable-blink-features=AutomationControlled", // not working // "auto-open-devtools-for-tabs", - // "log-net-log=tmp/out.json", "net-log-capture-mode=Everything", // note, does not log requests - // TODO: pass this in dyanmically from a hook in the har scraper - "log-net-log=./tmp/netlog.json", "net-log-capture-mode=Everything", + //"log-net-log=./tmp/netlog.json", "net-log-capture-mode=Everything", arkalis.debugOptions.browserDebug === "verbose" ? "enable-logging=stderr": "", arkalis.debugOptions.browserDebug === "verbose" ? "v=2" : "", - arkalis.scraperMeta.useGlobalBrowserCache ? `disk-cache-dir=${arkalis.debugOptions.globalBrowserCacheDir}` : "", - `window-position=${window.pos[0]},${window.pos[1]}`, - `window-size=${window.size[0]},${window.size[1]}`, - `host-rules=${blockDomains.map(blockDomain => `MAP ${blockDomain} 0.0.0.0`).join(", ")}`, // NOTE: detectable! - ] + // arkalis.scraperMeta.useGlobalBrowserCache ? `disk-cache-dir=${arkalis.debugOptions.globalBrowserCacheDir}` : "", + // `window-position=${window.pos[0]},${window.pos[1]}`, + // `window-size=${window.size[0]},${window.size[1]}`, + `host-rules="${blockDomains.map(blockDomain => `MAP ${blockDomain} 0.0.0.0`).join(", ")}"`, // NOTE: detectable! + + "remote-debugging-port=9222" + ].filter(s => s !== "") // apply proxy const proxy = (arkalis as Arkalis).proxy @@ -69,15 +119,9 @@ export const arkalisBrowser = async (arkalis: ArkalisCore) => { } // launch chrome - const instance = await ChromeLauncher.launch({ - chromeFlags: switches.map(s => s.length > 0 ? `--${s}` : ""), - ignoreDefaultFlags: true, - logLevel: arkalis.debugOptions.browserDebug ? "verbose" : "silent", - }) - - // connect to cdp client - arkalis.debugOptions.browserDebug && arkalis.log("connecting to cdp client") - arkalis.client = await CDP({ port: instance.port }) + const { closeBrowser, client } = await launchChromeViaOsRunDocker(arkalis, switches) + arkalis.client = client + await arkalis.client.Network.enable() await arkalis.client.Page.enable() await arkalis.client.Runtime.enable() @@ -92,18 +136,6 @@ export const arkalisBrowser = async (arkalis: ArkalisCore) => { await arkalis.client.Network.setBlockedURLs({ urls: arkalis.scraperMeta.blockUrls }) return { - close: async () => { - arkalis.debugOptions.browserDebug && arkalis.log("closing cdp client and browser") - - await arkalis.client.Network.disable().catch(() => {}) - await arkalis.client.Page.disable().catch(() => {}) - await arkalis.client.Runtime.disable().catch(() => {}) - await arkalis.client.DOM.disable().catch(() => {}) - - await arkalis.client.Browser.close().catch(() => {}) - await arkalis.client.close().catch(() => {}) - - instance.kill() - } + close: closeBrowser } } diff --git a/package-lock.json b/package-lock.json index f37a995..2be6474 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,11 +16,11 @@ "antd": "^5.6.3", "bezier-js": "^6.1.3", "bottleneck": "^2.19.5", - "chrome-launcher": "^0.15.2", "chrome-remote-interface": "^0.32.2", "cors": "^2.8.5", "cross-fetch": "^3.1.8", "dayjs": "^1.11.9", + "dockerode": "^3.3.5", "dotenv": "^16.1.3", "express": "^4.18.2", "express-jwt": "^8.4.1", @@ -42,6 +42,7 @@ "@types/bezier-js": "^4.1.0", "@types/chrome-remote-interface": "^0.31.10", "@types/cors": "^2.8.13", + "@types/dockerode": "^3.3.19", "@types/express": "^4.17.17", "@types/glob-to-regexp": "^0.4.1", "@types/har-format": "^1.2.10", @@ -508,6 +509,11 @@ "node": ">=6.9.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -2861,6 +2867,26 @@ "@types/node": "*" } }, + "node_modules/@types/docker-modem": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.3.tgz", + "integrity": "sha512-i1A2Etnav7uHizZ87vUf4EqwJehY3JOcTfBS0pGBlO+HQ0jg2lUMCaJRg9VQM8ldZkpYdIfsenxcTOCpwxPXEg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.19.tgz", + "integrity": "sha512-7CC5yIpQi+bHXwDK43b/deYXteP3Lem9gdocVVHJPSRJJLMfbiOchQV3rDmAPkMw+n3GIVj7m1six3JW+VcwwA==", + "dev": true, + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*" + } + }, "node_modules/@types/express": { "version": "4.17.17", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.17.tgz", @@ -3098,6 +3124,21 @@ "@types/node": "*" } }, + "node_modules/@types/ssh2": { + "version": "1.11.13", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.11.13.tgz", + "integrity": "sha512-08WbG68HvQ2YVi74n2iSUnYHYpUdFc/s2IsI0BHBdJwaqYJpWlVv9elL0tYShTv60yr0ObdxJR5NrCRiGJ/0CQ==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.17.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.6.tgz", + "integrity": "sha512-fGmT/P7z7ecA6bv/ia5DlaWCH4YeZvAQMNpUhrJjtAhOhZfoxS1VLUgU2pdk63efSjQaOJWdXMuAJsws+8I6dg==", + "dev": true + }, "node_modules/@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -4088,6 +4129,14 @@ "node": ">=8" } }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -4177,7 +4226,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "devOptional": true, "funding": [ { "type": "github", @@ -4193,6 +4241,14 @@ } ] }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, "node_modules/bezier-js": { "version": "6.1.3", "resolved": "https://registry.npmjs.org/bezier-js/-/bezier-js-6.1.3.tgz", @@ -4224,7 +4280,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -4456,7 +4511,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -4487,6 +4541,15 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/buildcheck": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", + "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -4743,33 +4806,10 @@ "node": ">= 6" } }, - "node_modules/chrome-launcher": { - "version": "0.15.2", - "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-0.15.2.tgz", - "integrity": "sha512-zdLEwNo3aUVzIhKhTtXfxhdvZhUghrnmkvcAq2NoDd+LeOHKf03H5jwZ8T/STsAlzyALkBVK552iaG1fGf1xVQ==", - "dependencies": { - "@types/node": "*", - "escape-string-regexp": "^4.0.0", - "is-wsl": "^2.2.0", - "lighthouse-logger": "^1.0.0" - }, - "bin": { - "print-chrome-path": "bin/print-chrome-path.js" - }, - "engines": { - "node": ">=12.13.0" - } - }, - "node_modules/chrome-launcher/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "node_modules/chrome-remote-interface": { "version": "0.32.2", @@ -5192,6 +5232,20 @@ "node": ">=10" } }, + "node_modules/cpu-features": { + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz", + "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.17.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -5569,6 +5623,33 @@ "node": ">=8" } }, + "node_modules/docker-modem": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-3.0.8.tgz", + "integrity": "sha512-f0ReSURdM3pcKPNS30mxOHSbaFLcknGmQjwSfmbcdOw1XWKXVhukM3NJHhr7NpY9BIyyWQb0EBo3KQvvuU5egQ==", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.11.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-3.3.5.tgz", + "integrity": "sha512-/0YNa3ZDNeLr/tSckmD69+Gq+qVNhvKfAHNeZJBnp7EOP6RGKV8ORrJHkUn20So5wU+xxT7+1n5u8PjHbfjbSA==", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "docker-modem": "^3.0.0", + "tar-fs": "~2.0.1" + }, + "engines": { + "node": ">= 8.0" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -5682,7 +5763,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "devOptional": true, "dependencies": { "once": "^1.4.0" } @@ -7088,6 +7168,11 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -8200,7 +8285,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -8557,6 +8641,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, "bin": { "is-docker": "cli.js" }, @@ -8830,6 +8915,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, "dependencies": { "is-docker": "^2.0.0" }, @@ -9197,28 +9283,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lighthouse-logger": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-1.4.2.tgz", - "integrity": "sha512-gPWxznF6TKmUHrOQjlVo2UbaL2EJ71mb2CCeRs/2qBpi4L/g4LUVc9+3lKQ6DTUZwJswfM7ainGrLO1+fOqa2g==", - "dependencies": { - "debug": "^2.6.9", - "marky": "^1.2.2" - } - }, - "node_modules/lighthouse-logger/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/lighthouse-logger/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, "node_modules/limiter": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", @@ -9819,11 +9883,6 @@ "node": ">= 12" } }, - "node_modules/marky": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.5.tgz", - "integrity": "sha512-q9JtQJKjpsVxCRVgQ+WapguSbKC3SQ5HEzFGPAJMStgh3QjCawp00UKv3MTTAArTmGmmPUvllHZoNbZ3gs0I+Q==" - }, "node_modules/md5-hex": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz", @@ -10043,6 +10102,11 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/mlly": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.4.0.tgz", @@ -10091,6 +10155,12 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "node_modules/nan": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", + "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.6", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", @@ -10562,7 +10632,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "devOptional": true, "dependencies": { "wrappy": "1" } @@ -11218,7 +11287,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -12935,12 +13003,34 @@ "integrity": "sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w==", "dev": true }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/ssh2": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.14.0.tgz", + "integrity": "sha512-AqzD1UCqit8tbOKoj6ztDDi1ffJZ2rV2SwlgrVVrHPkV5vWqGJOVp5pmtj18PunkPJAuKQsnInyKV+/Nb2bUnA==", + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.8", + "nan": "^2.17.0" + } + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -13190,6 +13280,32 @@ "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz", "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==" }, + "node_modules/tar-fs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", + "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/teeny-request": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.3.tgz", @@ -13404,6 +13520,11 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -14464,8 +14585,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "devOptional": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { "version": "3.0.3", diff --git a/package.json b/package.json index 11437ac..b28986c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@types/bezier-js": "^4.1.0", "@types/chrome-remote-interface": "^0.31.10", "@types/cors": "^2.8.13", + "@types/dockerode": "^3.3.19", "@types/express": "^4.17.17", "@types/glob-to-regexp": "^0.4.1", "@types/har-format": "^1.2.10", @@ -59,11 +60,11 @@ "antd": "^5.6.3", "bezier-js": "^6.1.3", "bottleneck": "^2.19.5", - "chrome-launcher": "^0.15.2", "chrome-remote-interface": "^0.32.2", "cors": "^2.8.5", "cross-fetch": "^3.1.8", "dayjs": "^1.11.9", + "dockerode": "^3.3.5", "dotenv": "^16.1.3", "express": "^4.18.2", "express-jwt": "^8.4.1",