From 93bd75c94dd6966494f9f4e8935681bed02fde31 Mon Sep 17 00:00:00 2001 From: jalas167 Date: Thu, 19 Oct 2023 16:00:59 +0200 Subject: [PATCH 01/38] Create outbound-ipfs.mjs --- .../accessing-internet/outbound-ipfs.mjs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs diff --git a/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs b/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs new file mode 100644 index 000000000..efe90ce29 --- /dev/null +++ b/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs @@ -0,0 +1,34 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { readFile } from "fs/promises"; + +const url = + "https://ipfs.io/ipfs/bafybeihkoviema7g3gxyt6la7vd5ho32ictqbilu3wnlo3rs7ewhnp7lly"; + +(async function main() { + // Load the manifest. + const manifest = await readFile(`./manifest.json`); + + // Create and configure a TaskExecutor instance. + const executor = await TaskExecutor.create({ + capabilities: ["inet", "manifest-support"], + yagnaOptions: { apiKey: "try_golem" }, + manifest: manifest.toString("base64"), + }); + + try { + await executor.run(async (ctx) => { + const result = await ctx.run(`curl ${url} -o /golem/work/example.jpg`); + + console.log((await ctx.run("ls -l")).stdout); + if (result.result === "Ok") { + console.log("File downloaded!"); + } else { + console.error("Failed to download the file!", result.stderr); + } + }); + } catch (err) { + console.error("The task failed due to", err); + } finally { + await executor.end(); + } +})(); From 22515255592646cab146999fa9a39d838d260391 Mon Sep 17 00:00:00 2001 From: jalas167 <42438166+jalas167@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:22:48 +0200 Subject: [PATCH 02/38] Update outbound-ipfs.mjs --- .../tutorials/accessing-internet/outbound-ipfs.mjs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs b/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs index efe90ce29..ccda53ba0 100644 --- a/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs +++ b/examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs @@ -1,8 +1,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; import { readFile } from "fs/promises"; -const url = - "https://ipfs.io/ipfs/bafybeihkoviema7g3gxyt6la7vd5ho32ictqbilu3wnlo3rs7ewhnp7lly"; +const url = "https://ipfs.io/ipfs/bafybeihkoviema7g3gxyt6la7vd5ho32ictqbilu3wnlo3rs7ewhnp7lly"; (async function main() { // Load the manifest. From 03e1c8db35a15d6f230ae3f8253ed299eefc50e3 Mon Sep 17 00:00:00 2001 From: jalas167 Date: Fri, 20 Oct 2023 11:15:40 +0200 Subject: [PATCH 03/38] doc: add manifest and outbound example to test list --- .../accessing-internet/manifest.json | 33 +++++++++++++++++++ tests/examples/examples.json | 1 + 2 files changed, 34 insertions(+) create mode 100644 examples/docs-examples/tutorials/accessing-internet/manifest.json diff --git a/examples/docs-examples/tutorials/accessing-internet/manifest.json b/examples/docs-examples/tutorials/accessing-internet/manifest.json new file mode 100644 index 000000000..1674f149e --- /dev/null +++ b/examples/docs-examples/tutorials/accessing-internet/manifest.json @@ -0,0 +1,33 @@ +{ + "version": "0.1.0", + "createdAt": "2023-10-19T11:23:08.156+02:00", + "expiresAt": "2024-01-17T11:23:08.156+01:00", + "metadata": { + "name": "outbound-docs", + "description": "", + "version": "1.0.0" + }, + "payload": [ + { + "platform": { + "os": "linux", + "arch": "x86_64" + }, + "hash": "sha3:dad8f776b0eb9f37ea0d63de42757034dd085fe30cc4537c2e119d80", + "urls": [ + "http://registry.golem.network/download/f37c8ba2b534ca631060fb8db4ac218d3199faf656aa2c92f402c2b700797c21" + ] + } + ], + "compManifest": { + "version": "0.1.0", + "net": { + "inet": { + "out": { + "urls": ["https://ipfs.io"], + "protocols": ["https"] + } + } + } + } +} diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 3e6bb188c..778459424 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -45,6 +45,7 @@ { "cmd": "node", "path": "examples/docs-examples/quickstarts/quickstart/requestor.mjs" }, { "cmd": "node", "path": "examples/docs-examples/tutorials/building-custom-image/index.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs" }, { "cmd": "node", "path": "examples/docs-examples/tutorials/quickstart/index.mjs" }, { "cmd": "node", From 0f332e7b0b6245ab812558009103038cf4212dbf Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Mon, 23 Oct 2023 11:14:44 +0200 Subject: [PATCH 04/38] test: retry compose down before erroring out --- tests/docker/manageDocker.sh | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/docker/manageDocker.sh diff --git a/tests/docker/manageDocker.sh b/tests/docker/manageDocker.sh new file mode 100644 index 000000000..8f64c3d76 --- /dev/null +++ b/tests/docker/manageDocker.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Maximum number of attempts to bring down the Docker Compose +max_attempts=5 + +# Counter for the number of attempts +attempt=0 + +# Path to your docker-compose file +compose_file="tests/docker/docker-compose.yml" + +# Function to bring up the services +start_services() { + docker-compose -f $compose_file up -d +} + +# Loop to attempt 'docker-compose down' with retries +while [ $attempt -lt $max_attempts ]; do + # Increment the attempt counter + ((attempt=attempt+1)) + + # Try to bring down the services + docker-compose -f $compose_file down + + # Check if the command succeeded + if [ $? -eq 0 ]; then + echo "Successfully brought down the services." + # If successful, break out of the loop + break + else + echo "Attempt $attempt failed..." + # If max attempts reached, show error and exit + if [ $attempt -eq $max_attempts ]; then + echo "Failed to bring down the services after $max_attempts attempts." + exit 1 + fi + # If not, wait for a bit before retrying + echo "Retrying in 5 seconds..." + sleep 5 + fi +done + +# If we reached here, it means we successfully brought the services down. +# So we start them up again. +start_services From a245d7f49ca1bd208cc9489be199e8f2b309929f Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Mon, 23 Oct 2023 11:16:58 +0200 Subject: [PATCH 05/38] test: use new script to manage docker on action run --- .github/workflows/cypress-nightly.yml | 2 +- .github/workflows/examples-nightly.yml | 2 +- .github/workflows/goth-nightly.yml | 2 +- .github/workflows/release.yml | 2 +- tests/docker/Requestor.Dockerfile | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cypress-nightly.yml b/.github/workflows/cypress-nightly.yml index ca7253bf4..6cf4b4440 100644 --- a/.github/workflows/cypress-nightly.yml +++ b/.github/workflows/cypress-nightly.yml @@ -24,7 +24,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d + run: /manageDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index a5daacefd..ad7612ae8 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -40,7 +40,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d + run: /manageDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index ca10cb93d..4e583f6bb 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -40,7 +40,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d + run: /manageDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef1b52405..d1a73088b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,7 +63,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d + run: /manageDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/tests/docker/Requestor.Dockerfile b/tests/docker/Requestor.Dockerfile index b64167fdd..2b56c5f76 100644 --- a/tests/docker/Requestor.Dockerfile +++ b/tests/docker/Requestor.Dockerfile @@ -40,5 +40,6 @@ RUN apt-get update -q \ COPY ./startRequestor.sh /startRequestor.sh +COPY ./manageDocker.sh /manageDocker.sh CMD ["bash", "-c", "/startRequestor.sh"] From 13e3eb1dca86f9ae7653945bd7ff5ce35873483f Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Mon, 23 Oct 2023 11:30:46 +0200 Subject: [PATCH 06/38] test: use correct file path --- .github/workflows/cypress-nightly.yml | 2 +- .github/workflows/examples-nightly.yml | 2 +- .github/workflows/goth-nightly.yml | 2 +- .github/workflows/release.yml | 2 +- tests/docker/Requestor.Dockerfile | 1 - tests/docker/{manageDocker.sh => startDocker.sh} | 0 6 files changed, 4 insertions(+), 5 deletions(-) rename tests/docker/{manageDocker.sh => startDocker.sh} (100%) mode change 100644 => 100755 diff --git a/.github/workflows/cypress-nightly.yml b/.github/workflows/cypress-nightly.yml index 6cf4b4440..f8e07d128 100644 --- a/.github/workflows/cypress-nightly.yml +++ b/.github/workflows/cypress-nightly.yml @@ -24,7 +24,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: /manageDocker.sh + run: tests/docker/startDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index ad7612ae8..0b4fd7de3 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -40,7 +40,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: /manageDocker.sh + run: tests/docker/startDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index 4e583f6bb..fcbc66d25 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -40,7 +40,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: /manageDocker.sh + run: tests/docker/startDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1a73088b..9ac91d286 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,7 +63,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: /manageDocker.sh + run: tests/docker/startDocker.sh - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/tests/docker/Requestor.Dockerfile b/tests/docker/Requestor.Dockerfile index 2b56c5f76..b64167fdd 100644 --- a/tests/docker/Requestor.Dockerfile +++ b/tests/docker/Requestor.Dockerfile @@ -40,6 +40,5 @@ RUN apt-get update -q \ COPY ./startRequestor.sh /startRequestor.sh -COPY ./manageDocker.sh /manageDocker.sh CMD ["bash", "-c", "/startRequestor.sh"] diff --git a/tests/docker/manageDocker.sh b/tests/docker/startDocker.sh old mode 100644 new mode 100755 similarity index 100% rename from tests/docker/manageDocker.sh rename to tests/docker/startDocker.sh From a014c73c48108a5c89d1e0d78277403c8e6d0454 Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Mon, 23 Oct 2023 11:32:51 +0200 Subject: [PATCH 07/38] test: fix command --- tests/docker/startDocker.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/docker/startDocker.sh b/tests/docker/startDocker.sh index 8f64c3d76..ca5df56e3 100755 --- a/tests/docker/startDocker.sh +++ b/tests/docker/startDocker.sh @@ -11,16 +11,16 @@ compose_file="tests/docker/docker-compose.yml" # Function to bring up the services start_services() { - docker-compose -f $compose_file up -d + docker compose -f $compose_file up -d } -# Loop to attempt 'docker-compose down' with retries +# Loop to attempt 'docker compose down' with retries while [ $attempt -lt $max_attempts ]; do # Increment the attempt counter ((attempt=attempt+1)) # Try to bring down the services - docker-compose -f $compose_file down + docker compose -f $compose_file down # Check if the command succeeded if [ $? -eq 0 ]; then From 44d2be60246f31e23382f938d440cbe81e8a45eb Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Mon, 23 Oct 2023 11:36:09 +0200 Subject: [PATCH 08/38] test: restart docker service --- tests/docker/startDocker.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker/startDocker.sh b/tests/docker/startDocker.sh index ca5df56e3..b5f5a24fd 100755 --- a/tests/docker/startDocker.sh +++ b/tests/docker/startDocker.sh @@ -1,5 +1,7 @@ #!/bin/bash +# Restart docker first as it may be in a bad state. +sudo service docker restart # Maximum number of attempts to bring down the Docker Compose max_attempts=5 From cee329a29ef9418c379b3ad5f56fc06b8c4cdf9a Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Tue, 24 Oct 2023 08:23:32 +0200 Subject: [PATCH 09/38] test: scratch script and just restart docker --- .github/workflows/cypress-nightly.yml | 2 +- .github/workflows/examples-nightly.yml | 2 +- .github/workflows/goth-nightly.yml | 2 +- .github/workflows/release.yml | 6 ++-- tests/docker/startDocker.sh | 47 -------------------------- 5 files changed, 6 insertions(+), 53 deletions(-) delete mode 100755 tests/docker/startDocker.sh diff --git a/.github/workflows/cypress-nightly.yml b/.github/workflows/cypress-nightly.yml index f8e07d128..9cfdee853 100644 --- a/.github/workflows/cypress-nightly.yml +++ b/.github/workflows/cypress-nightly.yml @@ -24,7 +24,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: tests/docker/startDocker.sh + run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 0b4fd7de3..7d2f311d9 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -40,7 +40,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: tests/docker/startDocker.sh + run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index fcbc66d25..34f9aeeca 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -40,7 +40,7 @@ jobs: - name: Start the docker containers # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: tests/docker/startDocker.sh + run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ac91d286..3c2d26a74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,15 +55,15 @@ jobs: uses: actions/checkout@v3 - name: Use random string for subnet + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - name: Build the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: docker compose -f tests/docker/docker-compose.yml build - name: Start the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 - run: tests/docker/startDocker.sh + # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run. + run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. diff --git a/tests/docker/startDocker.sh b/tests/docker/startDocker.sh deleted file mode 100755 index b5f5a24fd..000000000 --- a/tests/docker/startDocker.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Restart docker first as it may be in a bad state. -sudo service docker restart -# Maximum number of attempts to bring down the Docker Compose -max_attempts=5 - -# Counter for the number of attempts -attempt=0 - -# Path to your docker-compose file -compose_file="tests/docker/docker-compose.yml" - -# Function to bring up the services -start_services() { - docker compose -f $compose_file up -d -} - -# Loop to attempt 'docker compose down' with retries -while [ $attempt -lt $max_attempts ]; do - # Increment the attempt counter - ((attempt=attempt+1)) - - # Try to bring down the services - docker compose -f $compose_file down - - # Check if the command succeeded - if [ $? -eq 0 ]; then - echo "Successfully brought down the services." - # If successful, break out of the loop - break - else - echo "Attempt $attempt failed..." - # If max attempts reached, show error and exit - if [ $attempt -eq $max_attempts ]; then - echo "Failed to bring down the services after $max_attempts attempts." - exit 1 - fi - # If not, wait for a bit before retrying - echo "Retrying in 5 seconds..." - sleep 5 - fi -done - -# If we reached here, it means we successfully brought the services down. -# So we start them up again. -start_services From c6d7989522c7ca530af573513e34c7f12b29b48e Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Tue, 24 Oct 2023 09:32:17 +0200 Subject: [PATCH 10/38] test(example): reduce custom price array length check --- .../examples/selecting-providers/custom-price.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/docs-examples/examples/selecting-providers/custom-price.mjs b/examples/docs-examples/examples/selecting-providers/custom-price.mjs index d4a604006..411bbaa84 100644 --- a/examples/docs-examples/examples/selecting-providers/custom-price.mjs +++ b/examples/docs-examples/examples/selecting-providers/custom-price.mjs @@ -12,10 +12,10 @@ const myFilter = async (proposal) => { let counterIdx = usageVector.findIndex((ele) => ele === "golem.usage.duration_sec"); let proposedCost = proposal.properties["golem.com.pricing.model.linear.coeffs"][counterIdx]; costData.push(proposedCost); - if (costData.length < 11) return false; + if (costData.length < 6) return false; else { costData.shift(); - let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 10; + let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 5; if (proposedCost <= 1.2 * averageProposedCost) decision = true; if (decision) { console.log(proposedCost, averageProposedCost); From 458423cb8654e75e20b198b5f4a9d41bc1e7f6d0 Mon Sep 17 00:00:00 2001 From: Phillip Jensen Date: Tue, 24 Oct 2023 10:18:57 +0200 Subject: [PATCH 11/38] test: update comments --- .github/workflows/cypress-nightly.yml | 4 ++-- .github/workflows/examples-nightly.yml | 4 ++-- .github/workflows/goth-nightly.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cypress-nightly.yml b/.github/workflows/cypress-nightly.yml index 9cfdee853..65f60f231 100644 --- a/.github/workflows/cypress-nightly.yml +++ b/.github/workflows/cypress-nightly.yml @@ -16,14 +16,14 @@ jobs: uses: actions/checkout@v3 - name: Use random string for subnet + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - name: Build the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: docker compose -f tests/docker/docker-compose.yml build - name: Start the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run. run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 7d2f311d9..9ddb33568 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -32,14 +32,14 @@ jobs: uses: actions/checkout@v3 - name: Use random string for subnet + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - name: Build the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: docker compose -f tests/docker/docker-compose.yml build - name: Start the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run. run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index 34f9aeeca..f6c4cd505 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -32,14 +32,14 @@ jobs: uses: actions/checkout@v3 - name: Use random string for subnet + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - name: Build the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 run: docker compose -f tests/docker/docker-compose.yml build - name: Start the docker containers - # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run. run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - name: Fund the requestor From 8201d33ce3721d57fca1244b729b7b512da05752 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 30 Oct 2023 09:58:27 +0100 Subject: [PATCH 12/38] feat(work_context): added runAndStream method for streaming command results --- examples/package.json | 1 + examples/simple-usage/runStream.ts | 8 ++++++ src/activity/activity.ts | 46 +++++++++++++----------------- src/network/index.ts | 2 +- src/task/work.ts | 41 ++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 27 deletions(-) create mode 100644 examples/simple-usage/runStream.ts diff --git a/examples/package.json b/examples/package.json index 958b99e08..3c0a01122 100644 --- a/examples/package.json +++ b/examples/package.json @@ -7,6 +7,7 @@ "scripts": { "hello": "node ./hello-world/hello.mjs", "run": "ts-node ./simple-usage/run.ts", + "runStream": "ts-node ./simple-usage/runStream.ts", "map": "ts-node ./simple-usage/map.ts", "forEach": "ts-node ./simple-usage/forEach.ts", "fileTransfer": "ts-node ./simple-usage/fileTransfer.ts", diff --git a/examples/simple-usage/runStream.ts b/examples/simple-usage/runStream.ts new file mode 100644 index 000000000..ee8ac3cd3 --- /dev/null +++ b/examples/simple-usage/runStream.ts @@ -0,0 +1,8 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +const executor = await TaskExecutor.create("golem/alpine:latest"); +const streamOfResults = await executor.run(async (ctx) => ctx.runAndStream("while sleep 1; do date; done")); +streamOfResults?.on("data", (data) => console.log(data.stdout)); +streamOfResults?.on("error", () => executor.end()); + +setTimeout(() => executor.end(), 10_000); diff --git a/src/activity/activity.ts b/src/activity/activity.ts index 4aaa9bac4..df4fb45a0 100644 --- a/src/activity/activity.ts +++ b/src/activity/activity.ts @@ -1,12 +1,11 @@ import { Result, ResultState, StreamingBatchEvent } from "./results"; import EventSource from "eventsource"; import { Readable } from "stream"; -import { Logger } from "../utils"; +import { Logger, YagnaApi } from "../utils"; import sleep from "../utils/sleep"; import { ActivityFactory } from "./factory"; import { ActivityConfig } from "./config"; import { Events } from "../events"; -import { YagnaApi } from "../utils/yagna/yagna"; export enum ActivityStateEnum { New = "New", @@ -42,6 +41,7 @@ export class Activity { private readonly logger?: Logger; private isRunning = true; private currentState: ActivityStateEnum = ActivityStateEnum.New; + private eventSource?: EventSource; /** * @param id activity ID @@ -147,6 +147,7 @@ export class Activity { } private async end() { + this.eventSource?.close(); await this.yagnaApi.activity.control .destroyActivity(this.id, this.options.activityRequestTimeout / 1000, { timeout: this.options.activityRequestTimeout + 1000, @@ -218,14 +219,18 @@ export class Activity { } private async streamingBatch(batchId, batchSize, startTime, timeout): Promise { - const basePath = this.yagnaApi.yagnaOptions.basePath; + const basePath = this.yagnaApi.activity.control["basePath"]; const apiKey = this.yagnaApi.yagnaOptions.apiKey; + const eventSource = new EventSource(`${basePath}/activity/${this.id}/exec/${batchId}`, { headers: { Accept: "text/event-stream", Authorization: `Bearer ${apiKey}`, }, }); + eventSource.addEventListener("runtime", (event) => results.push(this.parseEventToResult(event.data, batchSize))); + eventSource.addEventListener("error", (error) => errors.push(error.data?.message ?? error)); + this.eventSource = eventSource; let isBatchFinished = false; const activityId = this.id; @@ -233,23 +238,25 @@ export class Activity { const activityExecuteTimeout = this.options.activityExecuteTimeout; const errors: object[] = []; - eventSource.addEventListener("error", (error) => errors.push(error.data.message ?? error)); - const results: Result[] = []; - eventSource.addEventListener("runtime", (event) => results.push(this.parseEventToResult(event.data, batchSize))); return new Readable({ objectMode: true, async read() { while (!isBatchFinished) { + let error: Error | undefined; if (startTime.valueOf() + (timeout || activityExecuteTimeout) <= new Date().valueOf()) { - return this.destroy(new Error(`Activity ${activityId} timeout.`)); + error = new Error(`Activity ${activityId} timeout.`); } if (!isRunning()) { - return this.destroy(new Error(`Activity ${activityId} has been interrupted.`)); + error = new Error(`Activity ${activityId} has been interrupted.`); } if (errors.length) { - return this.destroy(new Error(`GetExecBatchResults failed due to errors: ${JSON.stringify(errors)}`)); + error = new Error(`GetExecBatchResults failed due to errors: ${JSON.stringify(errors)}`); + } + if (error) { + eventSource?.close(); + return this.destroy(error); } if (results.length) { const result = results.shift(); @@ -259,6 +266,7 @@ export class Activity { await sleep(500, true); } this.push(null); + eventSource?.close(); }, }); } @@ -295,22 +303,6 @@ export class Activity { ); } - private isGsbError(error) { - // check if `err` is caused by "endpoint address not found" GSB error - if (!error.response) { - return false; - } - if (error.response.status !== 500) { - return false; - } - if (!error.response.data || !error.response.data.message) { - this.logger?.debug(`Cannot read error message, response: ${error.response}`); - return false; - } - const message = error.response.data.message; - return message.includes("endpoint address not found") && message.includes("GSB error"); - } - private async isTerminated(): Promise<{ terminated: boolean; reason?: string; errorMessage?: string }> { try { const { data } = await this.yagnaApi.activity.state.getActivityState(this.id); @@ -336,7 +328,9 @@ export class Activity { ? event?.kind?.finished?.return_code === 0 ? ResultState.Ok : ResultState.Error - : ResultState.Error, + : event?.kind?.stderr + ? ResultState.Error + : ResultState.Ok, stdout: event?.kind?.stdout, stderr: event?.kind?.stderr, message: event?.kind?.finished?.message, diff --git a/src/network/index.ts b/src/network/index.ts index 72b02d615..96433cd3d 100644 --- a/src/network/index.ts +++ b/src/network/index.ts @@ -1,3 +1,3 @@ export { Network } from "./network"; export { NetworkNode } from "./node"; -export { NetworkService } from "./service"; +export { NetworkService, NetworkServiceOptions } from "./service"; diff --git a/src/task/work.ts b/src/task/work.ts index d636c0f37..a25f6d073 100644 --- a/src/task/work.ts +++ b/src/task/work.ts @@ -16,6 +16,7 @@ import { NullStorageProvider, StorageProvider } from "../storage"; import { Logger, sleep } from "../utils"; import { Batch } from "./batch"; import { NetworkNode } from "../network"; +import { Readable } from "stream"; export type Worker = ( ctx: WorkContext, @@ -135,6 +136,46 @@ export class WorkContext { return this.runOneCommand(run, runOptions); } + /** + * Execute an executable on provider and return Promise of ReadableStream + * that streams Result objects containing the stdout and stderr of the command + * while it is being executed. + * + * @param commandLine Shell command to execute. + * @param options Additional run options. + */ + async runAndStream(commandLine: string, options?: Omit): Promise; + /** + * @param executable Executable to run. + * @param args Executable arguments. + * @param options Additional run options. + */ + async runAndStream(executable: string, args: string[], options?: Omit): Promise; + async runAndStream( + exeOrCmd: string, + argsOrOptions?: string[] | Omit, + options?: Omit, + ): Promise { + const isArray = Array.isArray(argsOrOptions); + const capture: Capture = { + stdout: { stream: { format: "string" } }, + stderr: { stream: { format: "string" } }, + }; + const run = isArray + ? new Run(exeOrCmd, argsOrOptions as string[], options?.env, capture) + : new Run("/bin/sh", ["-c", exeOrCmd], argsOrOptions?.env, capture); + const script = new Script([run]); + // In this case, the script consists only of the run command, + // so we skip the execution of script.before and script.after + return this.activity.execute(script.getExeScriptRequest(), true, options?.timeout).catch((e) => { + throw new Error( + `Script execution failed for command: ${JSON.stringify(run.toJson())}. ${ + e?.response?.data?.message || e?.message || e + }`, + ); + }); + } + /** * Generic transfer command, requires the user to provide a publicly readable transfer source * From ffcddc1144ce2c28ef7811bacf6c11baf18c49c1 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 30 Oct 2023 10:47:20 +0100 Subject: [PATCH 13/38] test(work_context): unit and e2e tests for streaming results --- src/task/work.spec.ts | 11 +++++++++++ tests/e2e/tasks.spec.ts | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/task/work.spec.ts b/src/task/work.spec.ts index 362d07de3..11c4a5491 100644 --- a/src/task/work.spec.ts +++ b/src/task/work.spec.ts @@ -46,6 +46,17 @@ describe("Work Context", () => { }); }); + describe("runAndStream()", () => { + it("should execute runAndStream command", async () => { + const expectedResult = ActivityMock.createResult({ stdout: "Ok" }); + activity.mockResults([expectedResult]); + const streamOfResults = await context.runAndStream("rm -rf"); + for await (const result of streamOfResults) { + expect(result).toBe(expectedResult); + } + }); + }); + describe("transfer()", () => { it("should execute transfer command", async () => { const result = ActivityMock.createResult({ stdout: "Ok" }); diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index 0d070e86c..9375d0870 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -180,4 +180,23 @@ describe("Task Executor", function () { const result = await executor.run(async (ctx) => ctx.getIp()); expect(["192.168.0.2", "192.168.0.3"]).toContain(result); }); + + it("should run simple task and get results as stream", async () => { + executor = await TaskExecutor.create({ + package: "golem/alpine:latest", + logger, + }); + const streamOfResults = await executor.run(async (ctx) => ctx.runAndStream("echo 'Hello World'")); + expect(streamOfResults).toBeDefined(); + for await (const result of streamOfResults!) { + expect(result.stdout).toContain("Hello World"); + expect(result.result).toContain("Ok"); + } + expect(logger.logs).toContain("Demand published on the market"); + expect(logger.logs).toContain("New proposal has been received"); + expect(logger.logs).toContain("Proposal has been responded"); + expect(logger.logs).toContain("New proposal added to pool"); + expect(logger.logs).toMatch(/Agreement confirmed by provider/); + expect(logger.logs).toMatch(/Activity .* created/); + }); }); From a69fa9bc7d8bdba683e77bd7bffe74531a34dafa Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 30 Oct 2023 13:19:36 +0100 Subject: [PATCH 14/38] test: fixed unit test --- tests/mock/utils/event_source.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/mock/utils/event_source.ts b/tests/mock/utils/event_source.ts index c11dfeb44..fc822d13f 100644 --- a/tests/mock/utils/event_source.ts +++ b/tests/mock/utils/event_source.ts @@ -29,6 +29,9 @@ export class EventSourceMock { }, 100); } } + close() { + // empty mock + } } export const setExpectedEvents = (activityId, expectedEvents) => { From eaa85e680aeff7b43fc132f4ea1b0eb26d255c83 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 30 Oct 2023 13:23:31 +0100 Subject: [PATCH 15/38] test: fixed e2e test --- tests/e2e/tasks.spec.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index 9375d0870..31a11f0d6 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -186,12 +186,13 @@ describe("Task Executor", function () { package: "golem/alpine:latest", logger, }); - const streamOfResults = await executor.run(async (ctx) => ctx.runAndStream("echo 'Hello World'")); - expect(streamOfResults).toBeDefined(); - for await (const result of streamOfResults!) { - expect(result.stdout).toContain("Hello World"); - expect(result.result).toContain("Ok"); - } + await executor.run(async (ctx) => { + const streamOfResults = await ctx.runAndStream("echo 'Hello World'"); + for await (const result of streamOfResults) { + expect(result.stdout).toContain("Hello World"); + expect(result.result).toContain("Ok"); + } + }); expect(logger.logs).toContain("Demand published on the market"); expect(logger.logs).toContain("New proposal has been received"); expect(logger.logs).toContain("Proposal has been responded"); From 0bff2f9a3fc0349097bf4db352b4c654ecd36436 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 31 Oct 2023 08:30:45 +0100 Subject: [PATCH 16/38] test: ci reporter --- .github/workflows/goth-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index ca10cb93d..2727a715f 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -51,7 +51,7 @@ jobs: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run build" - name: Start the e2e test - run: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run test:e2e -- --reporters github-actions --reporters summary" + run: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run test:e2e" - name: Cleanup Docker if: always() From f06e19cac0f134f095553b5b70b32342aaebe160 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 31 Oct 2023 08:52:19 +0100 Subject: [PATCH 17/38] test: fixed e2e --- tests/e2e/tasks.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index 31a11f0d6..dc5efd5bb 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -187,7 +187,11 @@ describe("Task Executor", function () { logger, }); await executor.run(async (ctx) => { - const streamOfResults = await ctx.runAndStream("echo 'Hello World'"); + // for some reason we do not receive events for very simple commands, + // it is probably related to a bug where the command ends and the event does not have time to be triggered or handled + // after creating the EventSource connection to yagna... to investigate. + // for now, sleep 1 has been added, which solves the problem temporarily + const streamOfResults = await ctx.runAndStream("sleep 1 && echo 'Hello World'"); for await (const result of streamOfResults) { expect(result.stdout).toContain("Hello World"); expect(result.result).toContain("Ok"); From beaf2782d06ce54e58c838cd1573314ebab9a4fe Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 31 Oct 2023 10:26:50 +0100 Subject: [PATCH 18/38] test: e2e --- tests/e2e/tasks.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index dc5efd5bb..ee7945bdc 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -190,8 +190,8 @@ describe("Task Executor", function () { // for some reason we do not receive events for very simple commands, // it is probably related to a bug where the command ends and the event does not have time to be triggered or handled // after creating the EventSource connection to yagna... to investigate. - // for now, sleep 1 has been added, which solves the problem temporarily - const streamOfResults = await ctx.runAndStream("sleep 1 && echo 'Hello World'"); + // for now, sleep 2 has been added, which solves the problem temporarily + const streamOfResults = await ctx.runAndStream("sleep 2 && echo 'Hello World'"); for await (const result of streamOfResults) { expect(result.stdout).toContain("Hello World"); expect(result.result).toContain("Ok"); From 7f6dacbc13504b7c355b8349c2add84298bd3c92 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 2 Nov 2023 08:32:55 +0100 Subject: [PATCH 19/38] test: fixed e2e --- tests/e2e/tasks.spec.ts | 15 +++++++-------- tests/mock/rest/yagna.ts | 3 --- tests/unit/activity.test.ts | 3 ++- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index ee7945bdc..4384f76ad 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -1,7 +1,6 @@ import { LoggerMock } from "../mock"; import { readFileSync } from "fs"; -import { TaskExecutor } from "../../src"; -import fs from "fs"; +import { Result, TaskExecutor } from "../../src"; const logger = new LoggerMock(false); describe("Task Executor", function () { @@ -186,17 +185,17 @@ describe("Task Executor", function () { package: "golem/alpine:latest", logger, }); + const results: Result[] = []; await executor.run(async (ctx) => { // for some reason we do not receive events for very simple commands, // it is probably related to a bug where the command ends and the event does not have time to be triggered or handled // after creating the EventSource connection to yagna... to investigate. - // for now, sleep 2 has been added, which solves the problem temporarily - const streamOfResults = await ctx.runAndStream("sleep 2 && echo 'Hello World'"); - for await (const result of streamOfResults) { - expect(result.stdout).toContain("Hello World"); - expect(result.result).toContain("Ok"); - } + // for now, sleep 1 has been added, which solves the problem temporarily + const streamOfResults = await ctx.runAndStream("sleep 1 && echo 'Hello World'"); + for await (const result of streamOfResults) results.push(result); }); + expect(results[0].stdout).toContain("Hello World"); + expect(results[0].result).toContain("Ok"); expect(logger.logs).toContain("Demand published on the market"); expect(logger.logs).toContain("New proposal has been received"); expect(logger.logs).toContain("Proposal has been responded"); diff --git a/tests/mock/rest/yagna.ts b/tests/mock/rest/yagna.ts index 661bdbe0f..4eaeab447 100644 --- a/tests/mock/rest/yagna.ts +++ b/tests/mock/rest/yagna.ts @@ -2,14 +2,11 @@ import { Yagna, YagnaApi } from "../../../src/utils"; import { RequestorControlApiMock, RequestorSateApiMock } from "./activity"; import { MarketApiMock } from "./market"; -import { EventSourceMock } from "../utils/event_source"; import { PaymentApiMock } from "./payment"; import { NetworkApiMock } from "./network"; import { IdentityApiMock } from "./identity"; import { GsbApiMock } from "./gsb"; -jest.mock("eventsource", () => EventSourceMock); - export class YagnaMock extends Yagna { constructor() { super({ apiKey: "test_api_key" }); diff --git a/tests/unit/activity.test.ts b/tests/unit/activity.test.ts index 3551b66c1..6ab74537a 100644 --- a/tests/unit/activity.test.ts +++ b/tests/unit/activity.test.ts @@ -1,10 +1,11 @@ import * as activityMock from "../mock/rest/activity"; -import { setExpectedErrorEvents, setExpectedEvents } from "../mock/utils/event_source"; +import { EventSourceMock, setExpectedErrorEvents, setExpectedEvents } from "../mock/utils/event_source"; import { StorageProviderMock, YagnaMock } from "../mock"; import { Activity, ActivityStateEnum } from "../../src/activity"; import { sleep } from "../../src/utils"; import { Deploy, Start, Run, Terminate, UploadFile, DownloadFile, Script, Capture } from "../../src/script"; +jest.mock("eventsource", () => EventSourceMock); describe("Activity", () => { const yagnaApi = new YagnaMock().getApi(); beforeEach(() => { From bf67242ad1245a31213a0e17a79e75ffa7f1f003 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 6 Nov 2023 12:54:05 +0100 Subject: [PATCH 20/38] chore: refactored runAndStream to spawn and RemoteProcess --- examples/package.json | 2 +- examples/simple-usage/runStream.ts | 8 ----- examples/simple-usage/spawn.ts | 15 +++++++++ src/task/spawn.ts | 53 ++++++++++++++++++++++++++++++ src/task/work.spec.ts | 11 ------- src/task/work.ts | 36 ++++++++++---------- tests/e2e/tasks.spec.ts | 24 +++++++------- 7 files changed, 100 insertions(+), 49 deletions(-) delete mode 100644 examples/simple-usage/runStream.ts create mode 100644 examples/simple-usage/spawn.ts create mode 100644 src/task/spawn.ts diff --git a/examples/package.json b/examples/package.json index 3c0a01122..6e36547ee 100644 --- a/examples/package.json +++ b/examples/package.json @@ -7,7 +7,7 @@ "scripts": { "hello": "node ./hello-world/hello.mjs", "run": "ts-node ./simple-usage/run.ts", - "runStream": "ts-node ./simple-usage/runStream.ts", + "spawn": "ts-node ./simple-usage/spawn.ts", "map": "ts-node ./simple-usage/map.ts", "forEach": "ts-node ./simple-usage/forEach.ts", "fileTransfer": "ts-node ./simple-usage/fileTransfer.ts", diff --git a/examples/simple-usage/runStream.ts b/examples/simple-usage/runStream.ts deleted file mode 100644 index ee8ac3cd3..000000000 --- a/examples/simple-usage/runStream.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { TaskExecutor } from "@golem-sdk/golem-js"; - -const executor = await TaskExecutor.create("golem/alpine:latest"); -const streamOfResults = await executor.run(async (ctx) => ctx.runAndStream("while sleep 1; do date; done")); -streamOfResults?.on("data", (data) => console.log(data.stdout)); -streamOfResults?.on("error", () => executor.end()); - -setTimeout(() => executor.end(), 10_000); diff --git a/examples/simple-usage/spawn.ts b/examples/simple-usage/spawn.ts new file mode 100644 index 000000000..9aafc7329 --- /dev/null +++ b/examples/simple-usage/spawn.ts @@ -0,0 +1,15 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +const executor = await TaskExecutor.create("golem/alpine:latest"); +const finalResult = await executor.run(async (ctx) => { + const remoteProcess = await ctx.spawn("sleep 1 && echo 'Hello World' && echo 'Hello Golem'"); + remoteProcess.stdout.on("data", (data) => console.log("stdout>", data)); + remoteProcess.stderr.on("data", (data) => console.error("stderr>", data)); + + const finalResult = await remoteProcess.waitForExit(); + return finalResult; +}); + +console.log(finalResult); + +await executor.end(); diff --git a/src/task/spawn.ts b/src/task/spawn.ts new file mode 100644 index 000000000..1fd40035c --- /dev/null +++ b/src/task/spawn.ts @@ -0,0 +1,53 @@ +import { Readable, Transform } from "stream"; +import { Result } from "../activity"; + +export class RemoteProcess { + readonly stdout: Readable; + readonly stderr: Readable; + private lastResult?: Result; + private streamError?: Error; + private defaultTimeout = 20_000; + constructor(private streamOfActivityResults: Readable) { + this.streamOfActivityResults.on("data", (data) => (this.lastResult = data)); + this.streamOfActivityResults.on("error", (error) => (this.streamError = error)); + const { stdout, stderr } = this.transformResultsStream(); + this.stdout = stdout; + this.stderr = stderr; + } + + waitForExit(timeout?: number): Promise { + return new Promise((res, rej) => { + const timeoutId = setTimeout( + () => rej(new Error("The waiting time for the final result has been exceeded")), + timeout ?? this.defaultTimeout, + ); + this.streamOfActivityResults.on("close", () => { + clearTimeout(timeoutId); + if (this.lastResult) { + res(this.lastResult); + } else { + rej(new Error(`An error occurred while retrieving the results. ${this.streamError}`)); + } + }); + }); + } + + private transformResultsStream(): { stdout: Readable; stderr: Readable } { + const stdoutTransform = new Transform({ + objectMode: true, + transform(chunk, encoding, callback) { + callback(null, chunk?.stdout ?? null); + }, + }); + const stderrTransform = new Transform({ + objectMode: true, + transform(chunk, encoding, callback) { + callback(null, chunk?.stderr ?? null); + }, + }); + return { + stdout: this.streamOfActivityResults.pipe(stdoutTransform), + stderr: this.streamOfActivityResults.pipe(stderrTransform), + }; + } +} diff --git a/src/task/work.spec.ts b/src/task/work.spec.ts index 11c4a5491..362d07de3 100644 --- a/src/task/work.spec.ts +++ b/src/task/work.spec.ts @@ -46,17 +46,6 @@ describe("Work Context", () => { }); }); - describe("runAndStream()", () => { - it("should execute runAndStream command", async () => { - const expectedResult = ActivityMock.createResult({ stdout: "Ok" }); - activity.mockResults([expectedResult]); - const streamOfResults = await context.runAndStream("rm -rf"); - for await (const result of streamOfResults) { - expect(result).toBe(expectedResult); - } - }); - }); - describe("transfer()", () => { it("should execute transfer command", async () => { const result = ActivityMock.createResult({ stdout: "Ok" }); diff --git a/src/task/work.ts b/src/task/work.ts index a25f6d073..767a1c4b0 100644 --- a/src/task/work.ts +++ b/src/task/work.ts @@ -16,7 +16,7 @@ import { NullStorageProvider, StorageProvider } from "../storage"; import { Logger, sleep } from "../utils"; import { Batch } from "./batch"; import { NetworkNode } from "../network"; -import { Readable } from "stream"; +import { RemoteProcess } from "./spawn"; export type Worker = ( ctx: WorkContext, @@ -137,25 +137,24 @@ export class WorkContext { } /** - * Execute an executable on provider and return Promise of ReadableStream - * that streams Result objects containing the stdout and stderr of the command - * while it is being executed. + * Execute an executable on provider and return RemoteProcess object + * that contain stdout and stderr Readable * * @param commandLine Shell command to execute. * @param options Additional run options. */ - async runAndStream(commandLine: string, options?: Omit): Promise; + async spawn(commandLine: string, options?: Omit): Promise; /** * @param executable Executable to run. * @param args Executable arguments. * @param options Additional run options. */ - async runAndStream(executable: string, args: string[], options?: Omit): Promise; - async runAndStream( + async spawn(executable: string, args: string[], options?: CommandOptions): Promise; + async spawn( exeOrCmd: string, - argsOrOptions?: string[] | Omit, - options?: Omit, - ): Promise { + argsOrOptions?: string[] | CommandOptions, + options?: CommandOptions, + ): Promise { const isArray = Array.isArray(argsOrOptions); const capture: Capture = { stdout: { stream: { format: "string" } }, @@ -167,13 +166,16 @@ export class WorkContext { const script = new Script([run]); // In this case, the script consists only of the run command, // so we skip the execution of script.before and script.after - return this.activity.execute(script.getExeScriptRequest(), true, options?.timeout).catch((e) => { - throw new Error( - `Script execution failed for command: ${JSON.stringify(run.toJson())}. ${ - e?.response?.data?.message || e?.message || e - }`, - ); - }); + const streamOfActivityResults = await this.activity + .execute(script.getExeScriptRequest(), true, options?.timeout) + .catch((e) => { + throw new Error( + `Script execution failed for command: ${JSON.stringify(run.toJson())}. ${ + e?.response?.data?.message || e?.message || e + }`, + ); + }); + return new RemoteProcess(streamOfActivityResults); } /** diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index 4384f76ad..f8d94d64a 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -180,22 +180,22 @@ describe("Task Executor", function () { expect(["192.168.0.2", "192.168.0.3"]).toContain(result); }); - it("should run simple task and get results as stream", async () => { + it("should run simple by spawn command as external process", async () => { executor = await TaskExecutor.create({ package: "golem/alpine:latest", logger, }); - const results: Result[] = []; - await executor.run(async (ctx) => { - // for some reason we do not receive events for very simple commands, - // it is probably related to a bug where the command ends and the event does not have time to be triggered or handled - // after creating the EventSource connection to yagna... to investigate. - // for now, sleep 1 has been added, which solves the problem temporarily - const streamOfResults = await ctx.runAndStream("sleep 1 && echo 'Hello World'"); - for await (const result of streamOfResults) results.push(result); - }); - expect(results[0].stdout).toContain("Hello World"); - expect(results[0].result).toContain("Ok"); + let stdout = ""; + let stderr = ""; + const finalResult = await executor.run(async (ctx) => { + const remoteProcess = await ctx.spawn("sleep 1 && echo 'Hello World' && echo 'Hello Golem' >&2"); + remoteProcess.stdout.on("data", (data) => (stdout += data.trim())); + remoteProcess.stderr.on("data", (data) => (stderr += data.trim())); + return remoteProcess.waitForExit(); + }); + expect(stdout).toContain("Hello World"); + expect(stderr).toContain("Hello World"); + expect(finalResult?.result).toContain("Ok"); expect(logger.logs).toContain("Demand published on the market"); expect(logger.logs).toContain("New proposal has been received"); expect(logger.logs).toContain("Proposal has been responded"); From 89a5cb196ef057b637f70f4805907c0cd3b5a6b8 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 6 Nov 2023 15:30:05 +0100 Subject: [PATCH 21/38] chore: fixed closing stream --- examples/simple-usage/spawn.ts | 2 +- src/task/spawn.ts | 14 +++++++++----- src/task/work.spec.ts | 16 ++++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/examples/simple-usage/spawn.ts b/examples/simple-usage/spawn.ts index 9aafc7329..145439fbf 100644 --- a/examples/simple-usage/spawn.ts +++ b/examples/simple-usage/spawn.ts @@ -2,7 +2,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create("golem/alpine:latest"); const finalResult = await executor.run(async (ctx) => { - const remoteProcess = await ctx.spawn("sleep 1 && echo 'Hello World' && echo 'Hello Golem'"); + const remoteProcess = await ctx.spawn("sleep 1 && echo 'Hello World' && echo 'Hello Golem' >&2"); remoteProcess.stdout.on("data", (data) => console.log("stdout>", data)); remoteProcess.stderr.on("data", (data) => console.error("stderr>", data)); diff --git a/src/task/spawn.ts b/src/task/spawn.ts index 1fd40035c..3c8177a93 100644 --- a/src/task/spawn.ts +++ b/src/task/spawn.ts @@ -8,7 +8,9 @@ export class RemoteProcess { private streamError?: Error; private defaultTimeout = 20_000; constructor(private streamOfActivityResults: Readable) { - this.streamOfActivityResults.on("data", (data) => (this.lastResult = data)); + this.streamOfActivityResults.on("data", (data) => { + this.lastResult = data; + }); this.streamOfActivityResults.on("error", (error) => (this.streamError = error)); const { stdout, stderr } = this.transformResultsStream(); this.stdout = stdout; @@ -21,14 +23,16 @@ export class RemoteProcess { () => rej(new Error("The waiting time for the final result has been exceeded")), timeout ?? this.defaultTimeout, ); - this.streamOfActivityResults.on("close", () => { + const end = () => { clearTimeout(timeoutId); if (this.lastResult) { res(this.lastResult); } else { rej(new Error(`An error occurred while retrieving the results. ${this.streamError}`)); } - }); + }; + if (this.streamOfActivityResults.closed) return end(); + this.streamOfActivityResults.on("close", () => end()); }); } @@ -36,13 +40,13 @@ export class RemoteProcess { const stdoutTransform = new Transform({ objectMode: true, transform(chunk, encoding, callback) { - callback(null, chunk?.stdout ?? null); + callback(null, chunk?.stdout); }, }); const stderrTransform = new Transform({ objectMode: true, transform(chunk, encoding, callback) { - callback(null, chunk?.stderr ?? null); + callback(null, chunk?.stderr); }, }); return { diff --git a/src/task/work.spec.ts b/src/task/work.spec.ts index 362d07de3..2d361740f 100644 --- a/src/task/work.spec.ts +++ b/src/task/work.spec.ts @@ -46,6 +46,22 @@ describe("Work Context", () => { }); }); + describe("spawn()", () => { + it("should execute spawn command", async () => { + const expectedResult = ActivityMock.createResult({ stdout: "Ok", stderr: "Error", isBatchFinished: true }); + activity.mockResults([expectedResult]); + const remoteProcess = await context.spawn("rm -rf"); + for await (const result of remoteProcess.stdout) { + expect(result).toBe("Ok"); + } + for await (const result of remoteProcess.stderr) { + expect(result).toBe("Error"); + } + const finalResult = await remoteProcess.waitForExit(); + expect(finalResult.result).toBe("Ok"); + }); + }); + describe("transfer()", () => { it("should execute transfer command", async () => { const result = ActivityMock.createResult({ stdout: "Ok" }); From 443050344e94cfa58273f335e1738bcb8e7e3a00 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 7 Nov 2023 07:50:02 +0100 Subject: [PATCH 22/38] chore: fixed e2e stream --- tests/e2e/tasks.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index f8d94d64a..e9abacb8f 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -194,7 +194,7 @@ describe("Task Executor", function () { return remoteProcess.waitForExit(); }); expect(stdout).toContain("Hello World"); - expect(stderr).toContain("Hello World"); + expect(stderr).toContain("Hello Golem"); expect(finalResult?.result).toContain("Ok"); expect(logger.logs).toContain("Demand published on the market"); expect(logger.logs).toContain("New proposal has been received"); From 542057d9d5656fe4bea86440b46fece01ee5bf3a Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 7 Nov 2023 08:49:24 +0100 Subject: [PATCH 23/38] feat(work_context): added spawn method returning a RemoteProcess instance --- src/activity/activity.ts | 2 ++ src/task/process.spec.ts | 53 +++++++++++++++++++++++++++++++ src/task/{spawn.ts => process.ts} | 48 ++++++++++++++++------------ src/task/work.spec.ts | 6 ++-- src/task/work.ts | 4 +-- 5 files changed, 88 insertions(+), 25 deletions(-) create mode 100644 src/task/process.spec.ts rename src/task/{spawn.ts => process.ts} (52%) diff --git a/src/activity/activity.ts b/src/activity/activity.ts index df4fb45a0..1e45234d1 100644 --- a/src/activity/activity.ts +++ b/src/activity/activity.ts @@ -321,6 +321,8 @@ export class Activity { private parseEventToResult(msg: string, batchSize: number): Result { try { const event: StreamingBatchEvent = JSON.parse(msg); + // StreamingBatchEvent has a slightly more extensive structure, + // including a return code that could be added to the Result entity... (?) return new Result({ index: event.index, eventDate: event.timestamp, diff --git a/src/task/process.spec.ts b/src/task/process.spec.ts new file mode 100644 index 000000000..bfec38453 --- /dev/null +++ b/src/task/process.spec.ts @@ -0,0 +1,53 @@ +import { RemoteProcess } from "./process"; +import { ActivityMock } from "../../tests/mock/activity.mock"; +import { YagnaMock } from "../../tests/mock"; +import { Run, Script } from "../script"; +import { ResultState } from "../activity"; + +describe("RemoteProcess", () => { + let activity: ActivityMock; + beforeEach(() => { + activity = new ActivityMock("test_id", "test_id", new YagnaMock().getApi()); + }); + + it("should create remote process", async () => { + const expectedResult = ActivityMock.createResult({ stdout: "Ok" }); + activity.mockResults([expectedResult]); + const exeScriptRequest = new Script([new Run("test_command")]).getExeScriptRequest(); + const streamOfActivityResults = await activity.execute(exeScriptRequest, true); + const remoteProcess = new RemoteProcess(streamOfActivityResults); + expect(remoteProcess).toBeDefined(); + }); + + it("should read stdout from remote process", async () => { + const expectedResult = ActivityMock.createResult({ stdout: "Output" }); + activity.mockResults([expectedResult]); + const exeScriptRequest = new Script([new Run("test_command")]).getExeScriptRequest(); + const streamOfActivityResults = await activity.execute(exeScriptRequest, true); + const remoteProcess = new RemoteProcess(streamOfActivityResults); + for await (const stdout of remoteProcess.stdout) { + expect(stdout).toEqual("Output"); + } + }); + + it("should read stderr from remote process", async () => { + const expectedResult = ActivityMock.createResult({ stderr: "Error" }); + activity.mockResults([expectedResult]); + const exeScriptRequest = new Script([new Run("test_command")]).getExeScriptRequest(); + const streamOfActivityResults = await activity.execute(exeScriptRequest, true); + const remoteProcess = new RemoteProcess(streamOfActivityResults); + for await (const stderr of remoteProcess.stderr) { + expect(stderr).toEqual("Error"); + } + }); + + it("should wait for exit", async () => { + const expectedResult = ActivityMock.createResult({ stdout: "Output", stderr: "Error" }); + activity.mockResults([expectedResult]); + const exeScriptRequest = new Script([new Run("test_command")]).getExeScriptRequest(); + const streamOfActivityResults = await activity.execute(exeScriptRequest, true); + const remoteProcess = new RemoteProcess(streamOfActivityResults); + const finalResult = await remoteProcess.waitForExit(); + expect(finalResult.result).toEqual(ResultState.Ok); + }); +}); diff --git a/src/task/spawn.ts b/src/task/process.ts similarity index 52% rename from src/task/spawn.ts rename to src/task/process.ts index 3c8177a93..b29603293 100644 --- a/src/task/spawn.ts +++ b/src/task/process.ts @@ -1,27 +1,40 @@ import { Readable, Transform } from "stream"; import { Result } from "../activity"; +const DEFAULTS = { + exitWaitingTimeout: 20_000, +}; +/** + * RemoteProcess class representing the process spawned on the provider by {@link WorkContext.spawn} + */ export class RemoteProcess { + /** + * Returns a stream connected to stdout from provider process + */ readonly stdout: Readable; + /** + * Returns a stream connected to stderr from provider process + */ readonly stderr: Readable; private lastResult?: Result; private streamError?: Error; - private defaultTimeout = 20_000; constructor(private streamOfActivityResults: Readable) { - this.streamOfActivityResults.on("data", (data) => { - this.lastResult = data; - }); + this.streamOfActivityResults.on("data", (data) => (this.lastResult = data)); this.streamOfActivityResults.on("error", (error) => (this.streamError = error)); const { stdout, stderr } = this.transformResultsStream(); this.stdout = stdout; this.stderr = stderr; } + /** + * Waits for the process to complete and returns the last part of the command's results as a {@link Result} object + * @param timeout - maximum waiting time for the final result (default: 20 sec) + */ waitForExit(timeout?: number): Promise { return new Promise((res, rej) => { const timeoutId = setTimeout( () => rej(new Error("The waiting time for the final result has been exceeded")), - timeout ?? this.defaultTimeout, + timeout ?? DEFAULTS.exitWaitingTimeout, ); const end = () => { clearTimeout(timeoutId); @@ -32,26 +45,21 @@ export class RemoteProcess { } }; if (this.streamOfActivityResults.closed) return end(); - this.streamOfActivityResults.on("close", () => end()); + this.streamOfActivityResults.on("close", end); }); } private transformResultsStream(): { stdout: Readable; stderr: Readable } { - const stdoutTransform = new Transform({ - objectMode: true, - transform(chunk, encoding, callback) { - callback(null, chunk?.stdout); - }, - }); - const stderrTransform = new Transform({ - objectMode: true, - transform(chunk, encoding, callback) { - callback(null, chunk?.stderr); - }, - }); + const transform = (std: string) => + new Transform({ + objectMode: true, + transform(chunk, encoding, callback) { + callback(null, chunk?.[std]); + }, + }); return { - stdout: this.streamOfActivityResults.pipe(stdoutTransform), - stderr: this.streamOfActivityResults.pipe(stderrTransform), + stdout: this.streamOfActivityResults.pipe(transform("stdout")), + stderr: this.streamOfActivityResults.pipe(transform("stderr")), }; } } diff --git a/src/task/work.spec.ts b/src/task/work.spec.ts index 2d361740f..8b7642cee 100644 --- a/src/task/work.spec.ts +++ b/src/task/work.spec.ts @@ -48,17 +48,17 @@ describe("Work Context", () => { describe("spawn()", () => { it("should execute spawn command", async () => { - const expectedResult = ActivityMock.createResult({ stdout: "Ok", stderr: "Error", isBatchFinished: true }); + const expectedResult = ActivityMock.createResult({ stdout: "Output", stderr: "Error", isBatchFinished: true }); activity.mockResults([expectedResult]); const remoteProcess = await context.spawn("rm -rf"); for await (const result of remoteProcess.stdout) { - expect(result).toBe("Ok"); + expect(result).toBe("Output"); } for await (const result of remoteProcess.stderr) { expect(result).toBe("Error"); } const finalResult = await remoteProcess.waitForExit(); - expect(finalResult.result).toBe("Ok"); + expect(finalResult.result).toBe(ResultState.Ok); }); }); diff --git a/src/task/work.ts b/src/task/work.ts index 767a1c4b0..d9a177fbe 100644 --- a/src/task/work.ts +++ b/src/task/work.ts @@ -16,7 +16,7 @@ import { NullStorageProvider, StorageProvider } from "../storage"; import { Logger, sleep } from "../utils"; import { Batch } from "./batch"; import { NetworkNode } from "../network"; -import { RemoteProcess } from "./spawn"; +import { RemoteProcess } from "./process"; export type Worker = ( ctx: WorkContext, @@ -138,7 +138,7 @@ export class WorkContext { /** * Execute an executable on provider and return RemoteProcess object - * that contain stdout and stderr Readable + * that contain stdout and stderr as Readable * * @param commandLine Shell command to execute. * @param options Additional run options. From 6f66ba82890ff48bb648dfa4a879f6ccbbd8b3e5 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 7 Nov 2023 08:56:24 +0100 Subject: [PATCH 24/38] chore: correcting typos --- src/task/process.ts | 4 ++-- src/task/work.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/task/process.ts b/src/task/process.ts index b29603293..0ff413847 100644 --- a/src/task/process.ts +++ b/src/task/process.ts @@ -9,11 +9,11 @@ const DEFAULTS = { */ export class RemoteProcess { /** - * Returns a stream connected to stdout from provider process + * Stream connected to stdout from provider process */ readonly stdout: Readable; /** - * Returns a stream connected to stderr from provider process + * Stream connected to stderr from provider process */ readonly stderr: Readable; private lastResult?: Result; diff --git a/src/task/work.ts b/src/task/work.ts index d9a177fbe..a02ca6b02 100644 --- a/src/task/work.ts +++ b/src/task/work.ts @@ -137,7 +137,7 @@ export class WorkContext { } /** - * Execute an executable on provider and return RemoteProcess object + * Spawn an executable on provider and return {@link RemoteProcess} object * that contain stdout and stderr as Readable * * @param commandLine Shell command to execute. @@ -164,7 +164,7 @@ export class WorkContext { ? new Run(exeOrCmd, argsOrOptions as string[], options?.env, capture) : new Run("/bin/sh", ["-c", exeOrCmd], argsOrOptions?.env, capture); const script = new Script([run]); - // In this case, the script consists only of the run command, + // In this case, the script consists only of one run command, // so we skip the execution of script.before and script.after const streamOfActivityResults = await this.activity .execute(script.getExeScriptRequest(), true, options?.timeout) From a3b94ca8256a951bac04f040783743362fe16583 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 9 Nov 2023 09:05:40 +0100 Subject: [PATCH 25/38] ci: update dependabot.yml --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3a3cce576..77c0a3c8c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,5 +7,6 @@ version: 2 updates: - package-ecosystem: "npm" # See documentation for possible values directory: "/" # Location of package manifests + target-branch: "beta" schedule: interval: "weekly" From 8d4dddf226735d5eee7209e3639c89218c4c309c Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 9 Nov 2023 15:43:12 +0100 Subject: [PATCH 26/38] chore: fixes in comments and names --- src/task/process.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/task/process.ts b/src/task/process.ts index 0ff413847..e2766b98c 100644 --- a/src/task/process.ts +++ b/src/task/process.ts @@ -27,21 +27,23 @@ export class RemoteProcess { } /** - * Waits for the process to complete and returns the last part of the command's results as a {@link Result} object - * @param timeout - maximum waiting time for the final result (default: 20 sec) + * Waits for the process to complete and returns the last part of the command's results as a {@link Result} object. + * If the timeout is reached, the return promise will be rejected. + * @param timeout - maximum waiting time im ms for the final result (default: 20_000) */ waitForExit(timeout?: number): Promise { - return new Promise((res, rej) => { + return new Promise((resolve, reject) => { + const timeoutInMs = timeout ?? DEFAULTS.exitWaitingTimeout; const timeoutId = setTimeout( - () => rej(new Error("The waiting time for the final result has been exceeded")), - timeout ?? DEFAULTS.exitWaitingTimeout, + () => reject(new Error(`The waiting time (${timeoutInMs} ms) for the final result has been exceeded`)), + timeoutInMs, ); const end = () => { clearTimeout(timeoutId); if (this.lastResult) { - res(this.lastResult); + resolve(this.lastResult); } else { - rej(new Error(`An error occurred while retrieving the results. ${this.streamError}`)); + reject(new Error(`An error occurred while retrieving the results. ${this.streamError}`)); } }; if (this.streamOfActivityResults.closed) return end(); From 90bd9b14e9a887dcf7c17406aa0c9ad264035cbe Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 9 Nov 2023 15:44:39 +0100 Subject: [PATCH 27/38] chore: fixed test name --- tests/e2e/tasks.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index e9abacb8f..07f7074ef 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -180,7 +180,7 @@ describe("Task Executor", function () { expect(["192.168.0.2", "192.168.0.3"]).toContain(result); }); - it("should run simple by spawn command as external process", async () => { + it("should spawn command as external process", async () => { executor = await TaskExecutor.create({ package: "golem/alpine:latest", logger, From ed16a8135821ffd18966ed052d177f576915bd3b Mon Sep 17 00:00:00 2001 From: Seweryn Kras Date: Fri, 10 Nov 2023 13:26:31 +0100 Subject: [PATCH 28/38] fix(executor): deprecate `map` and `forEach` --- src/executor/executor.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/executor/executor.ts b/src/executor/executor.ts index bb162c576..fbdaffd1c 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -339,6 +339,18 @@ export class TaskExecutor { } /** + * @deprecated This method is marked for removal in a future release. Migrate your code by using `Array.map` and `Promise.all` instead. + * @example + * ```typescript + * const data = [1, 2, 3, 4, 5]; + * const resultsPromises = data.map((item) => + * executor.run((ctx) => { + * console.log((await ctx.run(`echo "${item}"`)).stdout); + * }) + * ); + * await Promise.all(resultsPromises); + * ``` + * * Map iterable data to worker function and return computed Task result as AsyncIterable * * @param data Iterable data @@ -387,6 +399,19 @@ export class TaskExecutor { } /** + * @deprecated This method is marked for removal in a future release. + * Migrate your code by using `Array.map` and `Promise.all` instead. + * @example + * ```typescript + * const data = [1, 2, 3, 4, 5]; + * const resultsPromises = data.map((item) => + * executor.run((ctx) => { + * console.log((await ctx.run(`echo "${item}"`)).stdout); + * }), + * ); + * await Promise.all(resultsPromises); + * ``` + * * Iterates over given data and execute task using worker function * * @param data Iterable data From a8d1fc84edac8c797217c08393de54cc5fefee9a Mon Sep 17 00:00:00 2001 From: Seweryn Kras Date: Mon, 13 Nov 2023 19:51:28 +0100 Subject: [PATCH 29/38] chore: remove executor map and foreach from all examples and tests --- examples/blender/blender.ts | 26 +++++---- .../examples/executing-tasks/action_log.txt | 2 +- .../examples/executing-tasks/before-each.mjs | 17 +++--- .../examples/executing-tasks/foreach.mjs | 8 +-- .../examples/executing-tasks/map.mjs | 9 +++- .../executing-tasks/max-parallel-tasks.mjs | 9 +++- .../running-parallel-tasks/index.mjs | 34 ++++++------ examples/fibonacci/fibonacci.js | 15 ++++-- examples/simple-usage/forEach.ts | 11 ---- examples/simple-usage/map.ts | 19 +++---- examples/ssh/ssh.ts | 51 ++++++++++-------- examples/yacat/yacat.ts | 29 +++++----- src/executor/executor.ts | 8 +-- tests/e2e/blender.spec.ts | 25 +++++---- tests/e2e/gftp.spec.ts | 23 ++++---- tests/e2e/strategies.spec.ts | 54 ++++++++++--------- tests/e2e/tasks.spec.ts | 13 ++--- tests/e2e/yacat.spec.ts | 25 +++++---- 18 files changed, 212 insertions(+), 166 deletions(-) delete mode 100644 examples/simple-usage/forEach.ts diff --git a/examples/blender/blender.ts b/examples/blender/blender.ts index 1c8dc7710..0e2c5f827 100644 --- a/examples/blender/blender.ts +++ b/examples/blender/blender.ts @@ -35,17 +35,21 @@ async function main(subnetTag: string, driver?: string, network?: string, debug? await ctx.uploadFile(`${__dirname}/cubes.blend`, "/golem/resource/scene.blend"); }); - const results = executor.map([0, 10, 20, 30, 40, 50], async (ctx, frame) => { - const result = await ctx - .beginBatch() - .uploadJson(blender_params(frame), "/golem/work/params.json") - .run("/golem/entrypoints/run-blender.sh") - .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`) - .end() - .catch((e) => console.error(e)); - return result?.length ? `output_${frame}.png` : ""; - }); - for await (const result of results) console.log(result); + const futureResults = [0, 10, 20, 30, 40, 50].map((frame) => + executor.run(async (ctx) => { + const result = await ctx + .beginBatch() + .uploadJson(blender_params(frame), "/golem/work/params.json") + .run("/golem/entrypoints/run-blender.sh") + .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`) + .end() + .catch((e) => console.error(e)); + return result?.length ? `output_${frame}.png` : ""; + }), + ); + const results = await Promise.all(futureResults); + results.forEach((result) => console.log(result)); + await executor.end(); } diff --git a/examples/docs-examples/examples/executing-tasks/action_log.txt b/examples/docs-examples/examples/executing-tasks/action_log.txt index bb5749a1f..170536dc7 100644 --- a/examples/docs-examples/examples/executing-tasks/action_log.txt +++ b/examples/docs-examples/examples/executing-tasks/action_log.txt @@ -1 +1 @@ -some action log \ No newline at end of file +some action log diff --git a/examples/docs-examples/examples/executing-tasks/before-each.mjs b/examples/docs-examples/examples/executing-tasks/before-each.mjs index 31913ed4c..fa349bd8d 100644 --- a/examples/docs-examples/examples/executing-tasks/before-each.mjs +++ b/examples/docs-examples/examples/executing-tasks/before-each.mjs @@ -12,13 +12,18 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; await ctx.uploadFile("./action_log.txt", "/golem/input/action_log.txt"); }); - await executor.forEach([1, 2, 3, 4, 5], async (ctx, item) => { - await ctx - .beginBatch() - .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`) - .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt") - .end(); + const inputs = [1, 2, 3, 4, 5]; + + const futureResults = inputs.map(async (item) => { + return await executor.run(async (ctx) => { + await ctx + .beginBatch() + .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`) + .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt") + .end(); + }); }); + await Promise.all(futureResults); await executor.end(); })(); diff --git a/examples/docs-examples/examples/executing-tasks/foreach.mjs b/examples/docs-examples/examples/executing-tasks/foreach.mjs index e1cbd7fc1..521f62ac1 100644 --- a/examples/docs-examples/examples/executing-tasks/foreach.mjs +++ b/examples/docs-examples/examples/executing-tasks/foreach.mjs @@ -8,9 +8,11 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const data = [1, 2, 3, 4, 5]; - await executor.forEach(data, async (ctx, item) => { - console.log((await ctx.run(`echo "${item}"`)).stdout); - }); + for (const item of data) { + await executor.run(async (ctx) => { + console.log((await ctx.run(`echo "${item}"`)).stdout); + }); + } await executor.end(); })(); diff --git a/examples/docs-examples/examples/executing-tasks/map.mjs b/examples/docs-examples/examples/executing-tasks/map.mjs index 9fc089987..8ce0f91ee 100644 --- a/examples/docs-examples/examples/executing-tasks/map.mjs +++ b/examples/docs-examples/examples/executing-tasks/map.mjs @@ -8,9 +8,14 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const data = [1, 2, 3, 4, 5]; - const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); + const futureResults = data.map(async (item) => + executor.run(async (ctx) => { + return await ctx.run(`echo "${item}"`); + }), + ); - for await (const result of results) console.log(result.stdout); + const results = await Promise.all(futureResults); + results.forEach((result) => console.log(result.stdout)); await executor.end(); })(); diff --git a/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs b/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs index 0e1a11572..bc19a8bcd 100644 --- a/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs +++ b/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs @@ -9,9 +9,14 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const data = [1, 2, 3, 4, 5]; - const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); + const futureResults = data.map(async (item) => + executor.run(async (ctx) => { + return await ctx.run(`echo "${item}"`); + }), + ); - for await (const result of results) console.log(result.stdout); + const results = await Promise.all(futureResults); + results.forEach((result) => console.log(result.stdout)); await executor.end(); })(); diff --git a/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs index 6342bf373..71d20b2a9 100644 --- a/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs +++ b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs @@ -19,24 +19,28 @@ async function main(args) { const step = Math.floor(keyspace / args.numberOfProviders + 1); const range = [...Array(Math.floor(keyspace / step) + 1).keys()].map((i) => i * step); - const results = executor.map(range, async (ctx, skip = 0) => { - const results = await ctx - .beginBatch() - .run( - `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min( - keyspace, - skip + step, - )} -o pass.potfile`, - ) - .run("cat pass.potfile") - .end() - .catch((err) => console.error(err)); - if (!results?.[1]?.stdout) return false; - return results?.[1]?.stdout.toString().split(":")[1]; + const futureResults = range.map(async (skip = 0) => { + return executor.run(async (ctx) => { + const results = await ctx + .beginBatch() + .run( + `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min( + keyspace, + skip + step, + )} -o pass.potfile`, + ) + .run("cat pass.potfile") + .end() + .catch((err) => console.error(err)); + if (!results?.[1]?.stdout) return false; + return results?.[1]?.stdout.toString().split(":")[1]; + }); }); + const results = await Promise.all(futureResults); + let password = ""; - for await (const result of results) { + for (const result of results) { if (result) { password = result; break; diff --git a/examples/fibonacci/fibonacci.js b/examples/fibonacci/fibonacci.js index 79e6e9ce2..78430d651 100644 --- a/examples/fibonacci/fibonacci.js +++ b/examples/fibonacci/fibonacci.js @@ -9,12 +9,17 @@ async function main(fiboN = 1, tasksCount = 1, subnetTag, driver, network, debug logLevel: debug ? "debug" : "info", }); - const data = Array(tasksCount).fill(null); + const runningTasks = []; + for (let i = 0; i < tasksCount; i++) { + runningTasks.push( + executor.run(async (ctx) => { + const result = await ctx.run("/usr/local/bin/node", ["/golem/work/fibo.js", fiboN.toString()]); + console.log(result.stdout); + }), + ); + } - await executor.forEach(data, async (ctx) => { - const result = await ctx.run("/usr/local/bin/node", ["/golem/work/fibo.js", fiboN.toString()]); - console.log(result.stdout); - }); + await Promise.all(runningTasks); await executor.end(); } program diff --git a/examples/simple-usage/forEach.ts b/examples/simple-usage/forEach.ts deleted file mode 100644 index 9ef76ec6a..000000000 --- a/examples/simple-usage/forEach.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { TaskExecutor } from "@golem-sdk/golem-js"; - -(async function main() { - const executor = await TaskExecutor.create("golem/alpine:latest"); - const data = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]; - await executor.forEach(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - console.log(`Result=${res.stdout}`); - }); - await executor.end(); -})(); diff --git a/examples/simple-usage/map.ts b/examples/simple-usage/map.ts index be464a905..20b4652ea 100644 --- a/examples/simple-usage/map.ts +++ b/examples/simple-usage/map.ts @@ -4,14 +4,15 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create("golem/alpine:latest"); const data = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]; - const results = executor.map(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - return res.stdout?.toString().trim(); - }); - const finalOutput: string[] = []; - for await (const res of results) { - if (res) finalOutput.push(res); - } - console.log("RESULTS = ", finalOutput.join(", ")); + const futureResults = data.map((x) => + executor.run(async (ctx) => { + const res = await ctx.run(`echo "${x}"`); + return res.stdout?.toString().trim(); + }), + ); + + const results = await Promise.all(futureResults); + console.log(results); + await executor.end(); })(); diff --git a/examples/ssh/ssh.ts b/examples/ssh/ssh.ts index a2df5d088..6cf7253a1 100644 --- a/examples/ssh/ssh.ts +++ b/examples/ssh/ssh.ts @@ -12,31 +12,36 @@ async function main(subnetTag, driver, network, count = 2, sessionTimeout = 100, payment: { driver, network }, logLevel: debug ? "debug" : "info", }); - const data = new Array(count).fill(null); const appKey = process.env["YAGNA_APPKEY"]; - await executor.forEach(data, async (ctx) => { - const password = crypto.randomBytes(3).toString("hex"); - const results = await ctx - .beginBatch() - .run("syslogd") - .run("ssh-keygen -A") - .run(`echo -e "${password}\n${password}" | passwd`) - .run("/usr/sbin/sshd") - .end() - .catch((e) => console.error(e)); - if (!results) return; - console.log("\n------------------------------------------"); - console.log(`Connect via ssh to provider "${ctx.provider?.name}" with:`); - console.log( - `ssh -o ProxyCommand='websocat asyncstdio: ${ctx.getWebsocketUri( - 22, - )} --binary -H=Authorization:"Bearer ${appKey}"' root@${crypto.randomBytes(10).toString("hex")}`, + const runningTasks: Promise[] = []; + for (let i = 0; i < count; i++) { + runningTasks.push( + executor.run(async (ctx) => { + const password = crypto.randomBytes(3).toString("hex"); + const results = await ctx + .beginBatch() + .run("syslogd") + .run("ssh-keygen -A") + .run(`echo -e "${password}\n${password}" | passwd`) + .run("/usr/sbin/sshd") + .end() + .catch((e) => console.error(e)); + if (!results) return; + console.log("\n------------------------------------------"); + console.log(`Connect via ssh to provider "${ctx.provider?.name}" with:`); + console.log( + `ssh -o ProxyCommand='websocat asyncstdio: ${ctx.getWebsocketUri( + 22, + )} --binary -H=Authorization:"Bearer ${appKey}"' root@${crypto.randomBytes(10).toString("hex")}`, + ); + console.log(`Password: ${password}`); + console.log("------------------------------------------\n"); + await new Promise((res) => setTimeout(res, sessionTimeout * 1000)); + console.log(`Task completed. Session SSH closed after ${sessionTimeout} secs timeout.`); + }), ); - console.log(`Password: ${password}`); - console.log("------------------------------------------\n"); - await new Promise((res) => setTimeout(res, sessionTimeout * 1000)); - console.log(`Task completed. Session SSH closed after ${sessionTimeout} secs timeout.`); - }); + } + await Promise.all(runningTasks); await executor.end(); } diff --git a/examples/yacat/yacat.ts b/examples/yacat/yacat.ts index eec9dd8c0..139fcf50a 100644 --- a/examples/yacat/yacat.ts +++ b/examples/yacat/yacat.ts @@ -23,19 +23,22 @@ async function main(args) { const range = [...Array(Math.floor(keyspace / step)).keys()].map((i) => i * step); console.log(`Keyspace size computed. Keyspace size = ${keyspace}. Tasks to compute = ${range.length}`); - const results = executor.map(range, async (ctx, skip) => { - const results = await ctx - .beginBatch() - .run( - `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${ - skip! + step - } -o pass.potfile || true`, - ) - .run("cat pass.potfile || true") - .end(); - if (!results?.[1]?.stdout) return false; - return results?.[1]?.stdout.toString().trim().split(":")[1]; - }); + const futureResults = range.map((skip) => + executor.run(async (ctx) => { + const results = await ctx + .beginBatch() + .run( + `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${ + skip! + step + } -o pass.potfile || true`, + ) + .run("cat pass.potfile || true") + .end(); + if (!results?.[1]?.stdout) return false; + return results?.[1]?.stdout.toString().trim().split(":")[1]; + }), + ); + const results = await Promise.all(futureResults); let password = ""; for await (const result of results) { diff --git a/src/executor/executor.ts b/src/executor/executor.ts index fbdaffd1c..285b17533 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -343,12 +343,12 @@ export class TaskExecutor { * @example * ```typescript * const data = [1, 2, 3, 4, 5]; - * const resultsPromises = data.map((item) => + * const futureResults = data.map((item) => * executor.run((ctx) => { * console.log((await ctx.run(`echo "${item}"`)).stdout); * }) * ); - * await Promise.all(resultsPromises); + * const results = await Promise.all(futureResults); * ``` * * Map iterable data to worker function and return computed Task result as AsyncIterable @@ -404,12 +404,12 @@ export class TaskExecutor { * @example * ```typescript * const data = [1, 2, 3, 4, 5]; - * const resultsPromises = data.map((item) => + * const futureResults = data.map((item) => * executor.run((ctx) => { * console.log((await ctx.run(`echo "${item}"`)).stdout); * }), * ); - * await Promise.all(resultsPromises); + * await Promise.all(futureResults); * ``` * * Iterates over given data and execute task using worker function diff --git a/tests/e2e/blender.spec.ts b/tests/e2e/blender.spec.ts index 2f29a54db..5fce268be 100644 --- a/tests/e2e/blender.spec.ts +++ b/tests/e2e/blender.spec.ts @@ -38,20 +38,23 @@ describe("Blender rendering", function () { const data = [0, 10, 20, 30, 40, 50]; - const results = executor.map(data, async (ctx, frame) => { - const result = await ctx - .beginBatch() - .uploadJson(blenderParams(frame), "/golem/work/params.json") - .run("/golem/entrypoints/run-blender.sh") - .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `output_${frame}.png`) - .end() - .catch((error) => console.error(error.toString())); - return result ? `output_${frame}.png` : ""; - }); + const futureResults = data.map((frame) => + executor.run(async (ctx) => { + const result = await ctx + .beginBatch() + .uploadJson(blenderParams(frame), "/golem/work/params.json") + .run("/golem/entrypoints/run-blender.sh") + .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `output_${frame}.png`) + .end() + .catch((error) => console.error(error.toString())); + return result ? `output_${frame}.png` : ""; + }), + ); + const results = await Promise.all(futureResults); const expectedResults = data.map((d) => `output_${d}.png`); - for await (const result of results) { + for (const result of results) { expect(expectedResults).toContain(result); } diff --git a/tests/e2e/gftp.spec.ts b/tests/e2e/gftp.spec.ts index d57fafd9c..a38f11a52 100644 --- a/tests/e2e/gftp.spec.ts +++ b/tests/e2e/gftp.spec.ts @@ -20,19 +20,22 @@ describe("GFTP transfers", function () { const data = [0, 1, 2, 3, 4, 5]; - const results = executor.map(data, async (ctx, frame) => { - const result = await ctx - .beginBatch() - .run("ls -Alh /golem/work/eiffel.blend") - .downloadFile(`/golem/work/eiffel.blend`, `copy_${frame}.blend`) - .end() - .catch((error) => console.error(error.toString())); - return result ? `copy_${frame}.blend` : ""; - }); + const futureResults = data.map((frame) => + executor.run(async (ctx) => { + const result = await ctx + .beginBatch() + .run("ls -Alh /golem/work/eiffel.blend") + .downloadFile(`/golem/work/eiffel.blend`, `copy_${frame}.blend`) + .end() + .catch((error) => console.error(error.toString())); + return result ? `copy_${frame}.blend` : ""; + }), + ); + const results = await Promise.all(futureResults); const expectedResults = data.map((d) => `copy_${d}.blend`); - for await (const result of results) { + for (const result of results) { expect(expectedResults).toContain(result); } diff --git a/tests/e2e/strategies.spec.ts b/tests/e2e/strategies.spec.ts index b13528bc2..9c22225ad 100644 --- a/tests/e2e/strategies.spec.ts +++ b/tests/e2e/strategies.spec.ts @@ -15,12 +15,13 @@ describe("Strategies", function () { logger, }); const data = ["one", "two", "three"]; - const results = executor.map(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - return res.stdout?.toString().trim(); - }); - const finalOutputs: string[] = []; - for await (const res of results) if (res) finalOutputs.push(res); + const futureResults = data.map((x) => + executor.run(async (ctx) => { + const res = await ctx.run(`echo "${x}"`); + return res.stdout?.toString().trim(); + }), + ); + const finalOutputs = (await Promise.all(futureResults)).filter((x) => !!x); expect(finalOutputs).toEqual(expect.arrayContaining(data)); await logger.expectToInclude(`Proposal rejected by Proposal Filter`, 5000); await logger.expectToInclude(`Task 1 computed by provider provider-1`, 5000); @@ -36,12 +37,13 @@ describe("Strategies", function () { logger, }); const data = ["one", "two", "three"]; - const results = executor.map(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - return res.stdout?.toString().trim(); - }); - const finalOutputs: string[] = []; - for await (const res of results) if (res) finalOutputs.push(res); + const futureResults = data.map((x) => + executor.run(async (ctx) => { + const res = await ctx.run(`echo "${x}"`); + return res.stdout?.toString().trim(); + }), + ); + const finalOutputs = (await Promise.all(futureResults)).filter((x) => !!x); expect(finalOutputs).toEqual(expect.arrayContaining(data)); await logger.expectToInclude(`Proposal rejected by Proposal Filter`, 5000); await logger.expectToInclude(`Task 1 computed by provider provider-2`, 5000); @@ -58,12 +60,14 @@ describe("Strategies", function () { logger, }); const data = ["one", "two"]; - const results = executor.map(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - return res.stdout?.toString().trim(); - }); - const finalOutputs: string[] = []; - for await (const res of results) if (res) finalOutputs.push(res); + const futureResults = data.map((x) => + executor.run(async (ctx) => { + const res = await ctx.run(`echo "${x}"`); + return res.stdout?.toString().trim(); + }), + ); + const finalOutputs = (await Promise.all(futureResults)).filter((x) => !!x); + expect(finalOutputs).toEqual(expect.arrayContaining(data)); await executor.end(); await logger.expectToInclude(`Reason: Invoice rejected by Invoice Filter`, 100); @@ -76,12 +80,14 @@ describe("Strategies", function () { logger, }); const data = ["one", "two"]; - const results = executor.map(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - return res.stdout?.toString().trim(); - }); - const finalOutputs: string[] = []; - for await (const res of results) if (res) finalOutputs.push(res); + const futureResults = data.map((x) => + executor.run(async (ctx) => { + const res = await ctx.run(`echo "${x}"`); + return res.stdout?.toString().trim(); + }), + ); + const finalOutputs = (await Promise.all(futureResults)).filter((x) => !!x); + expect(finalOutputs).toEqual(expect.arrayContaining(data)); await executor.end(); await logger.expectToInclude(`DebitNote rejected by DebitNote Filter`, 100); diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index 0d070e86c..ff5c68ddc 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -67,12 +67,13 @@ describe("Task Executor", function () { logger, }); const data = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]; - const results = executor.map(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - return res.stdout?.toString().trim(); - }); - const finalOutputs: string[] = []; - for await (const res of results) if (res) finalOutputs.push(res); + const futureResults = data.map((x) => + executor.run(async (ctx) => { + const res = await ctx.run(`echo "${x}"`); + return res.stdout?.toString().trim(); + }), + ); + const finalOutputs = (await Promise.all(futureResults)).filter((x) => !!x); expect(finalOutputs).toEqual(expect.arrayContaining(data)); }); diff --git a/tests/e2e/yacat.spec.ts b/tests/e2e/yacat.spec.ts index f9612bc2a..22ebfc5dc 100644 --- a/tests/e2e/yacat.spec.ts +++ b/tests/e2e/yacat.spec.ts @@ -34,17 +34,22 @@ describe("Password cracking", function () { if (!keyspace) return; const step = Math.floor(keyspace / 3); const ranges = range(0, keyspace, step); - const results = executor.map(ranges, async (ctx, skip) => { - const results = await ctx - .beginBatch() - .run(`hashcat -a 3 -m 400 '${hash}' '${mask}' --skip=${skip} --limit=${skip! + step} -o pass.potfile -D 1,2`) - .run("cat pass.potfile") - .end(); - if (!results?.[1]?.stdout) return false; - return results?.[1]?.stdout.toString().split(":")?.[1]?.trim(); - }); + const futureResults = ranges.map((skip) => + executor.run(async (ctx) => { + const results = await ctx + .beginBatch() + .run( + `hashcat -a 3 -m 400 '${hash}' '${mask}' --skip=${skip} --limit=${skip! + step} -o pass.potfile -D 1,2`, + ) + .run("cat pass.potfile") + .end(); + if (!results?.[1]?.stdout) return false; + return results?.[1]?.stdout.toString().split(":")?.[1]?.trim(); + }), + ); + const results = await Promise.all(futureResults); let password = ""; - for await (const result of results) { + for (const result of results) { if (result) { password = result; break; From 3c7b11d84da1f7a653edac21070c5b7aa8265871 Mon Sep 17 00:00:00 2001 From: Seweryn Kras Date: Tue, 14 Nov 2023 10:43:41 +0100 Subject: [PATCH 30/38] chore: fix error message formatting --- src/executor/executor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/executor/executor.ts b/src/executor/executor.ts index 285b17533..f454ede24 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -573,8 +573,8 @@ export class TaskExecutor { proposalsCount.initial === 0 && proposalsCount.confirmed === 0 ? "Check your demand if it's not too restrictive or restart yagna." : proposalsCount.initial === proposalsCount.rejected - ? "All off proposals got rejected." - : "Check your proposal filters if they are not too restrictive."; + ? "All off proposals got rejected." + : "Check your proposal filters if they are not too restrictive."; this.handleCriticalError( new Error( `Could not start any work on Golem. Processed ${proposalsCount.initial} initial proposals from yagna, filters accepted ${proposalsCount.confirmed}. ${hint}`, From 2b42153e5cb2170f7833b35b31c24e36967b22a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 10:56:31 +0000 Subject: [PATCH 31/38] build(deps-dev): bump eslint from 8.51.0 to 8.53.0 Bumps [eslint](https://github.com/eslint/eslint) from 8.51.0 to 8.53.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Changelog](https://github.com/eslint/eslint/blob/main/CHANGELOG.md) - [Commits](https://github.com/eslint/eslint/compare/v8.51.0...v8.53.0) --- updated-dependencies: - dependency-name: eslint dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- package-lock.json | 27 +++++++++++++++++---------- package.json | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ef23bac6..5f77eced3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,7 @@ "@typescript-eslint/parser": "^6.11.0", "buffer": "^6.0.3", "cypress": "13.5.0", - "eslint": "~8.51.0", + "eslint": "~8.53.0", "eslint-config-prettier": "^9.0.0", "express": "^4.18.2", "husky": "^8.0.3", @@ -1180,9 +1180,9 @@ "dev": true }, "node_modules/@eslint/js": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.51.0.tgz", - "integrity": "sha512-HxjQ8Qn+4SI3/AFv6sOrDB+g6PpUTDwSJiQqOrnneEk8L71161srI9gjzzZvYVbzHiVg/BvcH95+cK/zfIt4pg==", + "version": "8.53.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.53.0.tgz", + "integrity": "sha512-Kn7K8dx/5U6+cT1yEhpX1w4PCSg0M+XyRILPgvwcEBjerFWCwQj5sbr3/VmxqV0JGHCBCzyd6LxypEuehypY1w==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -4049,6 +4049,12 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", + "dev": true + }, "node_modules/@webassemblyjs/ast": { "version": "1.11.6", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.6.tgz", @@ -6527,18 +6533,19 @@ } }, "node_modules/eslint": { - "version": "8.51.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.51.0.tgz", - "integrity": "sha512-2WuxRZBrlwnXi+/vFSJyjMqrNjtJqiasMzehF0shoLaW7DzS3/9Yvrmq5JiT66+pNjiX4UBnLDiKHcWAr/OInA==", + "version": "8.53.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.53.0.tgz", + "integrity": "sha512-N4VuiPjXDUa4xVeV/GC/RV3hQW9Nw+Y463lkWaKKXKYMvmRiRDAtfpuPFLN+E1/6ZhyR8J2ig+eVREnYgUsiag==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.2", - "@eslint/js": "8.51.0", - "@humanwhocodes/config-array": "^0.11.11", + "@eslint/eslintrc": "^2.1.3", + "@eslint/js": "8.53.0", + "@humanwhocodes/config-array": "^0.11.13", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", diff --git a/package.json b/package.json index fe9670f9a..9086a3b23 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "@typescript-eslint/parser": "^6.11.0", "buffer": "^6.0.3", "cypress": "13.5.0", - "eslint": "~8.51.0", + "eslint": "~8.53.0", "eslint-config-prettier": "^9.0.0", "express": "^4.18.2", "husky": "^8.0.3", From e7413945de67d566617d07594b9531c71326f9eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 10:56:43 +0000 Subject: [PATCH 32/38] build(deps-dev): bump @commitlint/config-conventional Bumps [@commitlint/config-conventional](https://github.com/conventional-changelog/commitlint/tree/HEAD/@commitlint/config-conventional) from 17.8.1 to 18.4.0. - [Release notes](https://github.com/conventional-changelog/commitlint/releases) - [Changelog](https://github.com/conventional-changelog/commitlint/blob/master/@commitlint/config-conventional/CHANGELOG.md) - [Commits](https://github.com/conventional-changelog/commitlint/commits/v18.4.0/@commitlint/config-conventional) --- updated-dependencies: - dependency-name: "@commitlint/config-conventional" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- package-lock.json | 20 ++++++++++---------- package.json | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ef23bac6..1c9ed1e9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ }, "devDependencies": { "@commitlint/cli": "^17.8.1", - "@commitlint/config-conventional": "^17.8.1", + "@commitlint/config-conventional": "^18.4.0", "@johanblumenberg/ts-mockito": "^1.0.40", "@rollup/plugin-alias": "^5.0.1", "@rollup/plugin-commonjs": "^25.0.7", @@ -790,15 +790,15 @@ } }, "node_modules/@commitlint/config-conventional": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-17.8.1.tgz", - "integrity": "sha512-NxCOHx1kgneig3VLauWJcDWS40DVjg7nKOpBEEK9E5fjJpQqLCilcnKkIIjdBH98kEO1q3NpE5NSrZ2kl/QGJg==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/config-conventional/-/config-conventional-18.4.0.tgz", + "integrity": "sha512-vArwCZopsZs0FnGsh9AR7uUTPZ5oVGk8+qnEZWq2KTsMjrE0k80b+oZ32GSQmXQT2iMKVrDC8pKX5uKNkCe9Sw==", "dev": true, "dependencies": { - "conventional-changelog-conventionalcommits": "^6.1.0" + "conventional-changelog-conventionalcommits": "^7.0.2" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/config-validator": { @@ -5542,15 +5542,15 @@ } }, "node_modules/conventional-changelog-conventionalcommits": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/conventional-changelog-conventionalcommits/-/conventional-changelog-conventionalcommits-6.1.0.tgz", - "integrity": "sha512-3cS3GEtR78zTfMzk0AizXKKIdN4OvSh7ibNz6/DPbhWWQu7LqE/8+/GqSodV+sywUR2gpJAdP/1JFf4XtN7Zpw==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/conventional-changelog-conventionalcommits/-/conventional-changelog-conventionalcommits-7.0.2.tgz", + "integrity": "sha512-NKXYmMR/Hr1DevQegFB4MwfM5Vv0m4UIxKZTTYuD98lpTknaZlSRrDOG4X7wIXpGkfsYxZTghUN+Qq+T0YQI7w==", "dev": true, "dependencies": { "compare-func": "^2.0.0" }, "engines": { - "node": ">=14" + "node": ">=16" } }, "node_modules/conventional-changelog-writer": { diff --git a/package.json b/package.json index fe9670f9a..7798783be 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ }, "devDependencies": { "@commitlint/cli": "^17.8.1", - "@commitlint/config-conventional": "^17.8.1", + "@commitlint/config-conventional": "^18.4.0", "@johanblumenberg/ts-mockito": "^1.0.40", "@rollup/plugin-alias": "^5.0.1", "@rollup/plugin-commonjs": "^25.0.7", From 82236eea200bf39bdce1f58b62e24c48bc35762b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Grzywacz?= Date: Wed, 15 Nov 2023 11:41:22 +0100 Subject: [PATCH 33/38] feat: added debug logging to WorkContext commands JST-589 --- .eslintrc | 2 +- .gitignore | 1 + .npmignore | 1 + .prettierignore | 3 +- examples/package-lock.json | 723 +++++++++++++++++++++++++++++++++++-- src/executor/executor.ts | 4 +- src/task/work.ts | 25 +- tsconfig.json | 2 +- 8 files changed, 712 insertions(+), 49 deletions(-) diff --git a/.eslintrc b/.eslintrc index 310fa942a..899ddc510 100644 --- a/.eslintrc +++ b/.eslintrc @@ -24,5 +24,5 @@ "extends": ["eslint:recommended"] } ], - "ignorePatterns": ["dist/", "handbook_gen/", "docs/", "bundle.js", "tests", "*.config.ts"] + "ignorePatterns": ["dist/", "handbook_gen/", "docs/", "bundle.js", "tests", "*.config.ts", "tmp/"] } diff --git a/.gitignore b/.gitignore index 803147597..b25392993 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ logs/ #Nuxt /examples/adv-web-example/.nuxt +tmp/ \ No newline at end of file diff --git a/.npmignore b/.npmignore index 194016ea1..c51168a14 100644 --- a/.npmignore +++ b/.npmignore @@ -6,6 +6,7 @@ /node_modules /src /tests +/tmp .github .gitignore package-lock.json diff --git a/.prettierignore b/.prettierignore index 5dba0d190..d890b167e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,3 @@ dist/ -*.gvmi \ No newline at end of file +*.gvmi +tmp/ \ No newline at end of file diff --git a/examples/package-lock.json b/examples/package-lock.json index 5250718a9..0404166cd 100644 --- a/examples/package-lock.json +++ b/examples/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@golem-sdk/golem-js": "file:..", "commander": "^9.1.0", + "express": "^4.18.2", "ts-node": "^10.7.0" }, "devDependencies": { @@ -26,62 +27,67 @@ "version": "0.8.0", "license": "LGPL-3.0", "dependencies": { - "@rauschma/stringio": "^1.4.0", - "axios": "^1.1.3", + "axios": "^1.6.1", "bottleneck": "^2.19.5", - "collect.js": "^4.34.3", + "collect.js": "^4.36.1", "eventsource": "^2.0.2", "flatbuffers": "^23.5.26", - "ip-num": "^1.4.1", - "js-sha3": "^0.8.0", - "pino": "^8.11.0", - "pino-pretty": "^10.0.1", + "ip-num": "^1.5.1", + "js-sha3": "^0.9.2", + "pino": "^8.16.1", + "pino-pretty": "^10.2.3", "tmp": "^0.2.1", - "uuid": "^9.0.0", + "uuid": "^9.0.1", "ya-ts-client": "^0.5.3" }, "devDependencies": { - "@commitlint/cli": "^17.7.1", - "@commitlint/config-conventional": "^17.7.0", - "@rollup/plugin-alias": "^5.0.0", - "@rollup/plugin-commonjs": "^25.0.3", - "@rollup/plugin-json": "^6.0.0", - "@rollup/plugin-node-resolve": "^15.0.1", - "@rollup/plugin-terser": "^0.4.0", - "@rollup/plugin-typescript": "^11.1.2", - "@types/eventsource": "^1.1.11", - "@types/jest": "^29.5.3", - "@types/node": "^20.4.2", - "@types/uuid": "^9.0.2", - "@typescript-eslint/eslint-plugin": "^6.0.0", - "@typescript-eslint/parser": "^6.0.0", + "@commitlint/cli": "^17.8.1", + "@commitlint/config-conventional": "^17.8.1", + "@johanblumenberg/ts-mockito": "^1.0.40", + "@rollup/plugin-alias": "^5.0.1", + "@rollup/plugin-commonjs": "^25.0.7", + "@rollup/plugin-json": "^6.0.1", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-terser": "^0.4.4", + "@rollup/plugin-typescript": "^11.1.5", + "@types/eventsource": "^1.1.15", + "@types/express": "^4.17.21", + "@types/jest": "^29.5.8", + "@types/node": "^20.9.0", + "@types/supertest": "^2.0.16", + "@types/uuid": "^9.0.7", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "@typescript-eslint/parser": "^6.11.0", "buffer": "^6.0.3", - "cypress": "12.17.4", - "eslint": "~8.47.0", + "cypress": "13.5.0", + "eslint": "~8.51.0", "eslint-config-prettier": "^9.0.0", + "express": "^4.18.2", "husky": "^8.0.3", - "jest": "^29.6.2", - "prettier": "^3.0.0", - "rollup": "^3.26.3", + "jest": "^29.7.0", + "prettier": "^3.1.0", + "rollup": "^3.29.4", + "rollup-plugin-filesize": "^10.0.0", "rollup-plugin-ignore": "^1.0.10", "rollup-plugin-polyfill-node": "^0.12.0", - "rollup-plugin-visualizer": "^5.9.0", - "semantic-release": "^21.0.7", + "rollup-plugin-visualizer": "^5.9.2", + "semantic-release": "^21.1.2", "stream-browserify": "^3.0.0", + "supertest": "^6.3.3", "ts-jest": "^29.1.1", - "ts-loader": "^9.4.1", + "ts-loader": "^9.5.0", "ts-node": "^10.9.1", - "tsconfig-paths": "^4.1.0", + "tsconfig-paths": "^4.2.0", "tslint-config-prettier": "^1.18.0", - "typedoc": "^0.24.0", - "typedoc-plugin-markdown": "^3.14.0", - "typedoc-plugin-merge-modules": "^5.0.1", - "typedoc-theme-hierarchy": "4.1.0", - "typescript": "^5.1.6", - "webpack": "^5.75.0" + "typedoc": "^0.25.3", + "typedoc-plugin-markdown": "^3.17.1", + "typedoc-plugin-merge-modules": "^5.1.0", + "typedoc-theme-hierarchy": "4.1.2", + "typescript": "^5.2.2", + "webpack": "^5.89.0" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.0.0" } }, "node_modules/@cspotcode/source-map-support": { @@ -146,6 +152,18 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.17.1.tgz", "integrity": "sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw==" }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", @@ -170,6 +188,55 @@ "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==" }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, + "node_modules/body-parser": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", + "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "dependencies": { + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.1", + "set-function-length": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/commander": { "version": "9.5.0", "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", @@ -178,11 +245,81 @@ "node": "^12.20.0 || >=14" } }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/define-data-property": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", + "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "dependencies": { + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -191,11 +328,485 @@ "node": ">=0.3.1" } }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", + "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "dependencies": { + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", + "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", + "dependencies": { + "get-intrinsic": "^1.2.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-function-length": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz", + "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==", + "dependencies": { + "define-data-property": "^1.1.1", + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/ts-node": { "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", @@ -238,6 +849,18 @@ } } }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typescript": { "version": "4.9.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", @@ -250,11 +873,35 @@ "node": ">=4.2.0" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==" }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/src/executor/executor.ts b/src/executor/executor.ts index 750b563a8..b8df4ed54 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -556,8 +556,8 @@ export class TaskExecutor { proposalsCount.initial === 0 && proposalsCount.confirmed === 0 ? "Check your demand if it's not too restrictive or restart yagna." : proposalsCount.initial === proposalsCount.rejected - ? "All off proposals got rejected." - : "Check your proposal filters if they are not too restrictive."; + ? "All off proposals got rejected." + : "Check your proposal filters if they are not too restrictive."; const errorMessage = `Could not start any work on Golem. Processed ${proposalsCount.initial} initial proposals from yagna, filters accepted ${proposalsCount.confirmed}. ${hint}`; if (this.options.exitOnNoProposals) { this.handleCriticalError(new Error(errorMessage)); diff --git a/src/task/work.ts b/src/task/work.ts index d636c0f37..4cee97039 100644 --- a/src/task/work.ts +++ b/src/task/work.ts @@ -13,7 +13,7 @@ import { UploadFile, } from "../script"; import { NullStorageProvider, StorageProvider } from "../storage"; -import { Logger, sleep } from "../utils"; +import { Logger, nullLogger, sleep } from "../utils"; import { Batch } from "./batch"; import { NetworkNode } from "../network"; @@ -53,7 +53,7 @@ export class WorkContext { public readonly agreementId: string; public readonly activityId: string; private readonly activityPreparingTimeout: number; - private readonly logger?: Logger; + private readonly logger: Logger; private readonly activityStateCheckingInterval: number; private readonly storageProvider: StorageProvider; private readonly networkNode?: NetworkNode; @@ -65,14 +65,14 @@ export class WorkContext { this.agreementId = this.activity.agreementId; this.activityId = this.activity.id; this.activityPreparingTimeout = options?.activityPreparingTimeout || DEFAULTS.activityPreparingTimeout; - this.logger = options?.logger; + this.logger = options?.logger ?? nullLogger(); this.activityStateCheckingInterval = options?.activityStateCheckingInterval || DEFAULTS.activityStateCheckInterval; this.provider = options?.provider; this.storageProvider = options?.storageProvider ?? new NullStorageProvider(); this.networkNode = options?.networkNode; } async before(): Promise { - let state = await this.activity.getState().catch((e) => this.logger?.debug(e)); + let state = await this.activity.getState().catch((e) => this.logger.debug(e)); if (state === ActivityStateEnum.Ready) { if (this.options?.initWorker) await this.options?.initWorker(this, undefined); return; @@ -101,7 +101,7 @@ export class WorkContext { ]).finally(() => clearTimeout(timeoutId)); } await sleep(this.activityStateCheckingInterval, true); - state = await this.activity.getState().catch((e) => this.logger?.warn(`${e} Provider: ${this.provider?.name}`)); + state = await this.activity.getState().catch((e) => this.logger.warn(`${e} Provider: ${this.provider?.name}`)); if (state !== ActivityStateEnum.Ready) { throw new Error(`Activity ${this.activity.id} cannot reach the Ready state. Current state: ${state}`); } @@ -127,6 +127,12 @@ export class WorkContext { async run(exeOrCmd: string, argsOrOptions?: string[] | CommandOptions, options?: CommandOptions): Promise { const isArray = Array.isArray(argsOrOptions); + if (isArray) { + this.logger.debug(`WorkContext: running command: ${exeOrCmd} ${argsOrOptions?.join(" ")}`); + } else { + this.logger.debug(`WorkContext: running command: ${exeOrCmd}`); + } + const run = isArray ? new Run(exeOrCmd, argsOrOptions as string[], options?.env, options?.capture) : new Run("/bin/sh", ["-c", exeOrCmd], argsOrOptions?.env, argsOrOptions?.capture); @@ -143,33 +149,40 @@ export class WorkContext { * @param options Additional run options. */ async transfer(from: string, to: string, options?: CommandOptions): Promise { + this.logger.debug(`WorkContext: transfering ${from} to ${to}`); return this.runOneCommand(new Transfer(from, to), options); } async uploadFile(src: string, dst: string, options?: CommandOptions): Promise { + this.logger.debug(`WorkContext: uploading file ${src} to ${dst}`); return this.runOneCommand(new UploadFile(this.storageProvider, src, dst), options); } // eslint-disable-next-line @typescript-eslint/no-explicit-any uploadJson(json: any, dst: string, options?: CommandOptions): Promise { + this.logger.debug(`WorkContext: uploading json to ${dst}`); const src = new TextEncoder().encode(JSON.stringify(json)); return this.runOneCommand(new UploadData(this.storageProvider, src, dst), options); } uploadData(data: Uint8Array, dst: string, options?: CommandOptions): Promise { + this.logger.debug(`WorkContext: uploading data to ${dst}`); return this.runOneCommand(new UploadData(this.storageProvider, data, dst), options); } downloadFile(src: string, dst: string, options?: CommandOptions): Promise { + this.logger.debug(`WorkContext: downloading file from ${src} to ${dst}`); return this.runOneCommand(new DownloadFile(this.storageProvider, src, dst), options); } downloadData(src: string, options?: CommandOptions): Promise> { + this.logger.debug(`WorkContext: downloading data from ${src}`); return this.runOneCommand(new DownloadData(this.storageProvider, src), options); } // eslint-disable-next-line @typescript-eslint/no-explicit-any async downloadJson(src: string, options?: CommandOptions): Promise { + this.logger.debug(`WorkContext: downloading json from ${src}`); const result = await this.downloadData(src, options); if (result.result !== ResultState.Ok) { return new Result({ @@ -244,7 +257,7 @@ export class WorkContext { `Error: ${err.message}. Stdout: ${err.stdout?.toString().trim()}. Stderr: ${err.stderr?.toString().trim()}`, ) .join(". "); - this.logger?.warn(`Task error on provider ${this.provider?.name || "'unknown'"}. ${errorMessage}`); + this.logger.warn(`Task error on provider ${this.provider?.name || "'unknown'"}. ${errorMessage}`); } return allResults[0]; diff --git a/tsconfig.json b/tsconfig.json index 3cfa5b16a..743a58d5e 100755 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,5 +30,5 @@ "excludeInternal": true }, "include": ["src"], - "exclude": ["**/*.spec.ts"] + "exclude": ["**/*.spec.ts", "tmp/"] } From bd1b12dc3cbb6c3035e63ee03c51d56b838b39b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 21:46:24 +0000 Subject: [PATCH 34/38] build(deps-dev): bump @commitlint/cli from 17.8.1 to 18.4.1 Bumps [@commitlint/cli](https://github.com/conventional-changelog/commitlint/tree/HEAD/@commitlint/cli) from 17.8.1 to 18.4.1. - [Release notes](https://github.com/conventional-changelog/commitlint/releases) - [Changelog](https://github.com/conventional-changelog/commitlint/blob/master/@commitlint/cli/CHANGELOG.md) - [Commits](https://github.com/conventional-changelog/commitlint/commits/v18.4.1/@commitlint/cli) --- updated-dependencies: - dependency-name: "@commitlint/cli" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- package-lock.json | 457 ++++++++++++++++++---------------------------- package.json | 2 +- 2 files changed, 180 insertions(+), 279 deletions(-) diff --git a/package-lock.json b/package-lock.json index 90e7bf61f..cbe7bb499 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "ya-ts-client": "^0.5.3" }, "devDependencies": { - "@commitlint/cli": "^17.8.1", + "@commitlint/cli": "^18.4.1", "@commitlint/config-conventional": "^18.4.0", "@johanblumenberg/ts-mockito": "^1.0.40", "@rollup/plugin-alias": "^5.0.1", @@ -766,16 +766,16 @@ } }, "node_modules/@commitlint/cli": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-17.8.1.tgz", - "integrity": "sha512-ay+WbzQesE0Rv4EQKfNbSMiJJ12KdKTDzIt0tcK4k11FdsWmtwP0Kp1NWMOUswfIWo6Eb7p7Ln721Nx9FLNBjg==", + "version": "18.4.1", + "resolved": "https://registry.npmjs.org/@commitlint/cli/-/cli-18.4.1.tgz", + "integrity": "sha512-4+jljfd29Udw9RDDyigavLO9LvdbmB8O9xjDzVZ0R3lJuG7nCeyHgnKWIVpFaN590isZMV/cMeQK0gH7hRF40A==", "dev": true, "dependencies": { - "@commitlint/format": "^17.8.1", - "@commitlint/lint": "^17.8.1", - "@commitlint/load": "^17.8.1", - "@commitlint/read": "^17.8.1", - "@commitlint/types": "^17.8.1", + "@commitlint/format": "^18.4.0", + "@commitlint/lint": "^18.4.0", + "@commitlint/load": "^18.4.1", + "@commitlint/read": "^18.4.0", + "@commitlint/types": "^18.4.0", "execa": "^5.0.0", "lodash.isfunction": "^3.0.9", "resolve-from": "5.0.0", @@ -786,7 +786,7 @@ "commitlint": "cli.js" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/config-conventional": { @@ -802,25 +802,25 @@ } }, "node_modules/@commitlint/config-validator": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-17.8.1.tgz", - "integrity": "sha512-UUgUC+sNiiMwkyiuIFR7JG2cfd9t/7MV8VB4TZ+q02ZFkHoduUS4tJGsCBWvBOGD9Btev6IecPMvlWUfJorkEA==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/config-validator/-/config-validator-18.4.0.tgz", + "integrity": "sha512-1y6qHMU3o4cYQSK+Y9EnmH6H1GRiwQGjnLIUOIKlekrmfc8MrMk1ByNmb8od4vK3qHJAaL/77/5n+1uyyIF5dA==", "dev": true, "dependencies": { - "@commitlint/types": "^17.8.1", + "@commitlint/types": "^18.4.0", "ajv": "^8.11.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/ensure": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/ensure/-/ensure-17.8.1.tgz", - "integrity": "sha512-xjafwKxid8s1K23NFpL8JNo6JnY/ysetKo8kegVM7c8vs+kWLP8VrQq+NbhgVlmCojhEDbzQKp4eRXSjVOGsow==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/ensure/-/ensure-18.4.0.tgz", + "integrity": "sha512-N5cJo/n61ULSwz3W5Iz/IZJ0I9H/PaHc+OMcF2XcRVbLa6B3YwzEW66XGCRKVULlsBNSrIH6tk5un9ayXAXIdw==", "dev": true, "dependencies": { - "@commitlint/types": "^17.8.1", + "@commitlint/types": "^18.4.0", "lodash.camelcase": "^4.3.0", "lodash.kebabcase": "^4.1.1", "lodash.snakecase": "^4.1.1", @@ -828,193 +828,194 @@ "lodash.upperfirst": "^4.3.1" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/execute-rule": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/execute-rule/-/execute-rule-17.8.1.tgz", - "integrity": "sha512-JHVupQeSdNI6xzA9SqMF+p/JjrHTcrJdI02PwesQIDCIGUrv04hicJgCcws5nzaoZbROapPs0s6zeVHoxpMwFQ==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/execute-rule/-/execute-rule-18.4.0.tgz", + "integrity": "sha512-g013SWki6ZWhURBLOSXTaVQGWHdA0QlPJGiW4a+YpThezmJOemvc4LiKVpn13AjSKQ40QnmBqpBrxujOaSo+3A==", "dev": true, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/format": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/format/-/format-17.8.1.tgz", - "integrity": "sha512-f3oMTyZ84M9ht7fb93wbCKmWxO5/kKSbwuYvS867duVomoOsgrgljkGGIztmT/srZnaiGbaK8+Wf8Ik2tSr5eg==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/format/-/format-18.4.0.tgz", + "integrity": "sha512-MiAe4D5/ahty38CzULdQbpRa3ReKZtx0kyigOWcntq+N5uqez+Ac4/MO7H+3j1kC4G7nfJVfBu6TqcXeyNvhCQ==", "dev": true, "dependencies": { - "@commitlint/types": "^17.8.1", + "@commitlint/types": "^18.4.0", "chalk": "^4.1.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/is-ignored": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-17.8.1.tgz", - "integrity": "sha512-UshMi4Ltb4ZlNn4F7WtSEugFDZmctzFpmbqvpyxD3la510J+PLcnyhf9chs7EryaRFJMdAKwsEKfNK0jL/QM4g==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/is-ignored/-/is-ignored-18.4.0.tgz", + "integrity": "sha512-vyBKBj3Q4N3Xe4ZQcJXW9ef6gVrDL9Fl2HXnnC3F0Qt/F6E4runhJkEuUh5DB3WCXTJUHIJkByKPqrnz4RNrZw==", "dev": true, "dependencies": { - "@commitlint/types": "^17.8.1", + "@commitlint/types": "^18.4.0", "semver": "7.5.4" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/lint": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-17.8.1.tgz", - "integrity": "sha512-aQUlwIR1/VMv2D4GXSk7PfL5hIaFSfy6hSHV94O8Y27T5q+DlDEgd/cZ4KmVI+MWKzFfCTiTuWqjfRSfdRllCA==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/lint/-/lint-18.4.0.tgz", + "integrity": "sha512-Wkkf1DPVeLdHYGqtzMBfWoMbUtCojvlzDR89OKVic1rid41iZbb0FzTcwgMYs/1TNWNxoIq9PVVwY7ovLX1aJQ==", "dev": true, "dependencies": { - "@commitlint/is-ignored": "^17.8.1", - "@commitlint/parse": "^17.8.1", - "@commitlint/rules": "^17.8.1", - "@commitlint/types": "^17.8.1" + "@commitlint/is-ignored": "^18.4.0", + "@commitlint/parse": "^18.4.0", + "@commitlint/rules": "^18.4.0", + "@commitlint/types": "^18.4.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/load": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-17.8.1.tgz", - "integrity": "sha512-iF4CL7KDFstP1kpVUkT8K2Wl17h2yx9VaR1ztTc8vzByWWcbO/WaKwxsnCOqow9tVAlzPfo1ywk9m2oJ9ucMqA==", + "version": "18.4.1", + "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-18.4.1.tgz", + "integrity": "sha512-o/plBiPJQgbSq/4ipDpsq4HCmURjBAEjr1EO/p2falr3VhwV0WGXTvb8NlihgI8xtSyO6lHvtycrE535GMLQbA==", "dev": true, "dependencies": { - "@commitlint/config-validator": "^17.8.1", - "@commitlint/execute-rule": "^17.8.1", - "@commitlint/resolve-extends": "^17.8.1", - "@commitlint/types": "^17.8.1", - "@types/node": "20.5.1", + "@commitlint/config-validator": "^18.4.0", + "@commitlint/execute-rule": "^18.4.0", + "@commitlint/resolve-extends": "^18.4.0", + "@commitlint/types": "^18.4.0", + "@types/node": "^18.11.9", "chalk": "^4.1.0", - "cosmiconfig": "^8.0.0", - "cosmiconfig-typescript-loader": "^4.0.0", + "cosmiconfig": "^8.3.6", + "cosmiconfig-typescript-loader": "^5.0.0", "lodash.isplainobject": "^4.0.6", "lodash.merge": "^4.6.2", "lodash.uniq": "^4.5.0", - "resolve-from": "^5.0.0", - "ts-node": "^10.8.1", - "typescript": "^4.6.4 || ^5.2.2" + "resolve-from": "^5.0.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/load/node_modules/@types/node": { - "version": "20.5.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz", - "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==", - "dev": true + "version": "18.18.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.9.tgz", + "integrity": "sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@commitlint/message": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/message/-/message-17.8.1.tgz", - "integrity": "sha512-6bYL1GUQsD6bLhTH3QQty8pVFoETfFQlMn2Nzmz3AOLqRVfNNtXBaSY0dhZ0dM6A2MEq4+2d7L/2LP8TjqGRkA==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/message/-/message-18.4.0.tgz", + "integrity": "sha512-3kg6NQO6pJ+VdBTWi51KInT8ngkxPJaW+iI7URtUALjKcO9K4XY3gf80ZPmS1hDessrjb7qCr1lau8eWMINAQw==", "dev": true, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/parse": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/parse/-/parse-17.8.1.tgz", - "integrity": "sha512-/wLUickTo0rNpQgWwLPavTm7WbwkZoBy3X8PpkUmlSmQJyWQTj0m6bDjiykMaDt41qcUbfeFfaCvXfiR4EGnfw==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/parse/-/parse-18.4.0.tgz", + "integrity": "sha512-SxTCSUZH8CJNYWOlFg18YUQ2RLz8ubXKbpHUIiSNwCbiQx7UDCydp1JnhoB4sOYOxgV8d3nuDwYluRU5KnEY4A==", "dev": true, "dependencies": { - "@commitlint/types": "^17.8.1", + "@commitlint/types": "^18.4.0", "conventional-changelog-angular": "^6.0.0", - "conventional-commits-parser": "^4.0.0" + "conventional-commits-parser": "^5.0.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/read": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-17.8.1.tgz", - "integrity": "sha512-Fd55Oaz9irzBESPCdMd8vWWgxsW3OWR99wOntBDHgf9h7Y6OOHjWEdS9Xzen1GFndqgyoaFplQS5y7KZe0kO2w==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/read/-/read-18.4.0.tgz", + "integrity": "sha512-IpnABCbDeOw5npZ09SZZGLfd3T7cFtsxUYm6wT3aGmIB2fXKE3fMeuj3jxXjMibiGIyA3Z5voCMuOcKWpkNySA==", "dev": true, "dependencies": { - "@commitlint/top-level": "^17.8.1", - "@commitlint/types": "^17.8.1", + "@commitlint/top-level": "^18.4.0", + "@commitlint/types": "^18.4.0", "fs-extra": "^11.0.0", "git-raw-commits": "^2.0.11", "minimist": "^1.2.6" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/resolve-extends": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-17.8.1.tgz", - "integrity": "sha512-W/ryRoQ0TSVXqJrx5SGkaYuAaE/BUontL1j1HsKckvM6e5ZaG0M9126zcwL6peKSuIetJi7E87PRQF8O86EW0Q==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-18.4.0.tgz", + "integrity": "sha512-qhgU6ach+S6sJMD9NjCYiEycOObGhxzWQLQzqlScJCv9zkPs15Bg0ffLXTQ3z7ipXv46XEKYMnSJzjLRw2Tlkg==", "dev": true, "dependencies": { - "@commitlint/config-validator": "^17.8.1", - "@commitlint/types": "^17.8.1", + "@commitlint/config-validator": "^18.4.0", + "@commitlint/types": "^18.4.0", "import-fresh": "^3.0.0", "lodash.mergewith": "^4.6.2", "resolve-from": "^5.0.0", "resolve-global": "^1.0.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/rules": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/rules/-/rules-17.8.1.tgz", - "integrity": "sha512-2b7OdVbN7MTAt9U0vKOYKCDsOvESVXxQmrvuVUZ0rGFMCrCPJWWP1GJ7f0lAypbDAhaGb8zqtdOr47192LBrIA==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/rules/-/rules-18.4.0.tgz", + "integrity": "sha512-T3ChRxQZ6g0iNCpVLc6KeQId0/86TnyQA8PFkng+dWElO2DAA5km/yirgKZV1Xlc+gF7Rf6d+a0ottxdKpOY+w==", "dev": true, "dependencies": { - "@commitlint/ensure": "^17.8.1", - "@commitlint/message": "^17.8.1", - "@commitlint/to-lines": "^17.8.1", - "@commitlint/types": "^17.8.1", + "@commitlint/ensure": "^18.4.0", + "@commitlint/message": "^18.4.0", + "@commitlint/to-lines": "^18.4.0", + "@commitlint/types": "^18.4.0", "execa": "^5.0.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/to-lines": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/to-lines/-/to-lines-17.8.1.tgz", - "integrity": "sha512-LE0jb8CuR/mj6xJyrIk8VLz03OEzXFgLdivBytoooKO5xLt5yalc8Ma5guTWobw998sbR3ogDd+2jed03CFmJA==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/to-lines/-/to-lines-18.4.0.tgz", + "integrity": "sha512-bZXuCtfBPjNgtEnG3gwJrveIgfKK2UdhIhFvKpMTrQl/gAwoto/3mzmE7qGAHwmuP4eZ2U8X7iwMnqIlWmv2Tw==", "dev": true, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/top-level": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/top-level/-/top-level-17.8.1.tgz", - "integrity": "sha512-l6+Z6rrNf5p333SHfEte6r+WkOxGlWK4bLuZKbtf/2TXRN+qhrvn1XE63VhD8Oe9oIHQ7F7W1nG2k/TJFhx2yA==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/top-level/-/top-level-18.4.0.tgz", + "integrity": "sha512-TfulcA8UHF7MZ6tm4Ci3aqZgMBZa1OoCg4prccWHvwG/hsHujZ7+0FKbeKqDbcSli/YWm4NJwEjl4uh5itIJeA==", "dev": true, "dependencies": { "find-up": "^5.0.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@commitlint/types": { - "version": "17.8.1", - "resolved": "https://registry.npmjs.org/@commitlint/types/-/types-17.8.1.tgz", - "integrity": "sha512-PXDQXkAmiMEG162Bqdh9ChML/GJZo6vU+7F03ALKDK8zYc6SuAr47LjG7hGYRqUOz+WK0dU7bQ0xzuqFMdxzeQ==", + "version": "18.4.0", + "resolved": "https://registry.npmjs.org/@commitlint/types/-/types-18.4.0.tgz", + "integrity": "sha512-MKeaFxt0I9fhqUb2E+YIzX/gZtmkuodJET/XKiZIMvXUff8Ee4Ih86eLg+yAm2jf1pwGBmU02uNOp0y094w2Uw==", "dev": true, "dependencies": { "chalk": "^4.1.0" }, "engines": { - "node": ">=v14" + "node": ">=v18" } }, "node_modules/@cspotcode/source-map-support": { @@ -2421,69 +2422,6 @@ "semantic-release": ">=20.1.0" } }, - "node_modules/@semantic-release/commit-analyzer/node_modules/conventional-commits-parser": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-5.0.0.tgz", - "integrity": "sha512-ZPMl0ZJbw74iS9LuX9YIAiW8pfM5p3yh2o/NbXHbkFuZzY5jvdi5jFycEOkmBW5H5I7nA+D6f3UcsCLP2vvSEA==", - "dev": true, - "dependencies": { - "is-text-path": "^2.0.0", - "JSONStream": "^1.3.5", - "meow": "^12.0.1", - "split2": "^4.0.0" - }, - "bin": { - "conventional-commits-parser": "cli.mjs" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/@semantic-release/commit-analyzer/node_modules/is-text-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-2.0.0.tgz", - "integrity": "sha512-+oDTluR6WEjdXEJMnC2z6A4FRwFoYuvShVVEGsS7ewc0UTi2QtAKMDJuL4BDEVt+5T7MjFo12RP8ghOM75oKJw==", - "dev": true, - "dependencies": { - "text-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@semantic-release/commit-analyzer/node_modules/meow": { - "version": "12.1.1", - "resolved": "https://registry.npmjs.org/meow/-/meow-12.1.1.tgz", - "integrity": "sha512-BhXM0Au22RwUneMPwSCnyhTOizdWoIEPU9sp0Aqa1PnDMR5Wv2FGXYDjuzJEIX+Eo2Rb8xuYe5jrnm5QowQFkw==", - "dev": true, - "engines": { - "node": ">=16.10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@semantic-release/commit-analyzer/node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "dev": true, - "engines": { - "node": ">= 10.x" - } - }, - "node_modules/@semantic-release/commit-analyzer/node_modules/text-extensions": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-2.4.0.tgz", - "integrity": "sha512-te/NtwBwfiNRLf9Ijqx3T0nlqZiQ2XrrtBvu+cLL8ZRrGkO0NHTug8MYFKyoSrv/sHTaSKfilUkizV6XhxMJ3g==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@semantic-release/error": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@semantic-release/error/-/error-4.0.0.tgz", @@ -3027,24 +2965,6 @@ "node": ">=16" } }, - "node_modules/@semantic-release/release-notes-generator/node_modules/conventional-commits-parser": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-5.0.0.tgz", - "integrity": "sha512-ZPMl0ZJbw74iS9LuX9YIAiW8pfM5p3yh2o/NbXHbkFuZzY5jvdi5jFycEOkmBW5H5I7nA+D6f3UcsCLP2vvSEA==", - "dev": true, - "dependencies": { - "is-text-path": "^2.0.0", - "JSONStream": "^1.3.5", - "meow": "^12.0.1", - "split2": "^4.0.0" - }, - "bin": { - "conventional-commits-parser": "cli.mjs" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/@semantic-release/release-notes-generator/node_modules/find-up": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", @@ -3085,18 +3005,6 @@ "node": "^16.14.0 || >=18.0.0" } }, - "node_modules/@semantic-release/release-notes-generator/node_modules/is-text-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-2.0.0.tgz", - "integrity": "sha512-+oDTluR6WEjdXEJMnC2z6A4FRwFoYuvShVVEGsS7ewc0UTi2QtAKMDJuL4BDEVt+5T7MjFo12RP8ghOM75oKJw==", - "dev": true, - "dependencies": { - "text-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/@semantic-release/release-notes-generator/node_modules/json-parse-even-better-errors": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", @@ -3142,18 +3050,6 @@ "node": "14 || >=16.14" } }, - "node_modules/@semantic-release/release-notes-generator/node_modules/meow": { - "version": "12.1.1", - "resolved": "https://registry.npmjs.org/meow/-/meow-12.1.1.tgz", - "integrity": "sha512-BhXM0Au22RwUneMPwSCnyhTOizdWoIEPU9sp0Aqa1PnDMR5Wv2FGXYDjuzJEIX+Eo2Rb8xuYe5jrnm5QowQFkw==", - "dev": true, - "engines": { - "node": ">=16.10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@semantic-release/release-notes-generator/node_modules/normalize-package-data": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.0.tgz", @@ -3274,27 +3170,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@semantic-release/release-notes-generator/node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "dev": true, - "engines": { - "node": ">= 10.x" - } - }, - "node_modules/@semantic-release/release-notes-generator/node_modules/text-extensions": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-2.4.0.tgz", - "integrity": "sha512-te/NtwBwfiNRLf9Ijqx3T0nlqZiQ2XrrtBvu+cLL8ZRrGkO0NHTug8MYFKyoSrv/sHTaSKfilUkizV6XhxMJ3g==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@semantic-release/release-notes-generator/node_modules/type-fest": { "version": "4.7.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.7.1.tgz", @@ -5603,21 +5478,33 @@ } }, "node_modules/conventional-commits-parser": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-4.0.0.tgz", - "integrity": "sha512-WRv5j1FsVM5FISJkoYMR6tPk07fkKT0UodruX4je86V4owk451yjXAKzKAPOs9l7y59E2viHUS9eQ+dfUA9NSg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-5.0.0.tgz", + "integrity": "sha512-ZPMl0ZJbw74iS9LuX9YIAiW8pfM5p3yh2o/NbXHbkFuZzY5jvdi5jFycEOkmBW5H5I7nA+D6f3UcsCLP2vvSEA==", "dev": true, "dependencies": { - "is-text-path": "^1.0.1", + "is-text-path": "^2.0.0", "JSONStream": "^1.3.5", - "meow": "^8.1.2", - "split2": "^3.2.2" + "meow": "^12.0.1", + "split2": "^4.0.0" }, "bin": { - "conventional-commits-parser": "cli.js" + "conventional-commits-parser": "cli.mjs" }, "engines": { - "node": ">=14" + "node": ">=16" + } + }, + "node_modules/conventional-commits-parser/node_modules/meow": { + "version": "12.1.1", + "resolved": "https://registry.npmjs.org/meow/-/meow-12.1.1.tgz", + "integrity": "sha512-BhXM0Au22RwUneMPwSCnyhTOizdWoIEPU9sp0Aqa1PnDMR5Wv2FGXYDjuzJEIX+Eo2Rb8xuYe5jrnm5QowQFkw==", + "dev": true, + "engines": { + "node": ">=16.10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/convert-source-map": { @@ -5680,17 +5567,19 @@ } }, "node_modules/cosmiconfig-typescript-loader": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.4.0.tgz", - "integrity": "sha512-BabizFdC3wBHhbI4kJh0VkQP9GkBfoHPydD0COMce1nJ1kJAB3F2TmJ/I7diULBKtmEWSwEbuN/KDtgnmUUVmw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-5.0.0.tgz", + "integrity": "sha512-+8cK7jRAReYkMwMiG+bxhcNKiHJDM6bR9FD/nGBXOWdMLuYawjF5cGrtLilJ+LGd3ZjCXnJjR5DkfWPoIVlqJA==", "dev": true, + "dependencies": { + "jiti": "^1.19.1" + }, "engines": { - "node": ">=v14.21.3" + "node": ">=v16" }, "peerDependencies": { "@types/node": "*", - "cosmiconfig": ">=7", - "ts-node": ">=10", + "cosmiconfig": ">=8.2", "typescript": ">=4" } }, @@ -7584,6 +7473,29 @@ "node": ">=10" } }, + "node_modules/git-raw-commits/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/git-raw-commits/node_modules/split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "dev": true, + "dependencies": { + "readable-stream": "^3.0.0" + } + }, "node_modules/glob": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", @@ -8420,15 +8332,15 @@ } }, "node_modules/is-text-path": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz", - "integrity": "sha512-xFuJpne9oFz5qDaodwmmG08e3CawH/2ZV8Qqza1Ko7Sk8POWbkRdwIoAWVhqvq0XeUzANEhKo2n0IXUGBm7A/w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-2.0.0.tgz", + "integrity": "sha512-+oDTluR6WEjdXEJMnC2z6A4FRwFoYuvShVVEGsS7ewc0UTi2QtAKMDJuL4BDEVt+5T7MjFo12RP8ghOM75oKJw==", "dev": true, "dependencies": { - "text-extensions": "^1.0.0" + "text-extensions": "^2.0.0" }, "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, "node_modules/is-typedarray": { @@ -9228,6 +9140,15 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/jiti": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz", + "integrity": "sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==", + "dev": true, + "bin": { + "jiti": "bin/jiti.js" + } + }, "node_modules/joycon": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", @@ -14407,14 +14328,6 @@ "split2": "^4.0.0" } }, - "node_modules/pino-abstract-transport/node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "engines": { - "node": ">= 10.x" - } - }, "node_modules/pino-pretty": { "version": "10.2.3", "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.3.tgz", @@ -16643,26 +16556,11 @@ } }, "node_modules/split2": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", - "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", - "dev": true, - "dependencies": { - "readable-stream": "^3.0.0" - } - }, - "node_modules/split2/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", "engines": { - "node": ">= 6" + "node": ">= 10.x" } }, "node_modules/sprintf-js": { @@ -17264,12 +17162,15 @@ } }, "node_modules/text-extensions": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", - "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-2.4.0.tgz", + "integrity": "sha512-te/NtwBwfiNRLf9Ijqx3T0nlqZiQ2XrrtBvu+cLL8ZRrGkO0NHTug8MYFKyoSrv/sHTaSKfilUkizV6XhxMJ3g==", "dev": true, "engines": { - "node": ">=0.10" + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/text-table": { diff --git a/package.json b/package.json index 74b9571f3..9ee09c01f 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "ya-ts-client": "^0.5.3" }, "devDependencies": { - "@commitlint/cli": "^17.8.1", + "@commitlint/cli": "^18.4.1", "@commitlint/config-conventional": "^18.4.0", "@johanblumenberg/ts-mockito": "^1.0.40", "@rollup/plugin-alias": "^5.0.1", From 24c88b989d4cdd80594cbf707ec22f43cee6697d Mon Sep 17 00:00:00 2001 From: Seweryn Kras Date: Thu, 16 Nov 2023 19:23:24 +0100 Subject: [PATCH 35/38] chore: refactor hashcat examples to use promise.any --- .../running-parallel-tasks/index.mjs | 48 +++++++++---------- examples/yacat/yacat.ts | 38 ++++++++------- tests/e2e/yacat.spec.ts | 33 +++++++------ 3 files changed, 59 insertions(+), 60 deletions(-) diff --git a/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs index 71d20b2a9..65aef11ee 100644 --- a/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs +++ b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs @@ -19,37 +19,35 @@ async function main(args) { const step = Math.floor(keyspace / args.numberOfProviders + 1); const range = [...Array(Math.floor(keyspace / step) + 1).keys()].map((i) => i * step); - const futureResults = range.map(async (skip = 0) => { - return executor.run(async (ctx) => { - const results = await ctx + const findPasswordInRange = async (skip) => { + const password = await executor.run(async (ctx) => { + const [, potfileResult] = await ctx .beginBatch() .run( - `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min( - keyspace, - skip + step, - )} -o pass.potfile`, + `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${ + skip + step + } -o pass.potfile || true`, ) - .run("cat pass.potfile") - .end() - .catch((err) => console.error(err)); - if (!results?.[1]?.stdout) return false; - return results?.[1]?.stdout.toString().split(":")[1]; + .run("cat pass.potfile || true") + .end(); + if (!potfileResult.stdout) return false; + // potfile format is: hash:password + return potfileResult.stdout.toString().trim().split(":")[1]; }); - }); - - const results = await Promise.all(futureResults); - - let password = ""; - for (const result of results) { - if (result) { - password = result; - break; + if (!password) { + throw new Error(`Cannot find password in range ${skip} - ${skip + step}`); } + return password; + }; + + try { + const password = await Promise.any(range.map(findPasswordInRange)); + console.log(`Password found: ${password}`); + } catch (err) { + console.log(`Password not found`); + } finally { + await executor.end(); } - - if (!password) console.log("No password found"); - else console.log(`Password found: ${password}`); - await executor.end(); } program diff --git a/examples/yacat/yacat.ts b/examples/yacat/yacat.ts index 139fcf50a..dc1646934 100644 --- a/examples/yacat/yacat.ts +++ b/examples/yacat/yacat.ts @@ -23,33 +23,35 @@ async function main(args) { const range = [...Array(Math.floor(keyspace / step)).keys()].map((i) => i * step); console.log(`Keyspace size computed. Keyspace size = ${keyspace}. Tasks to compute = ${range.length}`); - const futureResults = range.map((skip) => - executor.run(async (ctx) => { - const results = await ctx + const findPasswordInRange = async (skip: number) => { + const password = await executor.run(async (ctx) => { + const [, potfileResult] = await ctx .beginBatch() .run( `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${ - skip! + step + skip + step } -o pass.potfile || true`, ) .run("cat pass.potfile || true") .end(); - if (!results?.[1]?.stdout) return false; - return results?.[1]?.stdout.toString().trim().split(":")[1]; - }), - ); - const results = await Promise.all(futureResults); - - let password = ""; - for await (const result of results) { - if (result) { - password = result; - break; + if (!potfileResult.stdout) return false; + // potfile format is: hash:password + return potfileResult.stdout.toString().trim().split(":")[1]; + }); + if (!password) { + throw new Error(`Cannot find password in range ${skip} - ${skip + step}`); } + return password; + }; + + try { + const password = await Promise.any(range.map(findPasswordInRange)); + console.log(`Password found: ${password}`); + } catch (err) { + console.log(`Password not found`); + } finally { + await executor.end(); } - if (!password) console.log("No password found"); - else console.log(`Password found: ${password}`); - await executor.end(); } program diff --git a/tests/e2e/yacat.spec.ts b/tests/e2e/yacat.spec.ts index 22ebfc5dc..966779ebb 100644 --- a/tests/e2e/yacat.spec.ts +++ b/tests/e2e/yacat.spec.ts @@ -34,28 +34,27 @@ describe("Password cracking", function () { if (!keyspace) return; const step = Math.floor(keyspace / 3); const ranges = range(0, keyspace, step); - const futureResults = ranges.map((skip) => - executor.run(async (ctx) => { - const results = await ctx + + const findPasswordInRange = async (skip: number) => { + const password = await executor.run(async (ctx) => { + const [, potfileResult] = await ctx .beginBatch() .run( - `hashcat -a 3 -m 400 '${hash}' '${mask}' --skip=${skip} --limit=${skip! + step} -o pass.potfile -D 1,2`, + `hashcat -a 3 -m 400 '${hash}' '${mask}' --skip=${skip} --limit=${skip + step} -o pass.potfile || true`, ) - .run("cat pass.potfile") + .run("cat pass.potfile || true") .end(); - if (!results?.[1]?.stdout) return false; - return results?.[1]?.stdout.toString().split(":")?.[1]?.trim(); - }), - ); - const results = await Promise.all(futureResults); - let password = ""; - for (const result of results) { - if (result) { - password = result; - break; + if (!potfileResult.stdout) return false; + // potfile format is: hash:password + return potfileResult.stdout.toString().trim().split(":")[1]; + }); + if (!password) { + throw new Error(`Cannot find password in range ${skip} - ${skip + step}`); } - } - expect(password).toEqual("yo"); + return password; + }; + + await expect(Promise.any(ranges.map(findPasswordInRange))).resolves.toEqual("yo"); await executor.end(); }, 1000 * 60 * 5, From 7ac21f347564d7ec571a1f957f3e8d34b97fced9 Mon Sep 17 00:00:00 2001 From: Seweryn Kras Date: Fri, 17 Nov 2023 10:11:59 +0100 Subject: [PATCH 36/38] feat(executor): remove map and foreach BREAKING CHANGE: executor.map and executor.foreach have been removed --- src/executor/executor.ts | 95 ---------------------------------------- tests/e2e/tasks.spec.ts | 12 ----- 2 files changed, 107 deletions(-) diff --git a/src/executor/executor.ts b/src/executor/executor.ts index 263df29e2..47a09e617 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -342,101 +342,6 @@ export class TaskExecutor { return this.executeTask(worker, undefined, options); } - /** - * @deprecated This method is marked for removal in a future release. Migrate your code by using `Array.map` and `Promise.all` instead. - * @example - * ```typescript - * const data = [1, 2, 3, 4, 5]; - * const futureResults = data.map((item) => - * executor.run((ctx) => { - * console.log((await ctx.run(`echo "${item}"`)).stdout); - * }) - * ); - * const results = await Promise.all(futureResults); - * ``` - * - * Map iterable data to worker function and return computed Task result as AsyncIterable - * - * @param data Iterable data - * @param worker worker function - * @return AsyncIterable with results of computed tasks - * @example - * ```typescript - * const data = [1, 2, 3, 4, 5]; - * const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); - * for await (const result of results) console.log(result.stdout); - * ``` - */ - map( - data: Iterable, - worker: Worker, - ): AsyncIterable { - const inputs = [...data]; - const featureResults = inputs.map((value) => this.executeTask(worker, value)); - const results: OutputType[] = []; - let resultsCount = 0; - featureResults.forEach((featureResult) => - featureResult - .then((res) => { - results.push(res as OutputType); - }) - .catch((e) => this.handleCriticalError(e)), - ); - const isRunning = () => this.isRunning; - return { - [Symbol.asyncIterator](): AsyncIterator { - return { - async next() { - if (resultsCount === inputs.length) { - return Promise.resolve({ done: true, value: undefined }); - } - while (results.length === 0 && resultsCount < inputs.length && isRunning()) { - await sleep(1000, true); - } - if (!isRunning()) return Promise.resolve({ done: true, value: undefined }); - resultsCount += 1; - return Promise.resolve({ done: false, value: results.pop() }); - }, - }; - }, - }; - } - - /** - * @deprecated This method is marked for removal in a future release. - * Migrate your code by using `Array.map` and `Promise.all` instead. - * @example - * ```typescript - * const data = [1, 2, 3, 4, 5]; - * const futureResults = data.map((item) => - * executor.run((ctx) => { - * console.log((await ctx.run(`echo "${item}"`)).stdout); - * }), - * ); - * await Promise.all(futureResults); - * ``` - * - * Iterates over given data and execute task using worker function - * - * @param data Iterable data - * @param worker Worker function - * @example - * ```typescript - * const data = [1, 2, 3, 4, 5]; - * await executor.forEach(data, async (ctx, item) => { - * console.log((await ctx.run(`echo "${item}"`)).stdout); - * }); - * ``` - */ - async forEach( - data: Iterable, - worker: Worker, - ): Promise { - await Promise.all([...data].map((value) => this.executeTask(worker, value))).catch((e) => - this.handleCriticalError(e), - ); - } - private async createPackage( packageReference: RequireAtLeastOne< { imageHash: string; manifest: string; imageTag: string }, diff --git a/tests/e2e/tasks.spec.ts b/tests/e2e/tasks.spec.ts index ff5c68ddc..4974080a1 100644 --- a/tests/e2e/tasks.spec.ts +++ b/tests/e2e/tasks.spec.ts @@ -77,18 +77,6 @@ describe("Task Executor", function () { expect(finalOutputs).toEqual(expect.arrayContaining(data)); }); - it("should run simple tasks by forEach function", async () => { - executor = await TaskExecutor.create({ - package: "golem/alpine:latest", - logger, - }); - const data = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]; - await executor.forEach(data, async (ctx, x) => { - const res = await ctx.run(`echo "${x}"`); - expect(data).toContain(res?.stdout?.toString().trim()); - }); - }); - it("should run simple batch script and get results as stream", async () => { executor = await TaskExecutor.create({ package: "golem/alpine:latest", From f688fad197728a100d5ffe06baee9bf3be947d27 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 17 Nov 2023 13:35:28 +0100 Subject: [PATCH 37/38] chore: formatting fix --- src/activity/activity.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/activity/activity.ts b/src/activity/activity.ts index 1e45234d1..40bddda9e 100644 --- a/src/activity/activity.ts +++ b/src/activity/activity.ts @@ -331,8 +331,8 @@ export class Activity { ? ResultState.Ok : ResultState.Error : event?.kind?.stderr - ? ResultState.Error - : ResultState.Ok, + ? ResultState.Error + : ResultState.Ok, stdout: event?.kind?.stdout, stderr: event?.kind?.stderr, message: event?.kind?.finished?.message, From bf83da03270b31b41a4a617515189766e59e07fd Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 17 Nov 2023 13:45:30 +0100 Subject: [PATCH 38/38] chore: formatting fix --- src/activity/activity.ts | 4 ++-- src/executor/executor.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/activity/activity.ts b/src/activity/activity.ts index 1e45234d1..40bddda9e 100644 --- a/src/activity/activity.ts +++ b/src/activity/activity.ts @@ -331,8 +331,8 @@ export class Activity { ? ResultState.Ok : ResultState.Error : event?.kind?.stderr - ? ResultState.Error - : ResultState.Ok, + ? ResultState.Error + : ResultState.Ok, stdout: event?.kind?.stdout, stderr: event?.kind?.stderr, message: event?.kind?.finished?.message, diff --git a/src/executor/executor.ts b/src/executor/executor.ts index 929ede016..47a09e617 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -486,8 +486,8 @@ export class TaskExecutor { proposalsCount.initial === 0 && proposalsCount.confirmed === 0 ? "Check your demand if it's not too restrictive or restart yagna." : proposalsCount.initial === proposalsCount.rejected - ? "All off proposals got rejected." - : "Check your proposal filters if they are not too restrictive."; + ? "All off proposals got rejected." + : "Check your proposal filters if they are not too restrictive."; const errorMessage = `Could not start any work on Golem. Processed ${proposalsCount.initial} initial proposals from yagna, filters accepted ${proposalsCount.confirmed}. ${hint}`; if (this.options.exitOnNoProposals) { this.handleCriticalError(new Error(errorMessage));