From 99d42ffdf5d6c98ceb702f42b9e1b3c7c5155fe9 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 18 Sep 2023 17:22:49 +0200 Subject: [PATCH 01/56] test(examples): docs-examples and test script draft --- .github/workflows/goth-examples.yml | 101 +++++++++++++++ .../examples/composing-tasks/batch-end.mjs | 24 ++++ .../batch-endstream-chunks.mjs | 22 ++++ .../batch-endstream-forawait.mjs | 24 ++++ .../composing-tasks/multiple-run-prosaic.mjs | 21 +++ .../composing-tasks/singleCommand.mjs | 13 ++ .../examples/composing-tasks/worker.mjs | 1 + .../examples/executing-tasks/beforeeach.mjs | 24 ++++ .../examples/executing-tasks/foreach.mjs | 16 +++ .../examples/executing-tasks/map.mjs | 16 +++ .../executing-tasks/maxparalleltasks.mjs | 17 +++ .../examples/executing-tasks/singlerun.mjs | 14 ++ .../selecting-providers/custom-price.mjs | 39 ++++++ .../examples/selecting-providers/demand.mjs | 15 +++ .../selecting-providers/whitelist.mjs | 24 ++++ .../examples/sending-data/downloadingfile.mjs | 23 ++++ .../examples/sending-data/uploadingJSON.mjs | 21 +++ .../examples/sending-data/uploadingfile.mjs | 28 ++++ .../switching-to-mainnet/runOnPolygon.mjs | 15 +++ .../transferring-data/downloadFile.mjs | 23 ++++ .../transferDatainBrowser.html | 122 ++++++++++++++++++ .../examples/transferring-data/uploadFile.mjs | 28 ++++ .../uploadHSONinBrowser.html | 75 +++++++++++ .../examples/transferring-data/uploadJSON.mjs | 21 +++ .../examples/using-app-keys/index.mjs | 12 ++ .../examples/working-with-images/hash.mjs | 14 ++ .../examples/working-with-images/tag.mjs | 14 ++ .../working-with-results/multicommand-end.mjs | 24 ++++ .../multicommand-endstream.mjs | 22 ++++ .../multicommand-fail.mjs | 23 ++++ .../single-command-fail.mjs | 15 +++ .../working-with-results/single-command.mjs | 14 ++ .../quickstarts/quickstart/requestor.mjs | 11 ++ .../quickstarts/web-quickstart/index.html | 31 +++++ .../quickstarts/web-quickstart/requestor.mjs | 40 ++++++ .../converting-Docker-image/Dockerfile | 3 + .../building-custom-image/Dockerfile | 5 + .../tutorials/building-custom-image/index.mjs | 14 ++ .../tutorials/quickstart/index.mjs | 14 ++ .../tutorials/running-from-browser/index.html | 68 ++++++++++ .../running-parallel-tasks/index.mjs | 57 ++++++++ examples/test.mjs | 47 +++++++ package.json | 1 + 43 files changed, 1156 insertions(+) create mode 100644 .github/workflows/goth-examples.yml create mode 100644 examples/docs-examples/examples/composing-tasks/batch-end.mjs create mode 100644 examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs create mode 100644 examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs create mode 100644 examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs create mode 100644 examples/docs-examples/examples/composing-tasks/singleCommand.mjs create mode 100644 examples/docs-examples/examples/composing-tasks/worker.mjs create mode 100644 examples/docs-examples/examples/executing-tasks/beforeeach.mjs create mode 100644 examples/docs-examples/examples/executing-tasks/foreach.mjs create mode 100644 examples/docs-examples/examples/executing-tasks/map.mjs create mode 100644 examples/docs-examples/examples/executing-tasks/maxparalleltasks.mjs create mode 100644 examples/docs-examples/examples/executing-tasks/singlerun.mjs create mode 100644 examples/docs-examples/examples/selecting-providers/custom-price.mjs create mode 100644 examples/docs-examples/examples/selecting-providers/demand.mjs create mode 100644 examples/docs-examples/examples/selecting-providers/whitelist.mjs create mode 100644 examples/docs-examples/examples/sending-data/downloadingfile.mjs create mode 100644 examples/docs-examples/examples/sending-data/uploadingJSON.mjs create mode 100644 examples/docs-examples/examples/sending-data/uploadingfile.mjs create mode 100644 examples/docs-examples/examples/switching-to-mainnet/runOnPolygon.mjs create mode 100644 examples/docs-examples/examples/transferring-data/downloadFile.mjs create mode 100644 examples/docs-examples/examples/transferring-data/transferDatainBrowser.html create mode 100644 examples/docs-examples/examples/transferring-data/uploadFile.mjs create mode 100644 examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html create mode 100644 examples/docs-examples/examples/transferring-data/uploadJSON.mjs create mode 100644 examples/docs-examples/examples/using-app-keys/index.mjs create mode 100644 examples/docs-examples/examples/working-with-images/hash.mjs create mode 100644 examples/docs-examples/examples/working-with-images/tag.mjs create mode 100644 examples/docs-examples/examples/working-with-results/multicommand-end.mjs create mode 100644 examples/docs-examples/examples/working-with-results/multicommand-endstream.mjs create mode 100644 examples/docs-examples/examples/working-with-results/multicommand-fail.mjs create mode 100644 examples/docs-examples/examples/working-with-results/single-command-fail.mjs create mode 100644 examples/docs-examples/examples/working-with-results/single-command.mjs create mode 100644 examples/docs-examples/quickstarts/quickstart/requestor.mjs create mode 100644 examples/docs-examples/quickstarts/web-quickstart/index.html create mode 100644 examples/docs-examples/quickstarts/web-quickstart/requestor.mjs create mode 100644 examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile create mode 100644 examples/docs-examples/tutorials/building-custom-image/Dockerfile create mode 100644 examples/docs-examples/tutorials/building-custom-image/index.mjs create mode 100644 examples/docs-examples/tutorials/quickstart/index.mjs create mode 100644 examples/docs-examples/tutorials/running-from-browser/index.html create mode 100644 examples/docs-examples/tutorials/running-parallel-tasks/index.mjs create mode 100644 examples/test.mjs diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml new file mode 100644 index 000000000..d9208b5fe --- /dev/null +++ b/.github/workflows/goth-examples.yml @@ -0,0 +1,101 @@ +name: Nightly Examples Tests + +on: + schedule: + # run this workflow every day at 2:00 AM UTC + - cron: "0 3 * * *" + + # Allows triggering the workflow manually + workflow_dispatch: + +jobs: + prepare-matrix-master-only: + name: Prepare matrix JSON + runs-on: ubuntu-latest + outputs: + matrix-json: ${{ steps.get-matrix.outputs.matrix }} + steps: + # prepares JSON object representing strategy matrix with only the master branch + - name: Get matrix JSON + id: get-matrix + run: echo "::set-output name=matrix::{\"include\":[{\"branch\":\"master\"}]}" + + goth-tests: + runs-on: goth + needs: prepare-matrix-master-only + strategy: + matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }} + fail-fast: false + name: Run integration tests (nightly) on ${{ matrix.branch }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Build golem-js + run: | + sudo apt-get update -y + sudo apt-get install -y build-essential + npm install + npm run build + + - name: Configure python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install goth + run: | + pip install goth + rm -rf ../goth/assets + python -m goth create-assets ../goth/assets + sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml + sed -i '/^ router:/a \ \ \ \ \ \ \ \ environment:\n \ \ \ \ \ \ \ \ \ \ - YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/docker-compose.yml + + - name: Install websocat and sshpass + run: | + sudo wget https://github.com/vi/websocat/releases/download/v1.9.0/websocat_linux64 -O /usr/local/bin/websocat + sudo chmod +x /usr/local/bin/websocat + sudo apt-get install sshpass + + - name: Disconnect Docker containers from default network + continue-on-error: true + # related to this issue: https://github.com/moby/moby/issues/23302 + run: | + docker network inspect docker_default + sudo apt-get install -y jq + docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default + + - name: Remove Docker containers + continue-on-error: true + run: docker rm -f $(docker ps -a -q) + + - name: Restart Docker daemon + # related to this issue: https://github.com/moby/moby/issues/23302 + run: sudo systemctl restart docker + + - name: Log in to GitHub Docker repository + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin + + - name: Run test suite + env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd examples + node test.mjs + + - name: Upload test logs + uses: actions/upload-artifact@v2 + if: always() + with: + name: goth-logs + path: /tmp/goth-tests + + # Only relevant for self-hosted runners + - name: Remove test logs + if: always() + run: rm -rf /tmp/goth-tests diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs new file mode 100644 index 000000000..5ab69c1d2 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker2.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); // to be removed and replaced with try & catch + + return res[2]?.stdout; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs new file mode 100644 index 000000000..83db435de --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs @@ -0,0 +1,22 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + res.on("data", (data) => (data.index == 2 ? console.log(data.stdout) : "")); + res.on("error", (error) => console.error(error)); + res.on("close", () => executor.end()); + }); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs new file mode 100644 index 000000000..6f870819f --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + for await (const chunk of res) { + chunk.index == 2 ? console.log(chunk.stdout) : ""; + } + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs new file mode 100644 index 000000000..275ec7256 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs @@ -0,0 +1,21 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + isSubprocess: true, + }); + + const result = await executor.run(async (ctx) => { + await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs"); + await ctx.run("node /golem/input/worker.mjs > /golem/input/output.txt"); + const result = await ctx.run("cat /golem/input/output.txt"); + await ctx.downloadFile("/golem/input/output.txt", "./output.txt"); + return result.stdout; + }); + + console.log(result); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/singleCommand.mjs b/examples/docs-examples/examples/composing-tasks/singleCommand.mjs new file mode 100644 index 000000000..63717003c --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/singleCommand.mjs @@ -0,0 +1,13 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/worker.mjs b/examples/docs-examples/examples/composing-tasks/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/examples/docs-examples/examples/executing-tasks/beforeeach.mjs b/examples/docs-examples/examples/executing-tasks/beforeeach.mjs new file mode 100644 index 000000000..31913ed4c --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/beforeeach.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + maxParallelTasks: 3, + }); + + executor.beforeEach(async (ctx) => { + console.log(ctx.provider.name + " is downloading action_log file"); + await ctx.uploadFile("./action_log.txt", "/golem/input/action_log.txt"); + }); + + await executor.forEach([1, 2, 3, 4, 5], async (ctx, item) => { + await ctx + .beginBatch() + .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`) + .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt") + .end(); + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/foreach.mjs b/examples/docs-examples/examples/executing-tasks/foreach.mjs new file mode 100644 index 000000000..e1cbd7fc1 --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/foreach.mjs @@ -0,0 +1,16 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const data = [1, 2, 3, 4, 5]; + + await executor.forEach(data, async (ctx, item) => { + console.log((await ctx.run(`echo "${item}"`)).stdout); + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/map.mjs b/examples/docs-examples/examples/executing-tasks/map.mjs new file mode 100644 index 000000000..9fc089987 --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/map.mjs @@ -0,0 +1,16 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const data = [1, 2, 3, 4, 5]; + + const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); + + for await (const result of results) console.log(result.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/maxparalleltasks.mjs b/examples/docs-examples/examples/executing-tasks/maxparalleltasks.mjs new file mode 100644 index 000000000..0e1a11572 --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/maxparalleltasks.mjs @@ -0,0 +1,17 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + maxParallelTasks: 3, // default is 5 + }); + + const data = [1, 2, 3, 4, 5]; + + const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); + + for await (const result of results) console.log(result.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/singlerun.mjs b/examples/docs-examples/examples/executing-tasks/singlerun.mjs new file mode 100644 index 000000000..1f831abee --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/singlerun.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/custom-price.mjs b/examples/docs-examples/examples/selecting-providers/custom-price.mjs new file mode 100644 index 000000000..ce6db0f37 --- /dev/null +++ b/examples/docs-examples/examples/selecting-providers/custom-price.mjs @@ -0,0 +1,39 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +/** + * Example demonstrating how to write a custom proposal filter. + */ + +var costData = []; + +const myFilter = async (proposal) => { + let decision = false; + let usageVector = proposal.properties["golem.com.usage.vector"]; + let counterIdx = usageVector.findIndex((ele) => ele === "golem.usage.duration_sec"); + let proposedCost = proposal.properties["golem.com.pricing.model.linear.coeffs"][counterIdx]; + costData.push(proposedCost); + if (costData.length < 11) return false; + else { + costData.shift(); + let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 10; + if (proposedCost <= averageProposedCost) decision = true; + if (decision) { + console.log(proposedCost, averageProposedCost); + } + } + console.log(costData); + console.log(proposal.properties["golem.node.id.name"], proposal.properties["golem.com.pricing.model.linear.coeffs"]); + return decision; +}; + +(async function main() { + const executor = await TaskExecutor.create({ + package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + proposalFilter: myFilter, + yagnaOptions: { apiKey: "try_golem" }, + }); + await executor.run(async (ctx) => + console.log((await ctx.run(`echo "This task is run on ${ctx.provider.id}"`)).stdout, ctx.provider.id), + ); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/demand.mjs b/examples/docs-examples/examples/selecting-providers/demand.mjs new file mode 100644 index 000000000..acd734f83 --- /dev/null +++ b/examples/docs-examples/examples/selecting-providers/demand.mjs @@ -0,0 +1,15 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async function main() { + const executor = await TaskExecutor.create({ + package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + //minCpuCores : 2, + //minMemGib : 8, + //minStorageGib: 10, + minCpuThreads: 1, + yagnaOptions: { apiKey: "try_golem" }, + }); + + await executor.run(async (ctx) => console.log((await ctx.run("echo 'Hello World'")).stdout)); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/whitelist.mjs b/examples/docs-examples/examples/selecting-providers/whitelist.mjs new file mode 100644 index 000000000..202c8c454 --- /dev/null +++ b/examples/docs-examples/examples/selecting-providers/whitelist.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor, ProposalFilters } from "@golem-sdk/golem-js"; + +/** + * Example demonstrating how to use the predefined filter `whiteListProposalIdsFilter`, + * which only allows offers from a provider whose ID is in the array + */ + +const whiteListIds = ["0x3fc1d65ddc5258dc8807df30a29f71028e965e1b", "0x4506550de84d207f3ab90add6336f68119015836"]; +console.log("Will accept only proposals from:"); +for (let i = 0; i < whiteListIds.length; i++) { + console.log(whiteListIds[i]); +} + +(async function main() { + const executor = await TaskExecutor.create({ + package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + proposalFilter: ProposalFilters.whiteListProposalIdsFilter(whiteListIds), + yagnaOptions: { apiKey: "try_golem" }, + }); + await executor.run(async (ctx) => + console.log((await ctx.run(`echo "This task is run on ${ctx.provider.id}"`)).stdout, ctx.provider.id), + ); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/downloadingfile.mjs b/examples/docs-examples/examples/sending-data/downloadingfile.mjs new file mode 100644 index 000000000..2a44bc080 --- /dev/null +++ b/examples/docs-examples/examples/sending-data/downloadingfile.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .run("ls -l /golem > /golem/work/output.txt") + .run("cat /golem/work/output.txt") + .downloadFile("/golem/work/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); + + return res[2]?.stdout; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/uploadingJSON.mjs b/examples/docs-examples/examples/sending-data/uploadingJSON.mjs new file mode 100644 index 000000000..5b107da72 --- /dev/null +++ b/examples/docs-examples/examples/sending-data/uploadingJSON.mjs @@ -0,0 +1,21 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const output = await executor.run(async (ctx) => { + // Upload test JSON object + + await ctx.uploadJson({ input: "Hello World" }, "/golem/work/input.json"); + + // Read the content of the JSON object. + return await ctx.run("cat /golem/work/input.json"); + }); + + console.log(output.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/uploadingfile.mjs b/examples/docs-examples/examples/sending-data/uploadingfile.mjs new file mode 100644 index 000000000..dadc3540a --- /dev/null +++ b/examples/docs-examples/examples/sending-data/uploadingfile.mjs @@ -0,0 +1,28 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { createHash } from "node:crypto"; +import * as fs from "fs"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const buff = fs.readFileSync("worker.mjs"); + const hash = createHash("md5").update(buff).digest("hex"); + + const result = await executor.run(async (ctx) => { + await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs"); + + const res = await ctx.run( + `node -e "const crypto = require('node:crypto'); const fs = require('fs'); const buff = fs.readFileSync('/golem/input/worker.mjs'); const hash = crypto.createHash('md5').update(buff).digest('hex'); console.log(hash); "`, + ); + + return res.stdout; + }); + + console.log("md5 of the file sent to provider: ", result); + console.log("Locally computed md5: ", hash); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/switching-to-mainnet/runOnPolygon.mjs b/examples/docs-examples/examples/switching-to-mainnet/runOnPolygon.mjs new file mode 100644 index 000000000..3c8a2ab70 --- /dev/null +++ b/examples/docs-examples/examples/switching-to-mainnet/runOnPolygon.mjs @@ -0,0 +1,15 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + subnetTag: "public", // do we need to show subnet ?? + payment: { driver: "erc-20", network: "polygon" }, + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/transferring-data/downloadFile.mjs b/examples/docs-examples/examples/transferring-data/downloadFile.mjs new file mode 100644 index 000000000..2a44bc080 --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/downloadFile.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .run("ls -l /golem > /golem/work/output.txt") + .run("cat /golem/work/output.txt") + .downloadFile("/golem/work/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); + + return res[2]?.stdout; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/transferring-data/transferDatainBrowser.html b/examples/docs-examples/examples/transferring-data/transferDatainBrowser.html new file mode 100644 index 000000000..8962ef6c9 --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/transferDatainBrowser.html @@ -0,0 +1,122 @@ + + + + + WebRequestor Task API + + +

WebRequestor - Meme Example

+
+
+

Input data

+
+
+ + +
+
+ + +
+
+

Actions

+
+
+ +
+
+
+

Result Meme

+ +
+
+
+
+

Logs

+
    +
    +
    +
    + + + + diff --git a/examples/docs-examples/examples/transferring-data/uploadFile.mjs b/examples/docs-examples/examples/transferring-data/uploadFile.mjs new file mode 100644 index 000000000..dadc3540a --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/uploadFile.mjs @@ -0,0 +1,28 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { createHash } from "node:crypto"; +import * as fs from "fs"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const buff = fs.readFileSync("worker.mjs"); + const hash = createHash("md5").update(buff).digest("hex"); + + const result = await executor.run(async (ctx) => { + await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs"); + + const res = await ctx.run( + `node -e "const crypto = require('node:crypto'); const fs = require('fs'); const buff = fs.readFileSync('/golem/input/worker.mjs'); const hash = crypto.createHash('md5').update(buff).digest('hex'); console.log(hash); "`, + ); + + return res.stdout; + }); + + console.log("md5 of the file sent to provider: ", result); + console.log("Locally computed md5: ", hash); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html b/examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html new file mode 100644 index 000000000..3def7232f --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html @@ -0,0 +1,75 @@ + + + + + Golem JSON App + + + +

    JSON upload and download

    +
    +
    +

    Actions

    +
    +
    + +
    +
    +
    +

    Results

    +
      +
      +
      +
      +
      +

      Logs

      +
        +
        +
        +
        + + diff --git a/examples/docs-examples/examples/transferring-data/uploadJSON.mjs b/examples/docs-examples/examples/transferring-data/uploadJSON.mjs new file mode 100644 index 000000000..5b107da72 --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/uploadJSON.mjs @@ -0,0 +1,21 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const output = await executor.run(async (ctx) => { + // Upload test JSON object + + await ctx.uploadJson({ input: "Hello World" }, "/golem/work/input.json"); + + // Read the content of the JSON object. + return await ctx.run("cat /golem/work/input.json"); + }); + + console.log(output.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/using-app-keys/index.mjs b/examples/docs-examples/examples/using-app-keys/index.mjs new file mode 100644 index 000000000..18cc40b23 --- /dev/null +++ b/examples/docs-examples/examples/using-app-keys/index.mjs @@ -0,0 +1,12 @@ +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + // replace 'try_golem' with 'insert-your-32-char-app-key-here' + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/working-with-images/hash.mjs b/examples/docs-examples/examples/working-with-images/hash.mjs new file mode 100644 index 000000000..1f831abee --- /dev/null +++ b/examples/docs-examples/examples/working-with-images/hash.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/working-with-images/tag.mjs b/examples/docs-examples/examples/working-with-images/tag.mjs new file mode 100644 index 000000000..4a14209fa --- /dev/null +++ b/examples/docs-examples/examples/working-with-images/tag.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "golem/my_example:latest", + yagnaOptions: { appKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/working-with-results/multicommand-end.mjs b/examples/docs-examples/examples/working-with-results/multicommand-end.mjs new file mode 100644 index 000000000..46e7fea74 --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/multicommand-end.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); + + return res; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/multicommand-endstream.mjs b/examples/docs-examples/examples/working-with-results/multicommand-endstream.mjs new file mode 100644 index 000000000..9d95b9474 --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/multicommand-endstream.mjs @@ -0,0 +1,22 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + res.on("data", (result) => console.log(result)); + res.on("error", (error) => console.error(error)); + res.on("close", () => executor.end()); + }); +})(); diff --git a/examples/docs-examples/examples/working-with-results/multicommand-fail.mjs b/examples/docs-examples/examples/working-with-results/multicommand-fail.mjs new file mode 100644 index 000000000..e2633ba6d --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/multicommand-fail.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .run("cat /golem/input/output.txt > /golem/input/output.txt") + .downloadFile("/golem/output/output.txt", "./output.txt") // there is no such file in output folder + .run("ls -l /golem/") + .end() + .catch((error) => console.error(error)); + + return res; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/single-command-fail.mjs b/examples/docs-examples/examples/working-with-results/single-command-fail.mjs new file mode 100644 index 000000000..e4a92201c --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/single-command-fail.mjs @@ -0,0 +1,15 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + isSubprocess: true, + }); + + // there is a mistake and instead of 'node -v' we call 'node -w' + const result = await executor.run(async (ctx) => await ctx.run("node -w")); + console.log("Task result:", result); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/single-command.mjs b/examples/docs-examples/examples/working-with-results/single-command.mjs new file mode 100644 index 000000000..1075c8dba --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/single-command.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + isSubprocess: true, + }); + + const result = await executor.run(async (ctx) => await ctx.run("node -v")); + console.log("Task result:", result); + + await executor.end(); +})(); diff --git a/examples/docs-examples/quickstarts/quickstart/requestor.mjs b/examples/docs-examples/quickstarts/quickstart/requestor.mjs new file mode 100644 index 000000000..92fe00e9b --- /dev/null +++ b/examples/docs-examples/quickstarts/quickstart/requestor.mjs @@ -0,0 +1,11 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/quickstarts/web-quickstart/index.html b/examples/docs-examples/quickstarts/web-quickstart/index.html new file mode 100644 index 000000000..2b899c237 --- /dev/null +++ b/examples/docs-examples/quickstarts/web-quickstart/index.html @@ -0,0 +1,31 @@ + + + + + WebRequestor QuickStart + + +

        WebRequestor - QuickStart

        +
        +
        +

        Actions

        +
        +
        + +
        +
        +
        +

        Results

        +
          +
          +
          +
          +
          +

          Logs

          +
            +
            +
            +
            + + + diff --git a/examples/docs-examples/quickstarts/web-quickstart/requestor.mjs b/examples/docs-examples/quickstarts/web-quickstart/requestor.mjs new file mode 100644 index 000000000..c9716d7c1 --- /dev/null +++ b/examples/docs-examples/quickstarts/web-quickstart/requestor.mjs @@ -0,0 +1,40 @@ +import * as golem from "https://unpkg.com/@golem-sdk/golem-js"; + +function appendResults(result) { + const results = document.getElementById("results"); + const div = document.createElement("div"); + div.appendChild(document.createTextNode(result)); + results.appendChild(div); +} + +function appendLog(msg, level = "info") { + const logs = document.getElementById("logs"); + const div = document.createElement("div"); + div.appendChild(document.createTextNode(`[${new Date().toISOString()}] [${level}] ${msg}`)); + logs.appendChild(div); +} + +const logger = { + log: (msg) => appendLog(msg), + warn: (msg) => appendLog(msg, "warn"), + debug: (msg) => appendLog(msg, "debug"), + error: (msg) => appendLog(msg, "error"), + info: (msg) => appendLog(msg, "info"), + table: (msg) => appendLog(JSON.stringify(msg, null, "\t")), +}; + +async function run() { + const executor = await golem.TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + logger, + }).catch((e) => logger.error(e)); + + await executor + .run(async (ctx) => appendResults((await ctx.run("echo 'Hello World'")).stdout)) + .catch((e) => logger.error(e)); + + await executor.end(); +} + +document.getElementById("echo").onclick = run; diff --git a/examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile b/examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile new file mode 100644 index 000000000..d0de19d0f --- /dev/null +++ b/examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile @@ -0,0 +1,3 @@ +FROM debian:latest +VOLUME /golem/input /golem/output +WORKDIR /golem/work diff --git a/examples/docs-examples/tutorials/building-custom-image/Dockerfile b/examples/docs-examples/tutorials/building-custom-image/Dockerfile new file mode 100644 index 000000000..d29ee22e0 --- /dev/null +++ b/examples/docs-examples/tutorials/building-custom-image/Dockerfile @@ -0,0 +1,5 @@ +FROM node:latest +WORKDIR /golem/work +VOLUME /golem/work +COPY Dockerfile /golem/info/description.txt +COPY Dockerfile /golem/work/info.txt \ No newline at end of file diff --git a/examples/docs-examples/tutorials/building-custom-image/index.mjs b/examples/docs-examples/tutorials/building-custom-image/index.mjs new file mode 100644 index 000000000..b75048699 --- /dev/null +++ b/examples/docs-examples/tutorials/building-custom-image/index.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +(async () => { + const executor = await TaskExecutor.create({ + package: "8b238595299444d0733b41095f27fadd819a71d29002b614c665b27c", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + console.log("Description.txt: ", (await ctx.run("cat /golem/info/description.txt")).stdout); + console.log("/golem/work content: ", (await ctx.run("ls /golem/work")).stdout); + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/tutorials/quickstart/index.mjs b/examples/docs-examples/tutorials/quickstart/index.mjs new file mode 100644 index 000000000..5e4fcebe6 --- /dev/null +++ b/examples/docs-examples/tutorials/quickstart/index.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const taskResult = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", taskResult); +})(); diff --git a/examples/docs-examples/tutorials/running-from-browser/index.html b/examples/docs-examples/tutorials/running-from-browser/index.html new file mode 100644 index 000000000..55bfa0485 --- /dev/null +++ b/examples/docs-examples/tutorials/running-from-browser/index.html @@ -0,0 +1,68 @@ + + + + + WebRequestor Task API + + +

            WebRequestor - Hello World

            +
            +
            +

            Actions

            +
            +
            + +
            +
            +
            +

            Results

            +
              +
              +
              +
              +
              +

              Logs

              +
                +
                +
                +
                + + + + diff --git a/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs new file mode 100644 index 000000000..6342bf373 --- /dev/null +++ b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs @@ -0,0 +1,57 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { program } from "commander"; + +async function main(args) { + const executor = await TaskExecutor.create({ + package: "055911c811e56da4d75ffc928361a78ed13077933ffa8320fb1ec2db", + maxParallelTasks: args.numberOfProviders, + yagnaOptions: { apiKey: `try_golem` }, + }); + + const keyspace = await executor.run(async (ctx) => { + const result = await ctx.run(`hashcat --keyspace -a 3 ${args.mask} -m 400`); + return parseInt(result.stdout || ""); + }); + + if (!keyspace) throw new Error(`Cannot calculate keyspace`); + + console.log(`Keyspace size computed. Keyspace size = ${keyspace}.`); + const step = Math.floor(keyspace / args.numberOfProviders + 1); + const range = [...Array(Math.floor(keyspace / step) + 1).keys()].map((i) => i * step); + + const results = executor.map(range, async (ctx, skip = 0) => { + const results = await ctx + .beginBatch() + .run( + `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min( + keyspace, + skip + step, + )} -o pass.potfile`, + ) + .run("cat pass.potfile") + .end() + .catch((err) => console.error(err)); + if (!results?.[1]?.stdout) return false; + return results?.[1]?.stdout.toString().split(":")[1]; + }); + + let password = ""; + for await (const result of results) { + if (result) { + password = result; + break; + } + } + + if (!password) console.log("No password found"); + else console.log(`Password found: ${password}`); + await executor.end(); +} + +program + .option("--number-of-providers ", "number of providers", (value) => parseInt(value), 3) + .option("--mask ") + .requiredOption("--hash "); +program.parse(); +const options = program.opts(); +main(options).catch((e) => console.error(e)); diff --git a/examples/test.mjs b/examples/test.mjs new file mode 100644 index 000000000..7f75a4e68 --- /dev/null +++ b/examples/test.mjs @@ -0,0 +1,47 @@ +import { spawn } from "child_process"; +import { dirname, basename } from "path"; + +const examples = [ + { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-end.mjs" }, + { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, + { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, + { cmd: "node", path: "docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, + { cmd: "node", path: "docs-examples/examples/composing-tasks/singleCommand.mjs" }, +]; + +const criticalLogsRegexp = [/Task timeot/, /Task rejected/, /TODO/]; + +const timeoutPromise = (seconds) => + new Promise((res, rej) => + setTimeout( + () => rej(new Error(`The timeout was reached and the racing promise has rejected after ${seconds} seconds`)), + seconds * 1000, + ), + ); + +async function test(cmd, path, args = [], timeout = 120) { + const file = basename(path); + const cwd = dirname(path); + const spawnedExample = spawn(cmd, [file, ...args], { stdio: "inherit", cwd }); + const testPromise = new Promise((res, rej) => { + spawnedExample.on("close", (code, signal) => { + if (code === 0) return res(); + rej(`Example test exited with code ${code} by signal ${signal}`); + }); + }); + return Promise.race([timeoutPromise(timeout), testPromise]); +} + +async function testAll(examples) { + try { + for (const example of examples) { + console.log(`Starting test for example ${example.path}`); + await test(example.cmd, example.path, example.args); + } + } catch (error) { + console.error(error); + process.exit(1); + } + process.exit(0); +} +await testAll(examples); diff --git a/package.json b/package.json index e6003ea51..8cd5ffefe 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "test:e2e": "jest --config tests/e2e/jest.config.json tests/e2e/**.spec.ts --runInBand --forceExit", "test:e2e:no-goth": "jest tests/e2e/**.spec.ts --testTimeout=180000 --runInBand --forceExit", "test:cypress": "cypress run", + "test:examples": "node examples/test.mjs", "lint": "npm run lint:ts && npm run lint:ts:tests && npm run lint:eslint", "lint:ts": "tsc --project tsconfig.json --noEmit", "lint:ts:tests": "tsc --project tests/tsconfig.json --noEmit", From 4a299d645e2a8957c18d8c438ffea76f634385a7 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 18 Sep 2023 17:28:25 +0200 Subject: [PATCH 02/56] test(examples): docs-examples and test script draft --- .github/workflows/goth-examples.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index d9208b5fe..e45e2626c 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -1,9 +1,9 @@ name: Nightly Examples Tests on: - schedule: - # run this workflow every day at 2:00 AM UTC - - cron: "0 3 * * *" + push: + branches: + - "test/JST-363/example-tests" # Allows triggering the workflow manually workflow_dispatch: @@ -26,7 +26,7 @@ jobs: strategy: matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }} fail-fast: false - name: Run integration tests (nightly) on ${{ matrix.branch }} + name: Run example tests (nightly) on ${{ matrix.branch }} steps: - name: Checkout uses: actions/checkout@v3 From 416bf53a1dbd6d38316ddfb15966132ee9804315 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 18 Sep 2023 17:41:47 +0200 Subject: [PATCH 03/56] test(examples): goth ci debug --- .github/workflows/goth-examples.yml | 3 +-- examples/test.mjs => tests/examples/test.ts | 15 +++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) rename examples/test.mjs => tests/examples/test.ts (77%) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index e45e2626c..fac63739b 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -85,8 +85,7 @@ jobs: env: GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - cd examples - node test.mjs + python -m goth start ../goth/assets/goth-config.yml - name: Upload test logs uses: actions/upload-artifact@v2 diff --git a/examples/test.mjs b/tests/examples/test.ts similarity index 77% rename from examples/test.mjs rename to tests/examples/test.ts index 7f75a4e68..127999c41 100644 --- a/examples/test.mjs +++ b/tests/examples/test.ts @@ -1,8 +1,12 @@ import { spawn } from "child_process"; -import { dirname, basename } from "path"; +import { dirname, basename, resolve } from "path"; +import { Goth } from "../goth/goth"; + +const gothConfig = resolve("../goth/assets/goth-config.yml"); +const goth = new Goth(gothConfig); const examples = [ - { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-end.mjs" }, + { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, @@ -25,7 +29,7 @@ async function test(cmd, path, args = [], timeout = 120) { const spawnedExample = spawn(cmd, [file, ...args], { stdio: "inherit", cwd }); const testPromise = new Promise((res, rej) => { spawnedExample.on("close", (code, signal) => { - if (code === 0) return res(); + if (code === 0) return res(true); rej(`Example test exited with code ${code} by signal ${signal}`); }); }); @@ -33,6 +37,8 @@ async function test(cmd, path, args = [], timeout = 120) { } async function testAll(examples) { + const { apiKey } = await goth.start(); + console.log(`Goth started with APIKEY:`, apiKey); try { for (const example of examples) { console.log(`Starting test for example ${example.path}`); @@ -44,4 +50,5 @@ async function testAll(examples) { } process.exit(0); } -await testAll(examples); + +testAll(examples).then(); From 3e427439393ac82a0739c4701b07cafbafe0203a Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 18 Sep 2023 17:48:01 +0200 Subject: [PATCH 04/56] test(examples): goth ci debug --- .github/workflows/goth-examples.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index fac63739b..5d6aa642a 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -54,6 +54,7 @@ jobs: rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml + sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml sed -i '/^ router:/a \ \ \ \ \ \ \ \ environment:\n \ \ \ \ \ \ \ \ \ \ - YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/docker-compose.yml - name: Install websocat and sshpass From 79cf8c18a8d53cfd671fa3c87dc7e2a3df2b9aed Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 12:49:14 +0200 Subject: [PATCH 05/56] test(examples): goth ci debug --- .github/workflows/goth-examples.yml | 3 ++- package.json | 2 +- tests/examples/{test.ts => examples.test.ts} | 22 +++++++++++++------- tsconfig.spec.json | 8 ++++++- 4 files changed, 24 insertions(+), 11 deletions(-) rename tests/examples/{test.ts => examples.test.ts} (75%) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index 5d6aa642a..bc1fb686f 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -42,6 +42,7 @@ jobs: sudo apt-get install -y build-essential npm install npm run build + npm install ts-node - name: Configure python uses: actions/setup-python@v4 @@ -86,7 +87,7 @@ jobs: env: GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - python -m goth start ../goth/assets/goth-config.yml + npm run test:examples - name: Upload test logs uses: actions/upload-artifact@v2 diff --git a/package.json b/package.json index 8cd5ffefe..da961bc73 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:e2e": "jest --config tests/e2e/jest.config.json tests/e2e/**.spec.ts --runInBand --forceExit", "test:e2e:no-goth": "jest tests/e2e/**.spec.ts --testTimeout=180000 --runInBand --forceExit", "test:cypress": "cypress run", - "test:examples": "node examples/test.mjs", + "test:examples": "ts-node --project ./tsconfig.spec.json tests/examples/examples.test.ts", "lint": "npm run lint:ts && npm run lint:ts:tests && npm run lint:eslint", "lint:ts": "tsc --project tsconfig.json --noEmit", "lint:ts:tests": "tsc --project tests/tsconfig.json --noEmit", diff --git a/tests/examples/test.ts b/tests/examples/examples.test.ts similarity index 75% rename from tests/examples/test.ts rename to tests/examples/examples.test.ts index 127999c41..ac1d46bc0 100644 --- a/tests/examples/test.ts +++ b/tests/examples/examples.test.ts @@ -5,7 +5,14 @@ import { Goth } from "../goth/goth"; const gothConfig = resolve("../goth/assets/goth-config.yml"); const goth = new Goth(gothConfig); -const examples = [ +type Example = { + cmd: string; + path: string; + args?: string[]; + timeout?: number; +}; + +const examples: Example[] = [ { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, @@ -15,7 +22,7 @@ const examples = [ const criticalLogsRegexp = [/Task timeot/, /Task rejected/, /TODO/]; -const timeoutPromise = (seconds) => +const timeoutPromise = (seconds: number) => new Promise((res, rej) => setTimeout( () => rej(new Error(`The timeout was reached and the racing promise has rejected after ${seconds} seconds`)), @@ -23,26 +30,25 @@ const timeoutPromise = (seconds) => ), ); -async function test(cmd, path, args = [], timeout = 120) { +async function examplesTest(cmd: string, path: string, args: string[] = [], timeout = 120) { const file = basename(path); const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { stdio: "inherit", cwd }); const testPromise = new Promise((res, rej) => { spawnedExample.on("close", (code, signal) => { if (code === 0) return res(true); - rej(`Example test exited with code ${code} by signal ${signal}`); + rej(`Example test "${file}" exited with code ${code} by signal ${signal}`); }); }); return Promise.race([timeoutPromise(timeout), testPromise]); } -async function testAll(examples) { - const { apiKey } = await goth.start(); - console.log(`Goth started with APIKEY:`, apiKey); +async function testAll(examples: Example[]) { + await Promise.race([goth.start(), timeoutPromise(180)]); try { for (const example of examples) { console.log(`Starting test for example ${example.path}`); - await test(example.cmd, example.path, example.args); + await examplesTest(example.cmd, example.path, example.args); } } catch (error) { console.error(error); diff --git a/tsconfig.spec.json b/tsconfig.spec.json index 3d6abe8a5..ce95966b4 100755 --- a/tsconfig.spec.json +++ b/tsconfig.spec.json @@ -18,5 +18,11 @@ "typeRoots": ["node_modules/@types"] }, "exclude": ["dist", "node_modules", "examples", "cypress.config.ts"], - "include": ["src/**/*.spec.ts", "tests/unit/**/*.test.ts", "tests/unit/**/*.spec.ts"] + "include": ["src/**/*.spec.ts", "tests/unit/**/*.test.ts", "tests/unit/**/*.spec.ts"], + "ts-node": { + "esm": true, + "compilerOptions": { + "module": "nodenext" + } + } } From 9ca29d4c826a078e8a8dc466a8db30b53ad662ed Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 12:53:47 +0200 Subject: [PATCH 06/56] test(examples): goth ci debug --- .github/workflows/goth-examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index bc1fb686f..4faf7d4d4 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -42,6 +42,7 @@ jobs: sudo apt-get install -y build-essential npm install npm run build + npm install --prefix examples npm install ts-node - name: Configure python @@ -55,7 +56,6 @@ jobs: rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml sed -i '/^ router:/a \ \ \ \ \ \ \ \ environment:\n \ \ \ \ \ \ \ \ \ \ - YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/docker-compose.yml - name: Install websocat and sshpass From cba57b2b64d28a6d0396321fb3d34406b195f36c Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 14:58:09 +0200 Subject: [PATCH 07/56] test(examples): goth ci debug --- .github/workflows/goth-examples.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index 4faf7d4d4..63066da05 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -56,7 +56,7 @@ jobs: rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -i '/^ router:/a \ \ \ \ \ \ \ \ environment:\n \ \ \ \ \ \ \ \ \ \ - YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/docker-compose.yml + sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile - name: Install websocat and sshpass run: | From 0bd72029a16188a9c1318827193a60ed7e6949c4 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:01:21 +0200 Subject: [PATCH 08/56] test(examples): goth ci debug --- tests/examples/examples.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index ac1d46bc0..9108c64de 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -11,7 +11,7 @@ type Example = { args?: string[]; timeout?: number; }; - +// examples const examples: Example[] = [ { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, From 330ae8b5097d3ea412c6ae69ab8f639e8cdf8b36 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:05:07 +0200 Subject: [PATCH 09/56] test(examples): goth ci debug --- .github/workflows/goth-examples.yml | 8 ++++---- tests/examples/examples.test.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index 63066da05..b14f46a8c 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -1,13 +1,13 @@ name: Nightly Examples Tests on: - push: - branches: - - "test/JST-363/example-tests" - # Allows triggering the workflow manually workflow_dispatch: + push: + branches: + - "test/JST-363/*" + jobs: prepare-matrix-master-only: name: Prepare matrix JSON diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 9108c64de..ac1d46bc0 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -11,7 +11,7 @@ type Example = { args?: string[]; timeout?: number; }; -// examples + const examples: Example[] = [ { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, From 60805a7c4a3c0b0c3e45a67dfd19f002aa67164e Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:11:36 +0200 Subject: [PATCH 10/56] test(examples): goth ci debug 2 --- .github/workflows/goth-examples.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-examples.yml index b14f46a8c..44928c513 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-examples.yml @@ -58,12 +58,6 @@ jobs: sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile - - name: Install websocat and sshpass - run: | - sudo wget https://github.com/vi/websocat/releases/download/v1.9.0/websocat_linux64 -O /usr/local/bin/websocat - sudo chmod +x /usr/local/bin/websocat - sudo apt-get install sshpass - - name: Disconnect Docker containers from default network continue-on-error: true # related to this issue: https://github.com/moby/moby/issues/23302 From 9a29755ae6faa05c0c134d488dd21521cede9ba2 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:16:24 +0200 Subject: [PATCH 11/56] test(examples): ci debug --- .github/workflows/{goth-examples.yml => goth-example.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{goth-examples.yml => goth-example.yml} (97%) diff --git a/.github/workflows/goth-examples.yml b/.github/workflows/goth-example.yml similarity index 97% rename from .github/workflows/goth-examples.yml rename to .github/workflows/goth-example.yml index 44928c513..ff3067e74 100644 --- a/.github/workflows/goth-examples.yml +++ b/.github/workflows/goth-example.yml @@ -1,4 +1,4 @@ -name: Nightly Examples Tests +name: Nightly Example Tests on: # Allows triggering the workflow manually @@ -6,7 +6,7 @@ on: push: branches: - - "test/JST-363/*" + - "test/JST-363/example-tests" jobs: prepare-matrix-master-only: From 0931959abcc9676b7da2234af58b7e469658a68a Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:47:43 +0200 Subject: [PATCH 12/56] test(examples): added assertions for errors --- tests/examples/examples.test.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index ac1d46bc0..5f89416c5 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -1,4 +1,4 @@ -import { spawn } from "child_process"; +import { ChildProcess, spawn } from "child_process"; import { dirname, basename, resolve } from "path"; import { Goth } from "../goth/goth"; @@ -14,13 +14,13 @@ type Example = { const examples: Example[] = [ { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, - { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, - { cmd: "node", path: "docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, - { cmd: "node", path: "docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, - { cmd: "node", path: "docs-examples/examples/composing-tasks/singleCommand.mjs" }, + { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, + { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, + { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, + { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/singleCommand.mjs" }, ]; -const criticalLogsRegexp = [/Task timeot/, /Task rejected/, /TODO/]; +const criticalLogsRegexp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; const timeoutPromise = (seconds: number) => new Promise((res, rej) => @@ -29,12 +29,19 @@ const timeoutPromise = (seconds: number) => seconds * 1000, ), ); - +const spawnedExamples: ChildProcess[] = []; async function examplesTest(cmd: string, path: string, args: string[] = [], timeout = 120) { const file = basename(path); const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { stdio: "inherit", cwd }); + spawnedExamples.push(spawnedExample); + spawnedExample.stdout?.setEncoding("utf-8"); const testPromise = new Promise((res, rej) => { + spawnedExample.stdout?.on("data", (data: string) => { + if (criticalLogsRegexp.some((regexp) => data.match(regexp))) { + return rej(`Example test "${file}" failed.`); + } + }); spawnedExample.on("close", (code, signal) => { if (code === 0) return res(true); rej(`Example test "${file}" exited with code ${code} by signal ${signal}`); @@ -53,6 +60,9 @@ async function testAll(examples: Example[]) { } catch (error) { console.error(error); process.exit(1); + } finally { + await goth.end().catch((error) => console.error(error)); + spawnedExamples.forEach((example) => example?.kill()); } process.exit(0); } From 0392f136336664b5a89993452b74be8acf0aaae3 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:55:19 +0200 Subject: [PATCH 13/56] test(examples): added assertions for errors --- tests/examples/examples.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 5f89416c5..6c7d61f7c 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -38,6 +38,7 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time spawnedExample.stdout?.setEncoding("utf-8"); const testPromise = new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { + console.log(data.replace(/[\n\t\r]/g, "")); if (criticalLogsRegexp.some((regexp) => data.match(regexp))) { return rej(`Example test "${file}" failed.`); } @@ -51,6 +52,7 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time } async function testAll(examples: Example[]) { + let exitCode = 0; await Promise.race([goth.start(), timeoutPromise(180)]); try { for (const example of examples) { @@ -59,12 +61,12 @@ async function testAll(examples: Example[]) { } } catch (error) { console.error(error); - process.exit(1); + exitCode = 1; } finally { await goth.end().catch((error) => console.error(error)); spawnedExamples.forEach((example) => example?.kill()); } - process.exit(0); + process.exit(exitCode); } testAll(examples).then(); From e366264d1c0f48366d9a3609b3fa7d332df03f41 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 15:56:54 +0200 Subject: [PATCH 14/56] test(examples): added assertions for errors --- tests/examples/examples.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 6c7d61f7c..f3d0e1a69 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -56,7 +56,7 @@ async function testAll(examples: Example[]) { await Promise.race([goth.start(), timeoutPromise(180)]); try { for (const example of examples) { - console.log(`Starting test for example ${example.path}`); + console.log(`\n---- Starting test for example ${example.path} ----\n`); await examplesTest(example.cmd, example.path, example.args); } } catch (error) { From a13c7bfe5e441cdbd6799c017d66f139561d81a5 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 16:35:40 +0200 Subject: [PATCH 15/56] test(examples): added assertions for errors --- tests/examples/examples.test.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index f3d0e1a69..e06334e4c 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -33,7 +33,7 @@ const spawnedExamples: ChildProcess[] = []; async function examplesTest(cmd: string, path: string, args: string[] = [], timeout = 120) { const file = basename(path); const cwd = dirname(path); - const spawnedExample = spawn(cmd, [file, ...args], { stdio: "inherit", cwd }); + const spawnedExample = spawn(cmd, [file, ...args], { cwd }); spawnedExamples.push(spawnedExample); spawnedExample.stdout?.setEncoding("utf-8"); const testPromise = new Promise((res, rej) => { @@ -53,19 +53,22 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time async function testAll(examples: Example[]) { let exitCode = 0; + const failedTests = new Map(); await Promise.race([goth.start(), timeoutPromise(180)]); - try { - for (const example of examples) { - console.log(`\n---- Starting test for example ${example.path} ----\n`); + for (const example of examples) { + try { + console.log(`\n---- Starting test for example: "${example.path}" ----\n`); await examplesTest(example.cmd, example.path, example.args); + } catch (error) { + console.error(error); + failedTests.set(example.path, false); } - } catch (error) { - console.error(error); - exitCode = 1; - } finally { - await goth.end().catch((error) => console.error(error)); - spawnedExamples.forEach((example) => example?.kill()); } + await goth.end().catch((error) => console.error(error)); + spawnedExamples.forEach((example) => example?.kill()); + console.log( + `TESTS:\t${examples.length - failedTests.size} passed, ${failedTests.size} failed, ${examples.length} total`, + ); process.exit(exitCode); } From b042be5c9a98b48db5264223f75ea68b018d3e37 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 16:43:00 +0200 Subject: [PATCH 16/56] test(examples): added assertions for errors --- tests/examples/examples.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index e06334e4c..24ee43232 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -52,7 +52,6 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time } async function testAll(examples: Example[]) { - let exitCode = 0; const failedTests = new Map(); await Promise.race([goth.start(), timeoutPromise(180)]); for (const example of examples) { @@ -69,7 +68,7 @@ async function testAll(examples: Example[]) { console.log( `TESTS:\t${examples.length - failedTests.size} passed, ${failedTests.size} failed, ${examples.length} total`, ); - process.exit(exitCode); + process.exit(failedTests.size > 0 ? 1 : 0); } testAll(examples).then(); From 9098abbf38fca9e780f034f2d89af24b582d6cc0 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Tue, 19 Sep 2023 16:48:09 +0200 Subject: [PATCH 17/56] test(examples): added assertions for errors --- tests/examples/examples.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 24ee43232..21548a0e9 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -38,7 +38,7 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time spawnedExample.stdout?.setEncoding("utf-8"); const testPromise = new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { - console.log(data.replace(/[\n\t\r]/g, "")); + console.log(data.replace("\n", "")); if (criticalLogsRegexp.some((regexp) => data.match(regexp))) { return rej(`Example test "${file}" failed.`); } @@ -66,7 +66,7 @@ async function testAll(examples: Example[]) { await goth.end().catch((error) => console.error(error)); spawnedExamples.forEach((example) => example?.kill()); console.log( - `TESTS:\t${examples.length - failedTests.size} passed, ${failedTests.size} failed, ${examples.length} total`, + `\n\nTESTS: ${examples.length - failedTests.size} passed, ${failedTests.size} failed, ${examples.length} total`, ); process.exit(failedTests.size > 0 ? 1 : 0); } From a8d063288f2e40018833615dbe1cbf8006e2733d Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 10:17:53 +0200 Subject: [PATCH 18/56] test(examples): added chalk for logging --- tests/examples/examples.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 21548a0e9..37a6ffe6f 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -1,6 +1,7 @@ import { ChildProcess, spawn } from "child_process"; import { dirname, basename, resolve } from "path"; import { Goth } from "../goth/goth"; +import chalk from "chalk"; const gothConfig = resolve("../goth/assets/goth-config.yml"); const goth = new Goth(gothConfig); @@ -38,7 +39,7 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time spawnedExample.stdout?.setEncoding("utf-8"); const testPromise = new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { - console.log(data.replace("\n", "")); + console.log(chalk.cyanBright("[test]"), data.trim()); if (criticalLogsRegexp.some((regexp) => data.match(regexp))) { return rej(`Example test "${file}" failed.`); } @@ -56,17 +57,20 @@ async function testAll(examples: Example[]) { await Promise.race([goth.start(), timeoutPromise(180)]); for (const example of examples) { try { - console.log(`\n---- Starting test for example: "${example.path}" ----\n`); + console.log(chalk.yellow(`\n\tStarting test for example: "${example.path}"\n`)); await examplesTest(example.cmd, example.path, example.args); } catch (error) { - console.error(error); + console.error(chalk.red(error)); failedTests.set(example.path, false); } } await goth.end().catch((error) => console.error(error)); spawnedExamples.forEach((example) => example?.kill()); console.log( - `\n\nTESTS: ${examples.length - failedTests.size} passed, ${failedTests.size} failed, ${examples.length} total`, + chalk.cyan("\n\nTESTS RESULTS: "), + chalk.bgGreen(`${examples.length - failedTests.size} passed`), + chalk.bgRed(`${failedTests.size} failed`), + chalk.bgYellow(`${examples.length} total`), ); process.exit(failedTests.size > 0 ? 1 : 0); } From ea7ce192e676386d55ae17b060030c1fac7d1a39 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 11:06:46 +0200 Subject: [PATCH 19/56] test(examples): added no-goth arg --- package.json | 1 + tests/examples/examples.test.ts | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index da961bc73..4aab2ccf2 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "test:e2e:no-goth": "jest tests/e2e/**.spec.ts --testTimeout=180000 --runInBand --forceExit", "test:cypress": "cypress run", "test:examples": "ts-node --project ./tsconfig.spec.json tests/examples/examples.test.ts", + "test:examples:no-goth": "ts-node --project ./tsconfig.spec.json tests/examples/examples.test.ts --no-goth", "lint": "npm run lint:ts && npm run lint:ts:tests && npm run lint:eslint", "lint:ts": "tsc --project tsconfig.json --noEmit", "lint:ts:tests": "tsc --project tests/tsconfig.json --noEmit", diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 37a6ffe6f..6f4372d6c 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -3,6 +3,7 @@ import { dirname, basename, resolve } from "path"; import { Goth } from "../goth/goth"; import chalk from "chalk"; +const noGoth = process.argv[2] === "--no-goth"; const gothConfig = resolve("../goth/assets/goth-config.yml"); const goth = new Goth(gothConfig); @@ -21,7 +22,7 @@ const examples: Example[] = [ { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/singleCommand.mjs" }, ]; -const criticalLogsRegexp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; +const criticalLogsRegExp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; const timeoutPromise = (seconds: number) => new Promise((res, rej) => @@ -39,8 +40,8 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time spawnedExample.stdout?.setEncoding("utf-8"); const testPromise = new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { - console.log(chalk.cyanBright("[test]"), data.trim()); - if (criticalLogsRegexp.some((regexp) => data.match(regexp))) { + console.log(data.trim()); + if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { return rej(`Example test "${file}" failed.`); } }); @@ -53,25 +54,27 @@ async function examplesTest(cmd: string, path: string, args: string[] = [], time } async function testAll(examples: Example[]) { - const failedTests = new Map(); - await Promise.race([goth.start(), timeoutPromise(180)]); + const failedTests = new Set(); + if (!noGoth) await Promise.race([goth.start(), timeoutPromise(180)]); for (const example of examples) { try { - console.log(chalk.yellow(`\n\tStarting test for example: "${example.path}"\n`)); + console.log(chalk.yellow(`\n---- Starting test for example: "${example.path} ----"\n`)); await examplesTest(example.cmd, example.path, example.args); } catch (error) { console.error(chalk.red(error)); - failedTests.set(example.path, false); + failedTests.add(example.path); } } - await goth.end().catch((error) => console.error(error)); + if (!noGoth) await goth.end().catch((error) => console.error(error)); spawnedExamples.forEach((example) => example?.kill()); console.log( - chalk.cyan("\n\nTESTS RESULTS: "), - chalk.bgGreen(`${examples.length - failedTests.size} passed`), - chalk.bgRed(`${failedTests.size} failed`), - chalk.bgYellow(`${examples.length} total`), + chalk.bold.yellow("\n\nTESTS RESULTS: "), + chalk.bgGreen.black(`\t${examples.length - failedTests.size} passed\t`), + chalk.bgRed.black(`\t${failedTests.size} failed\t`), + chalk.bgCyan.black(`\t${examples.length} total\t\n`), ); + console.log(chalk.red("Failed tests:")); + failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); process.exit(failedTests.size > 0 ? 1 : 0); } From 5f7f65a4f78dea3cf8c1cd7fe91046a34b82fcd9 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 11:10:45 +0200 Subject: [PATCH 20/56] test(examples): fixed formatting --- tests/examples/examples.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 6f4372d6c..5fa668299 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -58,10 +58,10 @@ async function testAll(examples: Example[]) { if (!noGoth) await Promise.race([goth.start(), timeoutPromise(180)]); for (const example of examples) { try { - console.log(chalk.yellow(`\n---- Starting test for example: "${example.path} ----"\n`)); + console.log(chalk.yellow(`\n---- Starting test for example: "${example.path}" ----\n`)); await examplesTest(example.cmd, example.path, example.args); } catch (error) { - console.error(chalk.red(error)); + console.error(chalk.bgRed.white("FAIL"), chalk.red(error)); failedTests.add(example.path); } } @@ -69,11 +69,11 @@ async function testAll(examples: Example[]) { spawnedExamples.forEach((example) => example?.kill()); console.log( chalk.bold.yellow("\n\nTESTS RESULTS: "), - chalk.bgGreen.black(`\t${examples.length - failedTests.size} passed\t`), - chalk.bgRed.black(`\t${failedTests.size} failed\t`), - chalk.bgCyan.black(`\t${examples.length} total\t\n`), + chalk.bgGreen.black(` ${examples.length - failedTests.size} PASSED `), + chalk.bgRed.black(` ${failedTests.size} FAILED `), + chalk.bgCyan.black(` ${examples.length} TOTAL `), ); - console.log(chalk.red("Failed tests:")); + console.log(chalk.red("\nFailed tests:")); failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); process.exit(failedTests.size > 0 ? 1 : 0); } From 56fe57116897e56a79e3d172b7a4e64c627786f3 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 14:23:39 +0200 Subject: [PATCH 21/56] test(examples): added examples index --- .../examples/composing-tasks/batch-end.mjs | 2 +- .../composing-tasks/single-command.cjs | 13 ++++ .../{singleCommand.mjs => single-command.mjs} | 0 .../composing-tasks/single-command.ts | 13 ++++ .../{beforeeach.mjs => before-each.mjs} | 0 ...ralleltasks.mjs => max-parallel-tasks.mjs} | 0 .../{singlerun.mjs => single-run.mjs} | 0 ...wnloadingfile.mjs => downloading-file.mjs} | 0 .../{uploadingfile.mjs => uploading-file.mjs} | 0 .../{uploadingJSON.mjs => uploading-json.mjs} | 0 .../{runOnPolygon.mjs => run-on-polygon.mjs} | 0 .../{downloadFile.mjs => download-file.mjs} | 0 .../{uploadFile.mjs => upload-file.mjs} | 0 .../{uploadJSON.mjs => upload-json.mjs} | 0 ...icommand-end.mjs => multi-command-end.mjs} | 0 ...stream.mjs => multi-command-endstream.mjs} | 0 ...ommand-fail.mjs => multi-command-fail.mjs} | 0 package.json | 4 +- tests/examples/examples.json | 51 ++++++++++++++ tests/examples/examples.test.ts | 67 +++++++++---------- tests/tsconfig.json | 3 +- tsconfig.spec.json | 3 +- 22 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 examples/docs-examples/examples/composing-tasks/single-command.cjs rename examples/docs-examples/examples/composing-tasks/{singleCommand.mjs => single-command.mjs} (100%) create mode 100644 examples/docs-examples/examples/composing-tasks/single-command.ts rename examples/docs-examples/examples/executing-tasks/{beforeeach.mjs => before-each.mjs} (100%) rename examples/docs-examples/examples/executing-tasks/{maxparalleltasks.mjs => max-parallel-tasks.mjs} (100%) rename examples/docs-examples/examples/executing-tasks/{singlerun.mjs => single-run.mjs} (100%) rename examples/docs-examples/examples/sending-data/{downloadingfile.mjs => downloading-file.mjs} (100%) rename examples/docs-examples/examples/sending-data/{uploadingfile.mjs => uploading-file.mjs} (100%) rename examples/docs-examples/examples/sending-data/{uploadingJSON.mjs => uploading-json.mjs} (100%) rename examples/docs-examples/examples/switching-to-mainnet/{runOnPolygon.mjs => run-on-polygon.mjs} (100%) rename examples/docs-examples/examples/transferring-data/{downloadFile.mjs => download-file.mjs} (100%) rename examples/docs-examples/examples/transferring-data/{uploadFile.mjs => upload-file.mjs} (100%) rename examples/docs-examples/examples/transferring-data/{uploadJSON.mjs => upload-json.mjs} (100%) rename examples/docs-examples/examples/working-with-results/{multicommand-end.mjs => multi-command-end.mjs} (100%) rename examples/docs-examples/examples/working-with-results/{multicommand-endstream.mjs => multi-command-endstream.mjs} (100%) rename examples/docs-examples/examples/working-with-results/{multicommand-fail.mjs => multi-command-fail.mjs} (100%) create mode 100644 tests/examples/examples.json mode change 100755 => 100644 tsconfig.spec.json diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index 5ab69c1d2..8a5a9ebd0 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -9,7 +9,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const result = await executor.run(async (ctx) => { const res = await ctx .beginBatch() - .uploadFile("./worker2.mjs", "/golem/input/worker.mjs") + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") .run("node /golem/input/worker.mjs > /golem/input/output.txt") .run("cat /golem/input/output.txt") .downloadFile("/golem/input/output.txt", "./output.txt") diff --git a/examples/docs-examples/examples/composing-tasks/single-command.cjs b/examples/docs-examples/examples/composing-tasks/single-command.cjs new file mode 100644 index 000000000..79808a8ae --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/single-command.cjs @@ -0,0 +1,13 @@ +const { TaskExecutor } = require("@golem-sdk/golem-js"); + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/singleCommand.mjs b/examples/docs-examples/examples/composing-tasks/single-command.mjs similarity index 100% rename from examples/docs-examples/examples/composing-tasks/singleCommand.mjs rename to examples/docs-examples/examples/composing-tasks/single-command.mjs diff --git a/examples/docs-examples/examples/composing-tasks/single-command.ts b/examples/docs-examples/examples/composing-tasks/single-command.ts new file mode 100644 index 000000000..ad1d6a5a2 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/single-command.ts @@ -0,0 +1,13 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/beforeeach.mjs b/examples/docs-examples/examples/executing-tasks/before-each.mjs similarity index 100% rename from examples/docs-examples/examples/executing-tasks/beforeeach.mjs rename to examples/docs-examples/examples/executing-tasks/before-each.mjs diff --git a/examples/docs-examples/examples/executing-tasks/maxparalleltasks.mjs b/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs similarity index 100% rename from examples/docs-examples/examples/executing-tasks/maxparalleltasks.mjs rename to examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs diff --git a/examples/docs-examples/examples/executing-tasks/singlerun.mjs b/examples/docs-examples/examples/executing-tasks/single-run.mjs similarity index 100% rename from examples/docs-examples/examples/executing-tasks/singlerun.mjs rename to examples/docs-examples/examples/executing-tasks/single-run.mjs diff --git a/examples/docs-examples/examples/sending-data/downloadingfile.mjs b/examples/docs-examples/examples/sending-data/downloading-file.mjs similarity index 100% rename from examples/docs-examples/examples/sending-data/downloadingfile.mjs rename to examples/docs-examples/examples/sending-data/downloading-file.mjs diff --git a/examples/docs-examples/examples/sending-data/uploadingfile.mjs b/examples/docs-examples/examples/sending-data/uploading-file.mjs similarity index 100% rename from examples/docs-examples/examples/sending-data/uploadingfile.mjs rename to examples/docs-examples/examples/sending-data/uploading-file.mjs diff --git a/examples/docs-examples/examples/sending-data/uploadingJSON.mjs b/examples/docs-examples/examples/sending-data/uploading-json.mjs similarity index 100% rename from examples/docs-examples/examples/sending-data/uploadingJSON.mjs rename to examples/docs-examples/examples/sending-data/uploading-json.mjs diff --git a/examples/docs-examples/examples/switching-to-mainnet/runOnPolygon.mjs b/examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs similarity index 100% rename from examples/docs-examples/examples/switching-to-mainnet/runOnPolygon.mjs rename to examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs diff --git a/examples/docs-examples/examples/transferring-data/downloadFile.mjs b/examples/docs-examples/examples/transferring-data/download-file.mjs similarity index 100% rename from examples/docs-examples/examples/transferring-data/downloadFile.mjs rename to examples/docs-examples/examples/transferring-data/download-file.mjs diff --git a/examples/docs-examples/examples/transferring-data/uploadFile.mjs b/examples/docs-examples/examples/transferring-data/upload-file.mjs similarity index 100% rename from examples/docs-examples/examples/transferring-data/uploadFile.mjs rename to examples/docs-examples/examples/transferring-data/upload-file.mjs diff --git a/examples/docs-examples/examples/transferring-data/uploadJSON.mjs b/examples/docs-examples/examples/transferring-data/upload-json.mjs similarity index 100% rename from examples/docs-examples/examples/transferring-data/uploadJSON.mjs rename to examples/docs-examples/examples/transferring-data/upload-json.mjs diff --git a/examples/docs-examples/examples/working-with-results/multicommand-end.mjs b/examples/docs-examples/examples/working-with-results/multi-command-end.mjs similarity index 100% rename from examples/docs-examples/examples/working-with-results/multicommand-end.mjs rename to examples/docs-examples/examples/working-with-results/multi-command-end.mjs diff --git a/examples/docs-examples/examples/working-with-results/multicommand-endstream.mjs b/examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs similarity index 100% rename from examples/docs-examples/examples/working-with-results/multicommand-endstream.mjs rename to examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs diff --git a/examples/docs-examples/examples/working-with-results/multicommand-fail.mjs b/examples/docs-examples/examples/working-with-results/multi-command-fail.mjs similarity index 100% rename from examples/docs-examples/examples/working-with-results/multicommand-fail.mjs rename to examples/docs-examples/examples/working-with-results/multi-command-fail.mjs diff --git a/package.json b/package.json index 4aab2ccf2..af430c82f 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,8 @@ "test:e2e": "jest --config tests/e2e/jest.config.json tests/e2e/**.spec.ts --runInBand --forceExit", "test:e2e:no-goth": "jest tests/e2e/**.spec.ts --testTimeout=180000 --runInBand --forceExit", "test:cypress": "cypress run", - "test:examples": "ts-node --project ./tsconfig.spec.json tests/examples/examples.test.ts", - "test:examples:no-goth": "ts-node --project ./tsconfig.spec.json tests/examples/examples.test.ts --no-goth", + "test:examples": "ts-node --project tsconfig.spec.json tests/examples/examples.test.ts", + "test:examples:no-goth": "ts-node --project tsconfig.spec.json tests/examples/examples.test.ts --no-goth", "lint": "npm run lint:ts && npm run lint:ts:tests && npm run lint:eslint", "lint:ts": "tsc --project tsconfig.json --noEmit", "lint:ts:tests": "tsc --project tests/tsconfig.json --noEmit", diff --git a/tests/examples/examples.json b/tests/examples/examples.json new file mode 100644 index 000000000..62ff8f691 --- /dev/null +++ b/tests/examples/examples.json @@ -0,0 +1,51 @@ +[ + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.cjs" }, + { "cmd": "ts-node", "path": "examples/docs-examples/examples/composing-tasks/single-command.ts" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/before-each.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/foreach.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/map.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/single-run.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/demand.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/upoading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, + + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-json.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/using-app-key/index.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/hash.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command-fail.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/quickstarts/quickstart/requestor.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/tutorials/building-custom-image/index.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/tutorials/quickstart/index.mjs" }, + { + "cmd": "node", + "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", + "args": ["--mask '?a?a?a'", "--hash '$P$5ZDzPE45CLLhEx/72qt3NehVzwN2Ry/'"], + "timeout": 500 + } +] diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 5fa668299..c1020f8fe 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -1,12 +1,16 @@ -import { ChildProcess, spawn } from "child_process"; +import { spawn } from "child_process"; import { dirname, basename, resolve } from "path"; import { Goth } from "../goth/goth"; import chalk from "chalk"; +import examples from "./examples.json"; const noGoth = process.argv[2] === "--no-goth"; const gothConfig = resolve("../goth/assets/goth-config.yml"); +const gothStartingTimeout = 180; const goth = new Goth(gothConfig); +const criticalLogsRegExp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; + type Example = { cmd: string; path: string; @@ -14,64 +18,59 @@ type Example = { timeout?: number; }; -const examples: Example[] = [ - { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, - { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, - { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, - { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, - { cmd: "node", path: "examples/docs-examples/examples/composing-tasks/singleCommand.mjs" }, -]; - -const criticalLogsRegExp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; - -const timeoutPromise = (seconds: number) => - new Promise((res, rej) => - setTimeout( - () => rej(new Error(`The timeout was reached and the racing promise has rejected after ${seconds} seconds`)), - seconds * 1000, - ), - ); -const spawnedExamples: ChildProcess[] = []; -async function examplesTest(cmd: string, path: string, args: string[] = [], timeout = 120) { +async function test(cmd: string, path: string, args: string[] = [], timeout = 180) { const file = basename(path); const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); - spawnedExamples.push(spawnedExample); spawnedExample.stdout?.setEncoding("utf-8"); - const testPromise = new Promise((res, rej) => { + const timeoutId = setTimeout(() => spawnedExample.kill("SIGUSR1"), timeout * 1000); + return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { console.log(data.trim()); if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { - return rej(`Example test "${file}" failed.`); + spawnedExample.kill("SIGUSR2"); } }); spawnedExample.on("close", (code, signal) => { - if (code === 0) return res(true); - rej(`Example test "${file}" exited with code ${code} by signal ${signal}`); + if (code === 0 && signal === null) return res(true); + let errorMsg = ""; + if (signal === "SIGUSR1") errorMsg = `Test timeout was reached after ${timeout} seconds.`; + if (signal === "SIGUSR2") errorMsg = `A critical error occurred during the test.`; + rej(`Test example "${file}" failed. ${errorMsg}`); }); + }).finally(() => { + clearTimeout(timeoutId); + spawnedExample.kill(); }); - return Promise.race([timeoutPromise(timeout), testPromise]); } async function testAll(examples: Example[]) { const failedTests = new Set(); - if (!noGoth) await Promise.race([goth.start(), timeoutPromise(180)]); + if (!noGoth) + await Promise.race([ + goth.start(), + new Promise((res, rej) => + setTimeout( + () => rej(new Error(`The Goth starting timeout was reached after ${gothStartingTimeout} seconds`)), + gothStartingTimeout * 1000, + ), + ), + ]); for (const example of examples) { try { - console.log(chalk.yellow(`\n---- Starting test for example: "${example.path}" ----\n`)); - await examplesTest(example.cmd, example.path, example.args); + console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); + await test(example.cmd, example.path, example.args); } catch (error) { - console.error(chalk.bgRed.white("FAIL"), chalk.red(error)); + console.error(chalk.bgRed.white(" FAIL "), chalk.red(error)); failedTests.add(example.path); } } if (!noGoth) await goth.end().catch((error) => console.error(error)); - spawnedExamples.forEach((example) => example?.kill()); console.log( chalk.bold.yellow("\n\nTESTS RESULTS: "), - chalk.bgGreen.black(` ${examples.length - failedTests.size} PASSED `), - chalk.bgRed.black(` ${failedTests.size} FAILED `), - chalk.bgCyan.black(` ${examples.length} TOTAL `), + chalk.bgGreen.black(` ${examples.length - failedTests.size} passed `), + chalk.bgRed.black(` ${failedTests.size} failed `), + chalk.bgCyan.black(` ${examples.length} total `), ); console.log(chalk.red("\nFailed tests:")); failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index f71bd877a..882c3e83d 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -3,6 +3,7 @@ "include": ["."], "exclude": ["./cypress"], "compilerOptions": { - "types": ["jest"] + "types": ["jest"], + "resolveJsonModule": true } } diff --git a/tsconfig.spec.json b/tsconfig.spec.json old mode 100755 new mode 100644 index ce95966b4..50f9904d3 --- a/tsconfig.spec.json +++ b/tsconfig.spec.json @@ -15,7 +15,8 @@ "skipLibCheck": true, "lib": ["es2015", "es2016", "es2017", "es2018", "esnext", "dom"], "outDir": "dist", - "typeRoots": ["node_modules/@types"] + "typeRoots": ["node_modules/@types"], + "resolveJsonModule": true }, "exclude": ["dist", "node_modules", "examples", "cypress.config.ts"], "include": ["src/**/*.spec.ts", "tests/unit/**/*.test.ts", "tests/unit/**/*.spec.ts"], From 96435ee95d581d29d80964268f8b7b35a32d6c7e Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 14:54:43 +0200 Subject: [PATCH 22/56] test(examples): fixed examples --- .../batch-endstream-forawait.mjs | 2 +- .../composing-tasks/multiple-run-prosaic.mjs | 1 - tests/examples/examples.test.ts | 18 +++++++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs index 6f870819f..95606d5c5 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs @@ -16,7 +16,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; .endStream(); for await (const chunk of res) { - chunk.index == 2 ? console.log(chunk.stdout) : ""; + console.log(chunk.stdout); } }); diff --git a/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs index 275ec7256..acfd7a9e8 100644 --- a/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs +++ b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs @@ -4,7 +4,6 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", yagnaOptions: { apiKey: "try_golem" }, - isSubprocess: true, }); const result = await executor.run(async (ctx) => { diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index c1020f8fe..210f50e14 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -2,13 +2,15 @@ import { spawn } from "child_process"; import { dirname, basename, resolve } from "path"; import { Goth } from "../goth/goth"; import chalk from "chalk"; -import examples from "./examples.json"; +import testExamples from "./examples.json"; const noGoth = process.argv[2] === "--no-goth"; const gothConfig = resolve("../goth/assets/goth-config.yml"); const gothStartingTimeout = 180; const goth = new Goth(gothConfig); +const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; + const criticalLogsRegExp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; type Example = { @@ -23,19 +25,21 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); spawnedExample.stdout?.setEncoding("utf-8"); - const timeoutId = setTimeout(() => spawnedExample.kill("SIGUSR1"), timeout * 1000); + const timeoutId = setTimeout(() => { + spawnedExample.kill("SIGABRT"); + }, timeout * 1000); return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { console.log(data.trim()); if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { - spawnedExample.kill("SIGUSR2"); + spawnedExample.kill("SIGKILL"); } }); spawnedExample.on("close", (code, signal) => { if (code === 0 && signal === null) return res(true); let errorMsg = ""; - if (signal === "SIGUSR1") errorMsg = `Test timeout was reached after ${timeout} seconds.`; - if (signal === "SIGUSR2") errorMsg = `A critical error occurred during the test.`; + if (signal === "SIGABRT") errorMsg = `Test timeout was reached after ${timeout} seconds.`; + if (signal === "SIGKILL") errorMsg = `A critical error occurred during the test.`; rej(`Test example "${file}" failed. ${errorMsg}`); }); }).finally(() => { @@ -59,7 +63,7 @@ async function testAll(examples: Example[]) { for (const example of examples) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); - await test(example.cmd, example.path, example.args); + await test(example.cmd, example.path, example.args, example.timeout); } catch (error) { console.error(chalk.bgRed.white(" FAIL "), chalk.red(error)); failedTests.add(example.path); @@ -70,7 +74,7 @@ async function testAll(examples: Example[]) { chalk.bold.yellow("\n\nTESTS RESULTS: "), chalk.bgGreen.black(` ${examples.length - failedTests.size} passed `), chalk.bgRed.black(` ${failedTests.size} failed `), - chalk.bgCyan.black(` ${examples.length} total `), + chalk.bgCyan.black(` ${examples.length} total `), ); console.log(chalk.red("\nFailed tests:")); failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); From 9f08e1394df08cad979a14953a52609adee5c154 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 15:08:48 +0200 Subject: [PATCH 23/56] test(examples): fixed examples --- examples/docs-examples/examples/composing-tasks/batch-end.mjs | 2 +- examples/docs-examples/examples/executing-tasks/action_log.txt | 1 + tests/examples/examples.test.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 examples/docs-examples/examples/executing-tasks/action_log.txt diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index 8a5a9ebd0..ed9dcdf14 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -9,7 +9,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const result = await executor.run(async (ctx) => { const res = await ctx .beginBatch() - .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .uploadFile("./worker222.mjs", "/golem/input/worker.mjs") .run("node /golem/input/worker.mjs > /golem/input/output.txt") .run("cat /golem/input/output.txt") .downloadFile("/golem/input/output.txt", "./output.txt") diff --git a/examples/docs-examples/examples/executing-tasks/action_log.txt b/examples/docs-examples/examples/executing-tasks/action_log.txt new file mode 100644 index 000000000..bb5749a1f --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/action_log.txt @@ -0,0 +1 @@ +some action log \ No newline at end of file diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 210f50e14..8a8b65bee 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -36,7 +36,7 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 } }); spawnedExample.on("close", (code, signal) => { - if (code === 0 && signal === null) return res(true); + if (signal === null) return res(true); let errorMsg = ""; if (signal === "SIGABRT") errorMsg = `Test timeout was reached after ${timeout} seconds.`; if (signal === "SIGKILL") errorMsg = `A critical error occurred during the test.`; From bf1cf03ca2728aaafeda7ddb7464f5e0a68fc492 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 15:10:55 +0200 Subject: [PATCH 24/56] test(examples): fixed examples --- examples/docs-examples/examples/composing-tasks/batch-end.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index ed9dcdf14..8a5a9ebd0 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -9,7 +9,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const result = await executor.run(async (ctx) => { const res = await ctx .beginBatch() - .uploadFile("./worker222.mjs", "/golem/input/worker.mjs") + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") .run("node /golem/input/worker.mjs > /golem/input/output.txt") .run("cat /golem/input/output.txt") .downloadFile("/golem/input/output.txt", "./output.txt") From 51a22d4df47de125f46afac52721510ec6e855b1 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 15:21:33 +0200 Subject: [PATCH 25/56] test(examples): fixed examples --- .../examples/composing-tasks/batch-end.mjs | 2 +- tests/examples/examples.test.ts | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index 8a5a9ebd0..a0caa409e 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -9,7 +9,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const result = await executor.run(async (ctx) => { const res = await ctx .beginBatch() - .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .uploadFile("./worker22222.mjs", "/golem/input/worker.mjs") .run("node /golem/input/worker.mjs > /golem/input/output.txt") .run("cat /golem/input/output.txt") .downloadFile("/golem/input/output.txt", "./output.txt") diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 8a8b65bee..168fd82ed 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -25,21 +25,19 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); spawnedExample.stdout?.setEncoding("utf-8"); - const timeoutId = setTimeout(() => { - spawnedExample.kill("SIGABRT"); - }, timeout * 1000); + const timeoutId = setTimeout(() => spawnedExample.kill("SIGINT"), timeout * 1000); return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { console.log(data.trim()); if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { - spawnedExample.kill("SIGKILL"); + spawnedExample.kill("SIGTERM"); } }); spawnedExample.on("close", (code, signal) => { if (signal === null) return res(true); let errorMsg = ""; - if (signal === "SIGABRT") errorMsg = `Test timeout was reached after ${timeout} seconds.`; - if (signal === "SIGKILL") errorMsg = `A critical error occurred during the test.`; + if (signal === "SIGINT") errorMsg = `Test timeout was reached after ${timeout} seconds.`; + if (signal === "SIGTERM") errorMsg = `A critical error occurred during the test.`; rej(`Test example "${file}" failed. ${errorMsg}`); }); }).finally(() => { From d850b47d405aadc7ed4b21fd336bf03bf970eb00 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 16:02:20 +0200 Subject: [PATCH 26/56] test(examples): fixed examples --- tests/examples/examples.json | 2 +- tests/examples/examples.test.ts | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 62ff8f691..9bcd3d54d 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -27,7 +27,7 @@ { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-json.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/using-app-key/index.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/using-app-keys/index.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/hash.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs" }, diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 168fd82ed..783da6968 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -25,24 +25,28 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); spawnedExample.stdout?.setEncoding("utf-8"); - const timeoutId = setTimeout(() => spawnedExample.kill("SIGINT"), timeout * 1000); + let error = ""; + const timeoutId = setTimeout(() => { + error = `Test timeout was reached after ${timeout} seconds.`; + spawnedExample.kill("SIGTERM"); + spawnedExample.kill("SIGKILL"); + }, timeout * 1000); return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { console.log(data.trim()); if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { + error = `A critical error occurred during the test.`; spawnedExample.kill("SIGTERM"); + spawnedExample.kill("SIGKILL"); } }); - spawnedExample.on("close", (code, signal) => { - if (signal === null) return res(true); - let errorMsg = ""; - if (signal === "SIGINT") errorMsg = `Test timeout was reached after ${timeout} seconds.`; - if (signal === "SIGTERM") errorMsg = `A critical error occurred during the test.`; - rej(`Test example "${file}" failed. ${errorMsg}`); + spawnedExample.on("close", (code) => { + if (!error && code === 0) return res(true); + rej(`Test example "${file}" failed. ${error}`); }); }).finally(() => { clearTimeout(timeoutId); - spawnedExample.kill(); + spawnedExample.kill("SIGKILL"); }); } From 3c7f7bffe10c08b57a24919a74d5e4a2c1d02fce Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 16:42:51 +0200 Subject: [PATCH 27/56] test(examples): fixed examples --- .../examples/composing-tasks/batch-end.mjs | 2 +- tests/examples/examples.test.ts | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index a0caa409e..8a5a9ebd0 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -9,7 +9,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const result = await executor.run(async (ctx) => { const res = await ctx .beginBatch() - .uploadFile("./worker22222.mjs", "/golem/input/worker.mjs") + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") .run("node /golem/input/worker.mjs > /golem/input/output.txt") .run("cat /golem/input/output.txt") .downloadFile("/golem/input/output.txt", "./output.txt") diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 783da6968..eaf1460bf 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -28,16 +28,19 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 let error = ""; const timeoutId = setTimeout(() => { error = `Test timeout was reached after ${timeout} seconds.`; - spawnedExample.kill("SIGTERM"); - spawnedExample.kill("SIGKILL"); + spawnedExample.kill(); + // for some reason the process doesn't exit after kill, it does so after I type enter + spawnedExample.stdin.write("\n"); + spawnedExample.stdin.end(); }, timeout * 1000); return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { console.log(data.trim()); if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { error = `A critical error occurred during the test.`; - spawnedExample.kill("SIGTERM"); - spawnedExample.kill("SIGKILL"); + spawnedExample.kill(); + spawnedExample.stdin.write("\n"); + spawnedExample.stdin.end(); } }); spawnedExample.on("close", (code) => { @@ -66,8 +69,9 @@ async function testAll(examples: Example[]) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); await test(example.cmd, example.path, example.args, example.timeout); + console.log(chalk.bgGreen.white(" PASS "), chalk.green(example.path)); } catch (error) { - console.error(chalk.bgRed.white(" FAIL "), chalk.red(error)); + console.log(chalk.bgRed.white(" FAIL "), chalk.red(error)); failedTests.add(example.path); } } From 9032f3a5a36f5897d1fc6e4b2736f1530e140bcd Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 17:00:24 +0200 Subject: [PATCH 28/56] test(examples): fixed examples --- .../working-with-results/single-command-fail.mjs | 1 - .../examples/working-with-results/single-command.mjs | 1 - .../examples/working-with-results/worker.mjs | 1 + tests/examples/examples.test.ts | 9 ++++----- 4 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 examples/docs-examples/examples/working-with-results/worker.mjs diff --git a/examples/docs-examples/examples/working-with-results/single-command-fail.mjs b/examples/docs-examples/examples/working-with-results/single-command-fail.mjs index e4a92201c..d8a523fbb 100644 --- a/examples/docs-examples/examples/working-with-results/single-command-fail.mjs +++ b/examples/docs-examples/examples/working-with-results/single-command-fail.mjs @@ -4,7 +4,6 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", yagnaOptions: { apiKey: "try_golem" }, - isSubprocess: true, }); // there is a mistake and instead of 'node -v' we call 'node -w' diff --git a/examples/docs-examples/examples/working-with-results/single-command.mjs b/examples/docs-examples/examples/working-with-results/single-command.mjs index 1075c8dba..7a48cebf6 100644 --- a/examples/docs-examples/examples/working-with-results/single-command.mjs +++ b/examples/docs-examples/examples/working-with-results/single-command.mjs @@ -4,7 +4,6 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", yagnaOptions: { apiKey: "try_golem" }, - isSubprocess: true, }); const result = await executor.run(async (ctx) => await ctx.run("node -v")); diff --git a/examples/docs-examples/examples/working-with-results/worker.mjs b/examples/docs-examples/examples/working-with-results/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index eaf1460bf..9c464452b 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -29,9 +29,6 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 const timeoutId = setTimeout(() => { error = `Test timeout was reached after ${timeout} seconds.`; spawnedExample.kill(); - // for some reason the process doesn't exit after kill, it does so after I type enter - spawnedExample.stdin.write("\n"); - spawnedExample.stdin.end(); }, timeout * 1000); return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { @@ -39,8 +36,10 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { error = `A critical error occurred during the test.`; spawnedExample.kill(); - spawnedExample.stdin.write("\n"); - spawnedExample.stdin.end(); + } + // for some reason, sometimes the process doesn't exit after Executor shut down + if (data.indexOf("Task Executor has shut down") !== -1) { + spawnedExample.kill("SIGKILL"); } }); spawnedExample.on("close", (code) => { From 6158a7476450acb9b2fac2f67c38179e955cd7fc Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 17:20:33 +0200 Subject: [PATCH 29/56] test(examples): fixed examples --- examples/docs-examples/examples/using-app-keys/index.mjs | 2 ++ tests/examples/examples.json | 4 ++-- tests/examples/examples.test.ts | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/docs-examples/examples/using-app-keys/index.mjs b/examples/docs-examples/examples/using-app-keys/index.mjs index 18cc40b23..c5adf9a59 100644 --- a/examples/docs-examples/examples/using-app-keys/index.mjs +++ b/examples/docs-examples/examples/using-app-keys/index.mjs @@ -1,3 +1,5 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + (async () => { const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 9bcd3d54d..428f15be9 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -13,9 +13,9 @@ { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/single-run.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs", "noGoth": true }, { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/demand.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs", "noGoth": true }, { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/upoading-file.mjs" }, diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 9c464452b..860ea52a1 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -43,7 +43,8 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 } }); spawnedExample.on("close", (code) => { - if (!error && code === 0) return res(true); + console.log("EXIT CODE", code); + if (!error) return res(true); rej(`Test example "${file}" failed. ${error}`); }); }).finally(() => { From 94dfdfdb6a0ec242246ac9612c7645b9f4c70b54 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 17:47:32 +0200 Subject: [PATCH 30/56] test(examples): fixed examples --- tests/examples/examples.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 860ea52a1..68d41b540 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -42,9 +42,9 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 spawnedExample.kill("SIGKILL"); } }); + spawnedExample.stderr?.on("data", (data: string) => console.log(data.trim())); spawnedExample.on("close", (code) => { - console.log("EXIT CODE", code); - if (!error) return res(true); + if (!error && !code) return res(true); rej(`Test example "${file}" failed. ${error}`); }); }).finally(() => { From 88909e0ec94649b6ceb75dd73e501c273c6e7131 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Wed, 20 Sep 2023 18:34:53 +0200 Subject: [PATCH 31/56] test(examples): fixed examples --- tests/examples/examples.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 68d41b540..1eeb4841f 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -25,6 +25,7 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); spawnedExample.stdout?.setEncoding("utf-8"); + spawnedExample.stderr?.setEncoding("utf-8"); let error = ""; const timeoutId = setTimeout(() => { error = `Test timeout was reached after ${timeout} seconds.`; From 2ad34c2896dae5270076ef6b2efbaf9202f9c5c9 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 11:15:28 +0200 Subject: [PATCH 32/56] test(examples): fixed yacat examples --- examples/docs-examples/examples/transferring-data/worker.mjs | 1 + tests/examples/examples.json | 2 +- tests/examples/examples.test.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 examples/docs-examples/examples/transferring-data/worker.mjs diff --git a/examples/docs-examples/examples/transferring-data/worker.mjs b/examples/docs-examples/examples/transferring-data/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 428f15be9..d63ea05ba 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -45,7 +45,7 @@ { "cmd": "node", "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", - "args": ["--mask '?a?a?a'", "--hash '$P$5ZDzPE45CLLhEx/72qt3NehVzwN2Ry/'"], + "args": ["--mask", "'?a?a?a'", "--hash", "'$P$5ZDzPE45CLLhEx/72qt3NehVzwN2Ry/'"], "timeout": 500 } ] diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 1eeb4841f..ab2ecef59 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -10,7 +10,6 @@ const gothStartingTimeout = 180; const goth = new Goth(gothConfig); const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; - const criticalLogsRegExp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; type Example = { @@ -18,6 +17,7 @@ type Example = { path: string; args?: string[]; timeout?: number; + noGoth?: boolean; }; async function test(cmd: string, path: string, args: string[] = [], timeout = 180) { From c90975d7e29f5659127436337333be5c14bedfab Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 12:57:39 +0200 Subject: [PATCH 33/56] test(examples): skip examples with batch bug --- tests/examples/examples.json | 8 ++++++-- tests/examples/examples.test.ts | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index d63ea05ba..01806a9b2 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -34,7 +34,11 @@ { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-end.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs" }, + { + "cmd": "node", + "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs", + "skip": true + }, { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command-fail.mjs" }, @@ -45,7 +49,7 @@ { "cmd": "node", "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", - "args": ["--mask", "'?a?a?a'", "--hash", "'$P$5ZDzPE45CLLhEx/72qt3NehVzwN2Ry/'"], + "args": ["--mask", "'?a?a'", "--hash", "'$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0'"], "timeout": 500 } ] diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index ab2ecef59..62e5ac673 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -9,8 +9,8 @@ const gothConfig = resolve("../goth/assets/goth-config.yml"); const gothStartingTimeout = 180; const goth = new Goth(gothConfig); -const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; -const criticalLogsRegExp = [/Task timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; +const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth && !e.skip) : testExamples.filter((e) => !e.skip); +const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; type Example = { cmd: string; @@ -18,6 +18,7 @@ type Example = { args?: string[]; timeout?: number; noGoth?: boolean; + skip?: boolean; }; async function test(cmd: string, path: string, args: string[] = [], timeout = 180) { From 7495e542d8d123ebba57f9e305e55ed17aef0c79 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 13:37:52 +0200 Subject: [PATCH 34/56] test(examples): added skipping feat --- examples/docs-examples/examples/sending-data/worker.mjs | 1 + tests/examples/examples.test.ts | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 examples/docs-examples/examples/sending-data/worker.mjs diff --git a/examples/docs-examples/examples/sending-data/worker.mjs b/examples/docs-examples/examples/sending-data/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/sending-data/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 62e5ac673..98c023c50 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -57,6 +57,7 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 async function testAll(examples: Example[]) { const failedTests = new Set(); + const skippedTests = new Set(); if (!noGoth) await Promise.race([ goth.start(), @@ -70,6 +71,11 @@ async function testAll(examples: Example[]) { for (const example of examples) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); + if (example?.skip) { + console.log(chalk.bgYellow.white(" SKIP "), chalk.yellow(`The test was skipped`)); + skippedTests.add(example.path); + continue; + } await test(example.cmd, example.path, example.args, example.timeout); console.log(chalk.bgGreen.white(" PASS "), chalk.green(example.path)); } catch (error) { @@ -82,10 +88,13 @@ async function testAll(examples: Example[]) { chalk.bold.yellow("\n\nTESTS RESULTS: "), chalk.bgGreen.black(` ${examples.length - failedTests.size} passed `), chalk.bgRed.black(` ${failedTests.size} failed `), + skippedTests.size ? chalk.bgYellow.black(` ${skippedTests.size} skipped `) : "", chalk.bgCyan.black(` ${examples.length} total `), ); console.log(chalk.red("\nFailed tests:")); failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); + console.log(chalk.yellow("\nSkipped tests:")); + failedTests.forEach((test) => console.log(chalk.yellow(`\t- ${test}`))); process.exit(failedTests.size > 0 ? 1 : 0); } From 97d69ff88ff0bb9686f4ab396c3b9a089da22c6a Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 13:54:33 +0200 Subject: [PATCH 35/56] test(examples): removed rinkeby from goth setup --- tests/goth/goth.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index 504d6ddf5..8fea42ca4 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -34,8 +34,6 @@ export class Goth { process.env["GSB_URL"] = gsbUrl; process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; - // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network - process.env["PAYMENT_NETWORK"] = "rinkeby"; const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From 15fa554fb6a1cb6401d17f1c1f7f43795fe62986 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 14:19:05 +0200 Subject: [PATCH 36/56] test(examples): removed rinkeby from goth setup --- tests/examples/examples.json | 4 ++-- tests/examples/examples.test.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 01806a9b2..d64984166 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -1,5 +1,5 @@ [ - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs", "skip": true }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, @@ -21,7 +21,7 @@ { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/upoading-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, + { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 98c023c50..e8e47b527 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -9,7 +9,7 @@ const gothConfig = resolve("../goth/assets/goth-config.yml"); const gothStartingTimeout = 180; const goth = new Goth(gothConfig); -const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth && !e.skip) : testExamples.filter((e) => !e.skip); +const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; type Example = { @@ -21,7 +21,7 @@ type Example = { skip?: boolean; }; -async function test(cmd: string, path: string, args: string[] = [], timeout = 180) { +async function test(cmd: string, path: string, args: string[] = [], timeout = 120) { const file = basename(path); const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); @@ -72,12 +72,12 @@ async function testAll(examples: Example[]) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); if (example?.skip) { - console.log(chalk.bgYellow.white(" SKIP "), chalk.yellow(`The test was skipped`)); + console.log(chalk.bgYellow.black(" SKIP "), chalk.yellowBright(example.path)); skippedTests.add(example.path); - continue; + } else { + await test(example.cmd, example.path, example.args, example.timeout); + console.log(chalk.bgGreen.white(" PASS "), chalk.green(example.path)); } - await test(example.cmd, example.path, example.args, example.timeout); - console.log(chalk.bgGreen.white(" PASS "), chalk.green(example.path)); } catch (error) { console.log(chalk.bgRed.white(" FAIL "), chalk.red(error)); failedTests.add(example.path); From 32fce0588d0575be77a6f76ccf5347af4c1d7014 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 14:19:19 +0200 Subject: [PATCH 37/56] test(examples): removed rinkeby from goth setup --- tests/examples/examples.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index d64984166..7f008174f 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -1,5 +1,5 @@ [ - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs", "skip": true }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, From 2abb16d42056e698b3bcd9152a6f76f682716c02 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 14:32:51 +0200 Subject: [PATCH 38/56] test(examples): fixed list --- tests/examples/examples.json | 4 ++-- tests/examples/examples.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 7f008174f..26485f6a9 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -18,10 +18,10 @@ { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs", "noGoth": true }, { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/upoading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index e8e47b527..4af9055ec 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -10,7 +10,7 @@ const gothStartingTimeout = 180; const goth = new Goth(gothConfig); const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; -const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/]; +const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/, /ERROR: Error/]; type Example = { cmd: string; @@ -86,7 +86,7 @@ async function testAll(examples: Example[]) { if (!noGoth) await goth.end().catch((error) => console.error(error)); console.log( chalk.bold.yellow("\n\nTESTS RESULTS: "), - chalk.bgGreen.black(` ${examples.length - failedTests.size} passed `), + chalk.bgGreen.black(` ${examples.length - failedTests.size - skippedTests.size} passed `), chalk.bgRed.black(` ${failedTests.size} failed `), skippedTests.size ? chalk.bgYellow.black(` ${skippedTests.size} skipped `) : "", chalk.bgCyan.black(` ${examples.length} total `), From b61b4087d6093331ee85ef990e611450c85351d0 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 14:57:59 +0200 Subject: [PATCH 39/56] test(examples): fixed yacat args --- tests/examples/examples.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 26485f6a9..3e6bb188c 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -49,7 +49,7 @@ { "cmd": "node", "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", - "args": ["--mask", "'?a?a'", "--hash", "'$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0'"], + "args": ["--mask", "?a?a", "--hash", "$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0"], "timeout": 500 } ] From cd552016adc597e78880d3cb37fc4177cbd97538 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 15:38:47 +0200 Subject: [PATCH 40/56] test(examples): fixed yacat args --- tests/examples/examples.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 4af9055ec..0b3ae608e 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -94,7 +94,7 @@ async function testAll(examples: Example[]) { console.log(chalk.red("\nFailed tests:")); failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); console.log(chalk.yellow("\nSkipped tests:")); - failedTests.forEach((test) => console.log(chalk.yellow(`\t- ${test}`))); + skippedTests.forEach((test) => console.log(chalk.yellow(`\t- ${test}`))); process.exit(failedTests.size > 0 ? 1 : 0); } From 3cc890719ecf318b14a6b6f5a93274e8ec8248c1 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 21 Sep 2023 18:27:47 +0200 Subject: [PATCH 41/56] test(examples): fixed yacat args --- tests/examples/examples.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 0b3ae608e..1df7125ff 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -10,7 +10,7 @@ const gothStartingTimeout = 180; const goth = new Goth(gothConfig); const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; -const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/, /ERROR: Error/]; +const criticalLogsRegExp = [/Task *. timeot/g, /Task *. has been rejected/g, /ERROR: TypeError/g, /ERROR: Error/g]; type Example = { cmd: string; From 92c584f4633acff0e7e5867ea10e103ace8ced49 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Fri, 22 Sep 2023 09:51:09 +0200 Subject: [PATCH 42/56] test(examples): rollback rinkeby for goth --- tests/goth/goth.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index 8fea42ca4..504d6ddf5 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -34,6 +34,8 @@ export class Goth { process.env["GSB_URL"] = gsbUrl; process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; + // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network + process.env["PAYMENT_NETWORK"] = "rinkeby"; const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From ef62fdc9adc55f3a0db90e23c7ade8af82e9ec98 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Fri, 22 Sep 2023 12:45:20 +0200 Subject: [PATCH 43/56] test(examples): goth debug --- .../examples/working-with-results/multi-command-end.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/docs-examples/examples/working-with-results/multi-command-end.mjs b/examples/docs-examples/examples/working-with-results/multi-command-end.mjs index 46e7fea74..b821d2272 100644 --- a/examples/docs-examples/examples/working-with-results/multi-command-end.mjs +++ b/examples/docs-examples/examples/working-with-results/multi-command-end.mjs @@ -4,6 +4,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", yagnaOptions: { apiKey: "try_golem" }, + logLevel: "debug", }); const result = await executor.run(async (ctx) => { From 8df194fff4ecce78a4f9fc7f129ce0923a046f22 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Fri, 22 Sep 2023 13:00:43 +0200 Subject: [PATCH 44/56] test(examples): goth debug --- tests/examples/examples.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 1df7125ff..0ba642517 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -68,6 +68,8 @@ async function testAll(examples: Example[]) { ), ), ]); + // For debugging CI Goth, wait before starting tests + await new Promise((res) => setTimeout(res, 10_000)); for (const example of examples) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); From b0b6c3fc22e4b7c4fce8830cbc216244c6cc57b4 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Fri, 22 Sep 2023 13:08:58 +0200 Subject: [PATCH 45/56] test(examples): goth debug --- tests/goth/goth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index 504d6ddf5..a766fa77c 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -35,7 +35,7 @@ export class Goth { process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network - process.env["PAYMENT_NETWORK"] = "rinkeby"; + // process.env["PAYMENT_NETWORK"] = "rinkeby"; const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From 07b1a27ebfa9ef3e55665c5d9cd3dd19998a42e0 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 10:07:00 +0200 Subject: [PATCH 46/56] test(examples): goth debug --- tests/goth/goth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index a766fa77c..504d6ddf5 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -35,7 +35,7 @@ export class Goth { process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network - // process.env["PAYMENT_NETWORK"] = "rinkeby"; + process.env["PAYMENT_NETWORK"] = "rinkeby"; const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From 77a26711d3c1ae4421920848ea83644ceb785f7b Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 10:16:04 +0200 Subject: [PATCH 47/56] test(examples): goth debug --- .github/workflows/goth-example.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/goth-example.yml b/.github/workflows/goth-example.yml index ff3067e74..80414fa0a 100644 --- a/.github/workflows/goth-example.yml +++ b/.github/workflows/goth-example.yml @@ -56,6 +56,7 @@ jobs: rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml + sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile - name: Disconnect Docker containers from default network From a57d8d24640ce413f0b4efd92521b65deb95b326 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 10:28:29 +0200 Subject: [PATCH 48/56] test(examples): goth debug --- .github/workflows/goth-example.yml | 1 - examples/docs-examples/examples/composing-tasks/batch-end.mjs | 1 + tests/goth/goth.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/goth-example.yml b/.github/workflows/goth-example.yml index 80414fa0a..ff3067e74 100644 --- a/.github/workflows/goth-example.yml +++ b/.github/workflows/goth-example.yml @@ -56,7 +56,6 @@ jobs: rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile - name: Disconnect Docker containers from default network diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index 8a5a9ebd0..724619be2 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -4,6 +4,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", yagnaOptions: { apiKey: "try_golem" }, + logLevel: "debug", }); const result = await executor.run(async (ctx) => { diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index 504d6ddf5..a766fa77c 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -35,7 +35,7 @@ export class Goth { process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network - process.env["PAYMENT_NETWORK"] = "rinkeby"; + // process.env["PAYMENT_NETWORK"] = "rinkeby"; const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From 7f1e3fd6a29ee9a7d9a432c43021b5e6fe0700ec Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 13:26:52 +0200 Subject: [PATCH 49/56] test(examples): goth debug --- src/storage/gftp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/gftp.ts b/src/storage/gftp.ts index 4e1d73a81..777a542f3 100644 --- a/src/storage/gftp.ts +++ b/src/storage/gftp.ts @@ -114,7 +114,7 @@ export class GftpStorageProvider implements StorageProvider { return; } - await this.jsonrpc("close", { urls: Array.from(this.publishedUrls) }); + await this.jsonrpc("close", { urls: Array.from(this.publishedUrls).filter((url) => !!url) }); } async close() { From 25787ec100e10a1608021fda29eddf7be1139aa0 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 13:38:28 +0200 Subject: [PATCH 50/56] test(examples): goth debug --- tests/examples/examples.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 0ba642517..1df7125ff 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -68,8 +68,6 @@ async function testAll(examples: Example[]) { ), ), ]); - // For debugging CI Goth, wait before starting tests - await new Promise((res) => setTimeout(res, 10_000)); for (const example of examples) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); From a9f0019938932b5bf97cb64daa39ca755beba32d Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 14:18:50 +0200 Subject: [PATCH 51/56] test(examples): goth debug --- tests/goth/goth.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index a766fa77c..3a2d15348 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -36,6 +36,8 @@ export class Goth { process.env["YAGNA_SUBNET"] = subnetTag; // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network // process.env["PAYMENT_NETWORK"] = "rinkeby"; + // Trying to force Goerli network + spawn("yagna payment init --sender --network goerli"); const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From b830493734b38da6f8120b2f56491db3d685e065 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 14:33:21 +0200 Subject: [PATCH 52/56] test(examples): goth min ver --- .github/workflows/goth-example.yml | 2 +- tests/goth/goth.ts | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/goth-example.yml b/.github/workflows/goth-example.yml index ff3067e74..cc03e8a98 100644 --- a/.github/workflows/goth-example.yml +++ b/.github/workflows/goth-example.yml @@ -52,7 +52,7 @@ jobs: - name: Install goth run: | - pip install goth + pip install goth>=0.15.3 rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index 3a2d15348..8fea42ca4 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -34,10 +34,6 @@ export class Goth { process.env["GSB_URL"] = gsbUrl; process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; - // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network - // process.env["PAYMENT_NETWORK"] = "rinkeby"; - // Trying to force Goerli network - spawn("yagna payment init --sender --network goerli"); const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; From 0b433669ca534d1c0e552d5acde42fc267fd3c56 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 14:58:55 +0200 Subject: [PATCH 53/56] test(examples): fixed regexp matching --- .../examples/composing-tasks/batch-end.mjs | 1 - .../examples/working-with-images/tag.mjs | 2 +- tests/examples/examples.json | 56 +------------------ tests/examples/examples.test.ts | 10 +++- 4 files changed, 9 insertions(+), 60 deletions(-) diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs index 724619be2..8a5a9ebd0 100644 --- a/examples/docs-examples/examples/composing-tasks/batch-end.mjs +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -4,7 +4,6 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; const executor = await TaskExecutor.create({ package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", yagnaOptions: { apiKey: "try_golem" }, - logLevel: "debug", }); const result = await executor.run(async (ctx) => { diff --git a/examples/docs-examples/examples/working-with-images/tag.mjs b/examples/docs-examples/examples/working-with-images/tag.mjs index 4a14209fa..aae331455 100644 --- a/examples/docs-examples/examples/working-with-images/tag.mjs +++ b/examples/docs-examples/examples/working-with-images/tag.mjs @@ -2,7 +2,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js"; (async () => { const executor = await TaskExecutor.create({ - package: "golem/my_example:latest", + package: "golem/alpine:latest", yagnaOptions: { appKey: "try_golem" }, }); diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 3e6bb188c..60c17954a 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -1,55 +1 @@ -[ - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.cjs" }, - { "cmd": "ts-node", "path": "examples/docs-examples/examples/composing-tasks/single-command.ts" }, - - { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/before-each.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/foreach.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/map.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/single-run.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs", "noGoth": true }, - { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/demand.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs", "noGoth": true }, - - { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-file.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, - - { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-json.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/examples/using-app-keys/index.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/hash.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-end.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs" }, - { - "cmd": "node", - "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs", - "skip": true - }, - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command-fail.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/quickstarts/quickstart/requestor.mjs" }, - - { "cmd": "node", "path": "examples/docs-examples/tutorials/building-custom-image/index.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/tutorials/quickstart/index.mjs" }, - { - "cmd": "node", - "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", - "args": ["--mask", "?a?a", "--hash", "$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0"], - "timeout": 500 - } -] +[{ "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs", "noGoth": false }] diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index 1df7125ff..35da1e6b9 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -10,7 +10,7 @@ const gothStartingTimeout = 180; const goth = new Goth(gothConfig); const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; -const criticalLogsRegExp = [/Task *. timeot/g, /Task *. has been rejected/g, /ERROR: TypeError/g, /ERROR: Error/g]; +const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/, /ERROR: Error/gim]; type Example = { cmd: string; @@ -35,12 +35,16 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 12 return new Promise((res, rej) => { spawnedExample.stdout?.on("data", (data: string) => { console.log(data.trim()); - if (criticalLogsRegExp.some((regexp) => data.match(regexp))) { + const logWithoutColours = data.replace( + /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, + "", + ); + if (criticalLogsRegExp.some((regexp) => logWithoutColours.match(regexp))) { error = `A critical error occurred during the test.`; spawnedExample.kill(); } // for some reason, sometimes the process doesn't exit after Executor shut down - if (data.indexOf("Task Executor has shut down") !== -1) { + if (logWithoutColours.indexOf("Task Executor has shut down") !== -1) { spawnedExample.kill("SIGKILL"); } }); From 3395e341b8775b6696eff6eb2b6a847f1a6da6eb Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 14:59:17 +0200 Subject: [PATCH 54/56] test(examples): fixed regexp matching --- tests/examples/examples.json | 56 +++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 60c17954a..3e6bb188c 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -1 +1,55 @@ -[{ "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs", "noGoth": false }] +[ + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.cjs" }, + { "cmd": "ts-node", "path": "examples/docs-examples/examples/composing-tasks/single-command.ts" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/before-each.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/foreach.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/map.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/single-run.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs", "noGoth": true }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/demand.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs", "noGoth": true }, + + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, + + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-json.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/using-app-keys/index.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/hash.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs" }, + { + "cmd": "node", + "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs", + "skip": true + }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command-fail.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/quickstarts/quickstart/requestor.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/tutorials/building-custom-image/index.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/tutorials/quickstart/index.mjs" }, + { + "cmd": "node", + "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", + "args": ["--mask", "?a?a", "--hash", "$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0"], + "timeout": 500 + } +] From 36c0e3acbe369ee558f48715ebe85e0b9e5ba8da Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 17:14:28 +0200 Subject: [PATCH 55/56] test(examples): fixed ci config --- ...{goth-example.yml => examples-nightly.yml} | 11 +++++----- .github/workflows/release.yml | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) rename .github/workflows/{goth-example.yml => examples-nightly.yml} (92%) diff --git a/.github/workflows/goth-example.yml b/.github/workflows/examples-nightly.yml similarity index 92% rename from .github/workflows/goth-example.yml rename to .github/workflows/examples-nightly.yml index cc03e8a98..6bf5229f2 100644 --- a/.github/workflows/goth-example.yml +++ b/.github/workflows/examples-nightly.yml @@ -1,13 +1,13 @@ -name: Nightly Example Tests +name: Nightly Examples Tests on: + schedule: + # run this workflow every day at 2:00 AM UTC + - cron: "0 4 * * *" + # Allows triggering the workflow manually workflow_dispatch: - push: - branches: - - "test/JST-363/example-tests" - jobs: prepare-matrix-master-only: name: Prepare matrix JSON @@ -56,6 +56,7 @@ jobs: rm -rf ../goth/assets python -m goth create-assets ../goth/assets sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml + sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc21"/mg' ../goth/assets/goth-config.yml sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile - name: Disconnect Docker containers from default network diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06c9006b4..20f0e47cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,6 +161,26 @@ jobs: run: rm -rf .cypress #endregion + #region Examples test execution + - name: Run the Examples tests using Goth + env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npm run test:examples + + - name: Upload test logs + uses: actions/upload-artifact@v2 + if: always() + with: + name: goth-logs + path: /tmp/goth-tests + + # Only relevant for self-hosted runners + - name: Remove test logs + if: always() + run: rm -rf /tmp/goth-tests + #endregion + release: name: Release the SDK to NPM and GitHub needs: run-integration-and-e2e-tests From 6aa1e917b480c675e66772b58a092b523e3df21d Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Mon, 25 Sep 2023 17:15:17 +0200 Subject: [PATCH 56/56] test(examples): fixed ci config --- .github/workflows/examples-nightly.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 6bf5229f2..f7e5ab6a8 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -8,6 +8,10 @@ on: # Allows triggering the workflow manually workflow_dispatch: + push: + branches: + - "test/JST-363/example-tests" + jobs: prepare-matrix-master-only: name: Prepare matrix JSON