diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml new file mode 100644 index 000000000..f7e5ab6a8 --- /dev/null +++ b/.github/workflows/examples-nightly.yml @@ -0,0 +1,101 @@ +name: Nightly Examples Tests + +on: + schedule: + # run this workflow every day at 2:00 AM UTC + - cron: "0 4 * * *" + + # Allows triggering the workflow manually + workflow_dispatch: + + push: + branches: + - "test/JST-363/example-tests" + +jobs: + prepare-matrix-master-only: + name: Prepare matrix JSON + runs-on: ubuntu-latest + outputs: + matrix-json: ${{ steps.get-matrix.outputs.matrix }} + steps: + # prepares JSON object representing strategy matrix with only the master branch + - name: Get matrix JSON + id: get-matrix + run: echo "::set-output name=matrix::{\"include\":[{\"branch\":\"master\"}]}" + + goth-tests: + runs-on: goth + needs: prepare-matrix-master-only + strategy: + matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }} + fail-fast: false + name: Run example tests (nightly) on ${{ matrix.branch }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Configure node.js + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Build golem-js + run: | + sudo apt-get update -y + sudo apt-get install -y build-essential + npm install + npm run build + npm install --prefix examples + npm install ts-node + + - name: Configure python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install goth + run: | + pip install goth>=0.15.3 + rm -rf ../goth/assets + python -m goth create-assets ../goth/assets + sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml + sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc21"/mg' ../goth/assets/goth-config.yml + sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile + + - name: Disconnect Docker containers from default network + continue-on-error: true + # related to this issue: https://github.com/moby/moby/issues/23302 + run: | + docker network inspect docker_default + sudo apt-get install -y jq + docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default + + - name: Remove Docker containers + continue-on-error: true + run: docker rm -f $(docker ps -a -q) + + - name: Restart Docker daemon + # related to this issue: https://github.com/moby/moby/issues/23302 + run: sudo systemctl restart docker + + - name: Log in to GitHub Docker repository + run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin + + - name: Run test suite + env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npm run test:examples + + - name: Upload test logs + uses: actions/upload-artifact@v2 + if: always() + with: + name: goth-logs + path: /tmp/goth-tests + + # Only relevant for self-hosted runners + - name: Remove test logs + if: always() + run: rm -rf /tmp/goth-tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 06c9006b4..20f0e47cd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,6 +161,26 @@ jobs: run: rm -rf .cypress #endregion + #region Examples test execution + - name: Run the Examples tests using Goth + env: + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npm run test:examples + + - name: Upload test logs + uses: actions/upload-artifact@v2 + if: always() + with: + name: goth-logs + path: /tmp/goth-tests + + # Only relevant for self-hosted runners + - name: Remove test logs + if: always() + run: rm -rf /tmp/goth-tests + #endregion + release: name: Release the SDK to NPM and GitHub needs: run-integration-and-e2e-tests diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs new file mode 100644 index 000000000..8a5a9ebd0 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/batch-end.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); // to be removed and replaced with try & catch + + return res[2]?.stdout; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs new file mode 100644 index 000000000..83db435de --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs @@ -0,0 +1,22 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + res.on("data", (data) => (data.index == 2 ? console.log(data.stdout) : "")); + res.on("error", (error) => console.error(error)); + res.on("close", () => executor.end()); + }); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs new file mode 100644 index 000000000..95606d5c5 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + for await (const chunk of res) { + console.log(chunk.stdout); + } + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs new file mode 100644 index 000000000..acfd7a9e8 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs @@ -0,0 +1,20 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs"); + await ctx.run("node /golem/input/worker.mjs > /golem/input/output.txt"); + const result = await ctx.run("cat /golem/input/output.txt"); + await ctx.downloadFile("/golem/input/output.txt", "./output.txt"); + return result.stdout; + }); + + console.log(result); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/single-command.cjs b/examples/docs-examples/examples/composing-tasks/single-command.cjs new file mode 100644 index 000000000..79808a8ae --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/single-command.cjs @@ -0,0 +1,13 @@ +const { TaskExecutor } = require("@golem-sdk/golem-js"); + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/single-command.mjs b/examples/docs-examples/examples/composing-tasks/single-command.mjs new file mode 100644 index 000000000..63717003c --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/single-command.mjs @@ -0,0 +1,13 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/single-command.ts b/examples/docs-examples/examples/composing-tasks/single-command.ts new file mode 100644 index 000000000..ad1d6a5a2 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/single-command.ts @@ -0,0 +1,13 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/composing-tasks/worker.mjs b/examples/docs-examples/examples/composing-tasks/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/examples/docs-examples/examples/executing-tasks/action_log.txt b/examples/docs-examples/examples/executing-tasks/action_log.txt new file mode 100644 index 000000000..bb5749a1f --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/action_log.txt @@ -0,0 +1 @@ +some action log \ No newline at end of file diff --git a/examples/docs-examples/examples/executing-tasks/before-each.mjs b/examples/docs-examples/examples/executing-tasks/before-each.mjs new file mode 100644 index 000000000..31913ed4c --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/before-each.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + maxParallelTasks: 3, + }); + + executor.beforeEach(async (ctx) => { + console.log(ctx.provider.name + " is downloading action_log file"); + await ctx.uploadFile("./action_log.txt", "/golem/input/action_log.txt"); + }); + + await executor.forEach([1, 2, 3, 4, 5], async (ctx, item) => { + await ctx + .beginBatch() + .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`) + .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt") + .end(); + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/foreach.mjs b/examples/docs-examples/examples/executing-tasks/foreach.mjs new file mode 100644 index 000000000..e1cbd7fc1 --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/foreach.mjs @@ -0,0 +1,16 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const data = [1, 2, 3, 4, 5]; + + await executor.forEach(data, async (ctx, item) => { + console.log((await ctx.run(`echo "${item}"`)).stdout); + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/map.mjs b/examples/docs-examples/examples/executing-tasks/map.mjs new file mode 100644 index 000000000..9fc089987 --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/map.mjs @@ -0,0 +1,16 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const data = [1, 2, 3, 4, 5]; + + const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); + + for await (const result of results) console.log(result.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs b/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs new file mode 100644 index 000000000..0e1a11572 --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs @@ -0,0 +1,17 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + maxParallelTasks: 3, // default is 5 + }); + + const data = [1, 2, 3, 4, 5]; + + const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`)); + + for await (const result of results) console.log(result.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/executing-tasks/single-run.mjs b/examples/docs-examples/examples/executing-tasks/single-run.mjs new file mode 100644 index 000000000..1f831abee --- /dev/null +++ b/examples/docs-examples/examples/executing-tasks/single-run.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/custom-price.mjs b/examples/docs-examples/examples/selecting-providers/custom-price.mjs new file mode 100644 index 000000000..ce6db0f37 --- /dev/null +++ b/examples/docs-examples/examples/selecting-providers/custom-price.mjs @@ -0,0 +1,39 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +/** + * Example demonstrating how to write a custom proposal filter. + */ + +var costData = []; + +const myFilter = async (proposal) => { + let decision = false; + let usageVector = proposal.properties["golem.com.usage.vector"]; + let counterIdx = usageVector.findIndex((ele) => ele === "golem.usage.duration_sec"); + let proposedCost = proposal.properties["golem.com.pricing.model.linear.coeffs"][counterIdx]; + costData.push(proposedCost); + if (costData.length < 11) return false; + else { + costData.shift(); + let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 10; + if (proposedCost <= averageProposedCost) decision = true; + if (decision) { + console.log(proposedCost, averageProposedCost); + } + } + console.log(costData); + console.log(proposal.properties["golem.node.id.name"], proposal.properties["golem.com.pricing.model.linear.coeffs"]); + return decision; +}; + +(async function main() { + const executor = await TaskExecutor.create({ + package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + proposalFilter: myFilter, + yagnaOptions: { apiKey: "try_golem" }, + }); + await executor.run(async (ctx) => + console.log((await ctx.run(`echo "This task is run on ${ctx.provider.id}"`)).stdout, ctx.provider.id), + ); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/demand.mjs b/examples/docs-examples/examples/selecting-providers/demand.mjs new file mode 100644 index 000000000..acd734f83 --- /dev/null +++ b/examples/docs-examples/examples/selecting-providers/demand.mjs @@ -0,0 +1,15 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async function main() { + const executor = await TaskExecutor.create({ + package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + //minCpuCores : 2, + //minMemGib : 8, + //minStorageGib: 10, + minCpuThreads: 1, + yagnaOptions: { apiKey: "try_golem" }, + }); + + await executor.run(async (ctx) => console.log((await ctx.run("echo 'Hello World'")).stdout)); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/whitelist.mjs b/examples/docs-examples/examples/selecting-providers/whitelist.mjs new file mode 100644 index 000000000..202c8c454 --- /dev/null +++ b/examples/docs-examples/examples/selecting-providers/whitelist.mjs @@ -0,0 +1,24 @@ +import { TaskExecutor, ProposalFilters } from "@golem-sdk/golem-js"; + +/** + * Example demonstrating how to use the predefined filter `whiteListProposalIdsFilter`, + * which only allows offers from a provider whose ID is in the array + */ + +const whiteListIds = ["0x3fc1d65ddc5258dc8807df30a29f71028e965e1b", "0x4506550de84d207f3ab90add6336f68119015836"]; +console.log("Will accept only proposals from:"); +for (let i = 0; i < whiteListIds.length; i++) { + console.log(whiteListIds[i]); +} + +(async function main() { + const executor = await TaskExecutor.create({ + package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", + proposalFilter: ProposalFilters.whiteListProposalIdsFilter(whiteListIds), + yagnaOptions: { apiKey: "try_golem" }, + }); + await executor.run(async (ctx) => + console.log((await ctx.run(`echo "This task is run on ${ctx.provider.id}"`)).stdout, ctx.provider.id), + ); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/downloading-file.mjs b/examples/docs-examples/examples/sending-data/downloading-file.mjs new file mode 100644 index 000000000..2a44bc080 --- /dev/null +++ b/examples/docs-examples/examples/sending-data/downloading-file.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .run("ls -l /golem > /golem/work/output.txt") + .run("cat /golem/work/output.txt") + .downloadFile("/golem/work/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); + + return res[2]?.stdout; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/uploading-file.mjs b/examples/docs-examples/examples/sending-data/uploading-file.mjs new file mode 100644 index 000000000..dadc3540a --- /dev/null +++ b/examples/docs-examples/examples/sending-data/uploading-file.mjs @@ -0,0 +1,28 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { createHash } from "node:crypto"; +import * as fs from "fs"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const buff = fs.readFileSync("worker.mjs"); + const hash = createHash("md5").update(buff).digest("hex"); + + const result = await executor.run(async (ctx) => { + await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs"); + + const res = await ctx.run( + `node -e "const crypto = require('node:crypto'); const fs = require('fs'); const buff = fs.readFileSync('/golem/input/worker.mjs'); const hash = crypto.createHash('md5').update(buff).digest('hex'); console.log(hash); "`, + ); + + return res.stdout; + }); + + console.log("md5 of the file sent to provider: ", result); + console.log("Locally computed md5: ", hash); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/uploading-json.mjs b/examples/docs-examples/examples/sending-data/uploading-json.mjs new file mode 100644 index 000000000..5b107da72 --- /dev/null +++ b/examples/docs-examples/examples/sending-data/uploading-json.mjs @@ -0,0 +1,21 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const output = await executor.run(async (ctx) => { + // Upload test JSON object + + await ctx.uploadJson({ input: "Hello World" }, "/golem/work/input.json"); + + // Read the content of the JSON object. + return await ctx.run("cat /golem/work/input.json"); + }); + + console.log(output.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/sending-data/worker.mjs b/examples/docs-examples/examples/sending-data/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/sending-data/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs b/examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs new file mode 100644 index 000000000..3c8a2ab70 --- /dev/null +++ b/examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs @@ -0,0 +1,15 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + subnetTag: "public", // do we need to show subnet ?? + payment: { driver: "erc-20", network: "polygon" }, + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/transferring-data/download-file.mjs b/examples/docs-examples/examples/transferring-data/download-file.mjs new file mode 100644 index 000000000..2a44bc080 --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/download-file.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .run("ls -l /golem > /golem/work/output.txt") + .run("cat /golem/work/output.txt") + .downloadFile("/golem/work/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); + + return res[2]?.stdout; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/transferring-data/transferDatainBrowser.html b/examples/docs-examples/examples/transferring-data/transferDatainBrowser.html new file mode 100644 index 000000000..8962ef6c9 --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/transferDatainBrowser.html @@ -0,0 +1,122 @@ + + + + + WebRequestor Task API + + +

WebRequestor - Meme Example

+
+
+

Input data

+
+
+ + +
+
+ + +
+
+

Actions

+
+
+ +
+
+
+

Result Meme

+ +
+
+
+
+

Logs

+
    +
    +
    +
    + + + + diff --git a/examples/docs-examples/examples/transferring-data/upload-file.mjs b/examples/docs-examples/examples/transferring-data/upload-file.mjs new file mode 100644 index 000000000..dadc3540a --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/upload-file.mjs @@ -0,0 +1,28 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { createHash } from "node:crypto"; +import * as fs from "fs"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const buff = fs.readFileSync("worker.mjs"); + const hash = createHash("md5").update(buff).digest("hex"); + + const result = await executor.run(async (ctx) => { + await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs"); + + const res = await ctx.run( + `node -e "const crypto = require('node:crypto'); const fs = require('fs'); const buff = fs.readFileSync('/golem/input/worker.mjs'); const hash = crypto.createHash('md5').update(buff).digest('hex'); console.log(hash); "`, + ); + + return res.stdout; + }); + + console.log("md5 of the file sent to provider: ", result); + console.log("Locally computed md5: ", hash); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/transferring-data/upload-json.mjs b/examples/docs-examples/examples/transferring-data/upload-json.mjs new file mode 100644 index 000000000..5b107da72 --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/upload-json.mjs @@ -0,0 +1,21 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const output = await executor.run(async (ctx) => { + // Upload test JSON object + + await ctx.uploadJson({ input: "Hello World" }, "/golem/work/input.json"); + + // Read the content of the JSON object. + return await ctx.run("cat /golem/work/input.json"); + }); + + console.log(output.stdout); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html b/examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html new file mode 100644 index 000000000..3def7232f --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/uploadHSONinBrowser.html @@ -0,0 +1,75 @@ + + + + + Golem JSON App + + + +

    JSON upload and download

    +
    +
    +

    Actions

    +
    +
    + +
    +
    +
    +

    Results

    +
      +
      +
      +
      +
      +

      Logs

      +
        +
        +
        +
        + + diff --git a/examples/docs-examples/examples/transferring-data/worker.mjs b/examples/docs-examples/examples/transferring-data/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/transferring-data/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/examples/docs-examples/examples/using-app-keys/index.mjs b/examples/docs-examples/examples/using-app-keys/index.mjs new file mode 100644 index 000000000..c5adf9a59 --- /dev/null +++ b/examples/docs-examples/examples/using-app-keys/index.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + // replace 'try_golem' with 'insert-your-32-char-app-key-here' + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/working-with-images/hash.mjs b/examples/docs-examples/examples/working-with-images/hash.mjs new file mode 100644 index 000000000..1f831abee --- /dev/null +++ b/examples/docs-examples/examples/working-with-images/hash.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/working-with-images/tag.mjs b/examples/docs-examples/examples/working-with-images/tag.mjs new file mode 100644 index 000000000..aae331455 --- /dev/null +++ b/examples/docs-examples/examples/working-with-images/tag.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "golem/alpine:latest", + yagnaOptions: { appKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/examples/working-with-results/multi-command-end.mjs b/examples/docs-examples/examples/working-with-results/multi-command-end.mjs new file mode 100644 index 000000000..b821d2272 --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/multi-command-end.mjs @@ -0,0 +1,25 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + logLevel: "debug", + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .end() + .catch((error) => console.error(error)); + + return res; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs b/examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs new file mode 100644 index 000000000..9d95b9474 --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs @@ -0,0 +1,22 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + res.on("data", (result) => console.log(result)); + res.on("error", (error) => console.error(error)); + res.on("close", () => executor.end()); + }); +})(); diff --git a/examples/docs-examples/examples/working-with-results/multi-command-fail.mjs b/examples/docs-examples/examples/working-with-results/multi-command-fail.mjs new file mode 100644 index 000000000..e2633ba6d --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/multi-command-fail.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .run("cat /golem/input/output.txt > /golem/input/output.txt") + .downloadFile("/golem/output/output.txt", "./output.txt") // there is no such file in output folder + .run("ls -l /golem/") + .end() + .catch((error) => console.error(error)); + + return res; + }); + + console.log(result); + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/single-command-fail.mjs b/examples/docs-examples/examples/working-with-results/single-command-fail.mjs new file mode 100644 index 000000000..d8a523fbb --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/single-command-fail.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + // there is a mistake and instead of 'node -v' we call 'node -w' + const result = await executor.run(async (ctx) => await ctx.run("node -w")); + console.log("Task result:", result); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/single-command.mjs b/examples/docs-examples/examples/working-with-results/single-command.mjs new file mode 100644 index 000000000..7a48cebf6 --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/single-command.mjs @@ -0,0 +1,13 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => await ctx.run("node -v")); + console.log("Task result:", result); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/working-with-results/worker.mjs b/examples/docs-examples/examples/working-with-results/worker.mjs new file mode 100644 index 000000000..accefceba --- /dev/null +++ b/examples/docs-examples/examples/working-with-results/worker.mjs @@ -0,0 +1 @@ +console.log("Hello World"); diff --git a/examples/docs-examples/quickstarts/quickstart/requestor.mjs b/examples/docs-examples/quickstarts/quickstart/requestor.mjs new file mode 100644 index 000000000..92fe00e9b --- /dev/null +++ b/examples/docs-examples/quickstarts/quickstart/requestor.mjs @@ -0,0 +1,11 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + await executor.end(); + + console.log("Task result:", result); +})(); diff --git a/examples/docs-examples/quickstarts/web-quickstart/index.html b/examples/docs-examples/quickstarts/web-quickstart/index.html new file mode 100644 index 000000000..2b899c237 --- /dev/null +++ b/examples/docs-examples/quickstarts/web-quickstart/index.html @@ -0,0 +1,31 @@ + + + + + WebRequestor QuickStart + + +

        WebRequestor - QuickStart

        +
        +
        +

        Actions

        +
        +
        + +
        +
        +
        +

        Results

        +
          +
          +
          +
          +
          +

          Logs

          +
            +
            +
            +
            + + + diff --git a/examples/docs-examples/quickstarts/web-quickstart/requestor.mjs b/examples/docs-examples/quickstarts/web-quickstart/requestor.mjs new file mode 100644 index 000000000..c9716d7c1 --- /dev/null +++ b/examples/docs-examples/quickstarts/web-quickstart/requestor.mjs @@ -0,0 +1,40 @@ +import * as golem from "https://unpkg.com/@golem-sdk/golem-js"; + +function appendResults(result) { + const results = document.getElementById("results"); + const div = document.createElement("div"); + div.appendChild(document.createTextNode(result)); + results.appendChild(div); +} + +function appendLog(msg, level = "info") { + const logs = document.getElementById("logs"); + const div = document.createElement("div"); + div.appendChild(document.createTextNode(`[${new Date().toISOString()}] [${level}] ${msg}`)); + logs.appendChild(div); +} + +const logger = { + log: (msg) => appendLog(msg), + warn: (msg) => appendLog(msg, "warn"), + debug: (msg) => appendLog(msg, "debug"), + error: (msg) => appendLog(msg, "error"), + info: (msg) => appendLog(msg, "info"), + table: (msg) => appendLog(JSON.stringify(msg, null, "\t")), +}; + +async function run() { + const executor = await golem.TaskExecutor.create({ + package: "dcd99a5904bebf7ca655a833b73cc42b67fd40b4a111572e3d2007c3", + yagnaOptions: { apiKey: "try_golem" }, + logger, + }).catch((e) => logger.error(e)); + + await executor + .run(async (ctx) => appendResults((await ctx.run("echo 'Hello World'")).stdout)) + .catch((e) => logger.error(e)); + + await executor.end(); +} + +document.getElementById("echo").onclick = run; diff --git a/examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile b/examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile new file mode 100644 index 000000000..d0de19d0f --- /dev/null +++ b/examples/docs-examples/tool-examples/converting-Docker-image/Dockerfile @@ -0,0 +1,3 @@ +FROM debian:latest +VOLUME /golem/input /golem/output +WORKDIR /golem/work diff --git a/examples/docs-examples/tutorials/building-custom-image/Dockerfile b/examples/docs-examples/tutorials/building-custom-image/Dockerfile new file mode 100644 index 000000000..d29ee22e0 --- /dev/null +++ b/examples/docs-examples/tutorials/building-custom-image/Dockerfile @@ -0,0 +1,5 @@ +FROM node:latest +WORKDIR /golem/work +VOLUME /golem/work +COPY Dockerfile /golem/info/description.txt +COPY Dockerfile /golem/work/info.txt \ No newline at end of file diff --git a/examples/docs-examples/tutorials/building-custom-image/index.mjs b/examples/docs-examples/tutorials/building-custom-image/index.mjs new file mode 100644 index 000000000..b75048699 --- /dev/null +++ b/examples/docs-examples/tutorials/building-custom-image/index.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +(async () => { + const executor = await TaskExecutor.create({ + package: "8b238595299444d0733b41095f27fadd819a71d29002b614c665b27c", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + console.log("Description.txt: ", (await ctx.run("cat /golem/info/description.txt")).stdout); + console.log("/golem/work content: ", (await ctx.run("ls /golem/work")).stdout); + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/tutorials/quickstart/index.mjs b/examples/docs-examples/tutorials/quickstart/index.mjs new file mode 100644 index 000000000..5e4fcebe6 --- /dev/null +++ b/examples/docs-examples/tutorials/quickstart/index.mjs @@ -0,0 +1,14 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; + +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const taskResult = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout); + + await executor.end(); + + console.log("Task result:", taskResult); +})(); diff --git a/examples/docs-examples/tutorials/running-from-browser/index.html b/examples/docs-examples/tutorials/running-from-browser/index.html new file mode 100644 index 000000000..55bfa0485 --- /dev/null +++ b/examples/docs-examples/tutorials/running-from-browser/index.html @@ -0,0 +1,68 @@ + + + + + WebRequestor Task API + + +

            WebRequestor - Hello World

            +
            +
            +

            Actions

            +
            +
            + +
            +
            +
            +

            Results

            +
              +
              +
              +
              +
              +

              Logs

              +
                +
                +
                +
                + + + + diff --git a/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs new file mode 100644 index 000000000..6342bf373 --- /dev/null +++ b/examples/docs-examples/tutorials/running-parallel-tasks/index.mjs @@ -0,0 +1,57 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +import { program } from "commander"; + +async function main(args) { + const executor = await TaskExecutor.create({ + package: "055911c811e56da4d75ffc928361a78ed13077933ffa8320fb1ec2db", + maxParallelTasks: args.numberOfProviders, + yagnaOptions: { apiKey: `try_golem` }, + }); + + const keyspace = await executor.run(async (ctx) => { + const result = await ctx.run(`hashcat --keyspace -a 3 ${args.mask} -m 400`); + return parseInt(result.stdout || ""); + }); + + if (!keyspace) throw new Error(`Cannot calculate keyspace`); + + console.log(`Keyspace size computed. Keyspace size = ${keyspace}.`); + const step = Math.floor(keyspace / args.numberOfProviders + 1); + const range = [...Array(Math.floor(keyspace / step) + 1).keys()].map((i) => i * step); + + const results = executor.map(range, async (ctx, skip = 0) => { + const results = await ctx + .beginBatch() + .run( + `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min( + keyspace, + skip + step, + )} -o pass.potfile`, + ) + .run("cat pass.potfile") + .end() + .catch((err) => console.error(err)); + if (!results?.[1]?.stdout) return false; + return results?.[1]?.stdout.toString().split(":")[1]; + }); + + let password = ""; + for await (const result of results) { + if (result) { + password = result; + break; + } + } + + if (!password) console.log("No password found"); + else console.log(`Password found: ${password}`); + await executor.end(); +} + +program + .option("--number-of-providers ", "number of providers", (value) => parseInt(value), 3) + .option("--mask ") + .requiredOption("--hash "); +program.parse(); +const options = program.opts(); +main(options).catch((e) => console.error(e)); diff --git a/package.json b/package.json index e6003ea51..af430c82f 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,8 @@ "test:e2e": "jest --config tests/e2e/jest.config.json tests/e2e/**.spec.ts --runInBand --forceExit", "test:e2e:no-goth": "jest tests/e2e/**.spec.ts --testTimeout=180000 --runInBand --forceExit", "test:cypress": "cypress run", + "test:examples": "ts-node --project tsconfig.spec.json tests/examples/examples.test.ts", + "test:examples:no-goth": "ts-node --project tsconfig.spec.json tests/examples/examples.test.ts --no-goth", "lint": "npm run lint:ts && npm run lint:ts:tests && npm run lint:eslint", "lint:ts": "tsc --project tsconfig.json --noEmit", "lint:ts:tests": "tsc --project tests/tsconfig.json --noEmit", diff --git a/src/storage/gftp.ts b/src/storage/gftp.ts index 4e1d73a81..777a542f3 100644 --- a/src/storage/gftp.ts +++ b/src/storage/gftp.ts @@ -114,7 +114,7 @@ export class GftpStorageProvider implements StorageProvider { return; } - await this.jsonrpc("close", { urls: Array.from(this.publishedUrls) }); + await this.jsonrpc("close", { urls: Array.from(this.publishedUrls).filter((url) => !!url) }); } async close() { diff --git a/tests/examples/examples.json b/tests/examples/examples.json new file mode 100644 index 000000000..3e6bb188c --- /dev/null +++ b/tests/examples/examples.json @@ -0,0 +1,55 @@ +[ + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.cjs" }, + { "cmd": "ts-node", "path": "examples/docs-examples/examples/composing-tasks/single-command.ts" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/before-each.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/foreach.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/map.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/single-run.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs", "noGoth": true }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/demand.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs", "noGoth": true }, + + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, + + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-json.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/using-app-keys/index.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/hash.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-end.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs" }, + { + "cmd": "node", + "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs", + "skip": true + }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/single-command-fail.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/quickstarts/quickstart/requestor.mjs" }, + + { "cmd": "node", "path": "examples/docs-examples/tutorials/building-custom-image/index.mjs" }, + { "cmd": "node", "path": "examples/docs-examples/tutorials/quickstart/index.mjs" }, + { + "cmd": "node", + "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", + "args": ["--mask", "?a?a", "--hash", "$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0"], + "timeout": 500 + } +] diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts new file mode 100644 index 000000000..35da1e6b9 --- /dev/null +++ b/tests/examples/examples.test.ts @@ -0,0 +1,105 @@ +import { spawn } from "child_process"; +import { dirname, basename, resolve } from "path"; +import { Goth } from "../goth/goth"; +import chalk from "chalk"; +import testExamples from "./examples.json"; + +const noGoth = process.argv[2] === "--no-goth"; +const gothConfig = resolve("../goth/assets/goth-config.yml"); +const gothStartingTimeout = 180; +const goth = new Goth(gothConfig); + +const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; +const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/, /ERROR: Error/gim]; + +type Example = { + cmd: string; + path: string; + args?: string[]; + timeout?: number; + noGoth?: boolean; + skip?: boolean; +}; + +async function test(cmd: string, path: string, args: string[] = [], timeout = 120) { + const file = basename(path); + const cwd = dirname(path); + const spawnedExample = spawn(cmd, [file, ...args], { cwd }); + spawnedExample.stdout?.setEncoding("utf-8"); + spawnedExample.stderr?.setEncoding("utf-8"); + let error = ""; + const timeoutId = setTimeout(() => { + error = `Test timeout was reached after ${timeout} seconds.`; + spawnedExample.kill(); + }, timeout * 1000); + return new Promise((res, rej) => { + spawnedExample.stdout?.on("data", (data: string) => { + console.log(data.trim()); + const logWithoutColours = data.replace( + /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, + "", + ); + if (criticalLogsRegExp.some((regexp) => logWithoutColours.match(regexp))) { + error = `A critical error occurred during the test.`; + spawnedExample.kill(); + } + // for some reason, sometimes the process doesn't exit after Executor shut down + if (logWithoutColours.indexOf("Task Executor has shut down") !== -1) { + spawnedExample.kill("SIGKILL"); + } + }); + spawnedExample.stderr?.on("data", (data: string) => console.log(data.trim())); + spawnedExample.on("close", (code) => { + if (!error && !code) return res(true); + rej(`Test example "${file}" failed. ${error}`); + }); + }).finally(() => { + clearTimeout(timeoutId); + spawnedExample.kill("SIGKILL"); + }); +} + +async function testAll(examples: Example[]) { + const failedTests = new Set(); + const skippedTests = new Set(); + if (!noGoth) + await Promise.race([ + goth.start(), + new Promise((res, rej) => + setTimeout( + () => rej(new Error(`The Goth starting timeout was reached after ${gothStartingTimeout} seconds`)), + gothStartingTimeout * 1000, + ), + ), + ]); + for (const example of examples) { + try { + console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); + if (example?.skip) { + console.log(chalk.bgYellow.black(" SKIP "), chalk.yellowBright(example.path)); + skippedTests.add(example.path); + } else { + await test(example.cmd, example.path, example.args, example.timeout); + console.log(chalk.bgGreen.white(" PASS "), chalk.green(example.path)); + } + } catch (error) { + console.log(chalk.bgRed.white(" FAIL "), chalk.red(error)); + failedTests.add(example.path); + } + } + if (!noGoth) await goth.end().catch((error) => console.error(error)); + console.log( + chalk.bold.yellow("\n\nTESTS RESULTS: "), + chalk.bgGreen.black(` ${examples.length - failedTests.size - skippedTests.size} passed `), + chalk.bgRed.black(` ${failedTests.size} failed `), + skippedTests.size ? chalk.bgYellow.black(` ${skippedTests.size} skipped `) : "", + chalk.bgCyan.black(` ${examples.length} total `), + ); + console.log(chalk.red("\nFailed tests:")); + failedTests.forEach((test) => console.log(chalk.red(`\t- ${test}`))); + console.log(chalk.yellow("\nSkipped tests:")); + skippedTests.forEach((test) => console.log(chalk.yellow(`\t- ${test}`))); + process.exit(failedTests.size > 0 ? 1 : 0); +} + +testAll(examples).then(); diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts index 504d6ddf5..8fea42ca4 100644 --- a/tests/goth/goth.ts +++ b/tests/goth/goth.ts @@ -34,8 +34,6 @@ export class Goth { process.env["GSB_URL"] = gsbUrl; process.env["PATH"] = `${path}:${process.env["PATH"]}`; process.env["YAGNA_SUBNET"] = subnetTag; - // Note: rinkeby is a test network which is dead, but our goth runners exist on a custom deployment of this network - process.env["PAYMENT_NETWORK"] = "rinkeby"; const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; diff --git a/tests/tsconfig.json b/tests/tsconfig.json index f71bd877a..882c3e83d 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -3,6 +3,7 @@ "include": ["."], "exclude": ["./cypress"], "compilerOptions": { - "types": ["jest"] + "types": ["jest"], + "resolveJsonModule": true } } diff --git a/tsconfig.spec.json b/tsconfig.spec.json old mode 100755 new mode 100644 index 3d6abe8a5..50f9904d3 --- a/tsconfig.spec.json +++ b/tsconfig.spec.json @@ -15,8 +15,15 @@ "skipLibCheck": true, "lib": ["es2015", "es2016", "es2017", "es2018", "esnext", "dom"], "outDir": "dist", - "typeRoots": ["node_modules/@types"] + "typeRoots": ["node_modules/@types"], + "resolveJsonModule": true }, "exclude": ["dist", "node_modules", "examples", "cypress.config.ts"], - "include": ["src/**/*.spec.ts", "tests/unit/**/*.test.ts", "tests/unit/**/*.spec.ts"] + "include": ["src/**/*.spec.ts", "tests/unit/**/*.test.ts", "tests/unit/**/*.spec.ts"], + "ts-node": { + "esm": true, + "compilerOptions": { + "module": "nodenext" + } + } }