Merge branch 'beta' into feature/JST-526

# Conflicts:
#	examples/blender/blender.ts
#	examples/docs-examples/tutorials/running-parallel-tasks/index.mjs
#	examples/fibonacci/fibonacci.js
#	examples/package-lock.json
#	examples/simple-usage/forEach.ts
#	examples/simple-usage/map.ts
#	examples/ssh/ssh.ts
#	examples/yacat/yacat.ts
#	tests/e2e/yacat.spec.ts

.eslintrc

@@ -24,5 +24,5 @@
       "extends": ["eslint:recommended"]
     }
   ],
-  "ignorePatterns": ["dist/", "handbook_gen/", "docs/", "bundle.js", "tests", "*.config.ts"]
+  "ignorePatterns": ["dist/", "handbook_gen/", "docs/", "bundle.js", "tests", "*.config.ts", "tmp/"]
 }

.github/dependabot.yml

@@ -7,5 +7,6 @@ version: 2
 updates:
   - package-ecosystem: "npm" # See documentation for possible values
     directory: "/" # Location of package manifests
+    target-branch: "beta"
     schedule:
       interval: "weekly"

.github/workflows/cypress-nightly.yml

@@ -16,15 +16,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

.github/workflows/examples-nightly.yml

@@ -32,15 +32,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

.github/workflows/goth-nightly.yml

@@ -32,15 +32,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.
@@ -51,7 +51,7 @@ jobs:
           docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run build"
 
       - name: Start the e2e test
-        run: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run test:e2e -- --reporters github-actions --reporters summary"
+        run: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run test:e2e"
 
       - name: Cleanup Docker
         if: always()

.github/workflows/release.yml

@@ -55,15 +55,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

.gitignore

@@ -32,3 +32,4 @@ logs/
 
 #Nuxt
 /examples/adv-web-example/.nuxt
+tmp/

.npmignore

@@ -6,6 +6,7 @@
 /node_modules
 /src
 /tests
+/tmp
 .github
 .gitignore
 package-lock.json

.prettierignore

@@ -1,2 +1,3 @@
 dist/
-*.gvmi
+*.gvmi
+tmp/

examples/blender/blender.ts

@@ -35,17 +35,21 @@ async function main(subnetTag: string, driver?: string, network?: string, debug?
     await ctx.uploadFile(`${__dirname}/cubes.blend`, "/golem/resource/scene.blend");
   });
 
-  const results = executor.map<number, string>([0, 10, 20, 30, 40, 50], async (ctx, frame) => {
-    const result = await ctx
-      .beginBatch()
-      .uploadJson(blender_params(frame), "/golem/work/params.json")
-      .run("/golem/entrypoints/run-blender.sh")
-      .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`)
-      .end()
-      .catch((e) => console.error(e));
-    return result?.length ? `output_${frame}.png` : "";
-  });
-  for await (const result of results) console.log(result);
+  const futureResults = [0, 10, 20, 30, 40, 50].map((frame) =>
+    executor.run(async (ctx) => {
+      const result = await ctx
+        .beginBatch()
+        .uploadJson(blender_params(frame), "/golem/work/params.json")
+        .run("/golem/entrypoints/run-blender.sh")
+        .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`)
+        .end()
+        .catch((e) => console.error(e));
+      return result?.length ? `output_${frame}.png` : "";
+    }),
+  );
+  const results = await Promise.all(futureResults);
+  results.forEach((result) => console.log(result));
+
   await executor.shutdown();
 }
 

examples/docs-examples/examples/executing-tasks/action_log.txt

@@ -1 +1 @@
-some action log
+some action log

examples/docs-examples/examples/executing-tasks/before-each.mjs

@@ -12,13 +12,18 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     await ctx.uploadFile("./action_log.txt", "/golem/input/action_log.txt");
   });
 
-  await executor.forEach([1, 2, 3, 4, 5], async (ctx, item) => {
-    await ctx
-      .beginBatch()
-      .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`)
-      .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt")
-      .end();
+  const inputs = [1, 2, 3, 4, 5];
+
+  const futureResults = inputs.map(async (item) => {
+    return await executor.run(async (ctx) => {
+      await ctx
+        .beginBatch()
+        .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`)
+        .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt")
+        .end();
+    });
   });
+  await Promise.all(futureResults);
 
   await executor.shutdown();
 })();

examples/docs-examples/examples/executing-tasks/foreach.mjs

@@ -8,9 +8,11 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const data = [1, 2, 3, 4, 5];
 
-  await executor.forEach(data, async (ctx, item) => {
-    console.log((await ctx.run(`echo "${item}"`)).stdout);
-  });
+  for (const item of data) {
+    await executor.run(async (ctx) => {
+      console.log((await ctx.run(`echo "${item}"`)).stdout);
+    });
+  }
 
   await executor.shutdown();
 })();

examples/docs-examples/examples/executing-tasks/map.mjs

@@ -8,9 +8,14 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const data = [1, 2, 3, 4, 5];
 
-  const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`));
+  const futureResults = data.map(async (item) =>
+    executor.run(async (ctx) => {
+      return await ctx.run(`echo "${item}"`);
+    }),
+  );
 
-  for await (const result of results) console.log(result.stdout);
+  const results = await Promise.all(futureResults);
+  results.forEach((result) => console.log(result.stdout));
 
   await executor.shutdown();
 })();

examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs

@@ -9,9 +9,14 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const data = [1, 2, 3, 4, 5];
 
-  const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`));
+  const futureResults = data.map(async (item) =>
+    executor.run(async (ctx) => {
+      return await ctx.run(`echo "${item}"`);
+    }),
+  );
 
-  for await (const result of results) console.log(result.stdout);
+  const results = await Promise.all(futureResults);
+  results.forEach((result) => console.log(result.stdout));
 
   await executor.shutdown();
 })();

examples/docs-examples/examples/selecting-providers/custom-price.mjs

@@ -12,10 +12,10 @@ const myFilter = async (proposal) => {
   let counterIdx = usageVector.findIndex((ele) => ele === "golem.usage.duration_sec");
   let proposedCost = proposal.properties["golem.com.pricing.model.linear.coeffs"][counterIdx];
   costData.push(proposedCost);
-  if (costData.length < 11) return false;
+  if (costData.length < 6) return false;
   else {
     costData.shift();
-    let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 10;
+    let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 5;
     if (proposedCost <= 1.2 * averageProposedCost) decision = true;
     if (decision) {
       console.log(proposedCost, averageProposedCost);

examples/docs-examples/tutorials/accessing-internet/manifest.json

@@ -0,0 +1,33 @@
+{
+  "version": "0.1.0",
+  "createdAt": "2023-10-19T11:23:08.156+02:00",
+  "expiresAt": "2024-01-17T11:23:08.156+01:00",
+  "metadata": {
+    "name": "outbound-docs",
+    "description": "",
+    "version": "1.0.0"
+  },
+  "payload": [
+    {
+      "platform": {
+        "os": "linux",
+        "arch": "x86_64"
+      },
+      "hash": "sha3:dad8f776b0eb9f37ea0d63de42757034dd085fe30cc4537c2e119d80",
+      "urls": [
+        "http://registry.golem.network/download/f37c8ba2b534ca631060fb8db4ac218d3199faf656aa2c92f402c2b700797c21"
+      ]
+    }
+  ],
+  "compManifest": {
+    "version": "0.1.0",
+    "net": {
+      "inet": {
+        "out": {
+          "urls": ["https://ipfs.io"],
+          "protocols": ["https"]
+        }
+      }
+    }
+  }
+}

examples/docs-examples/tutorials/accessing-internet/outbound-ipfs.mjs

@@ -0,0 +1,33 @@
+import { TaskExecutor } from "@golem-sdk/golem-js";
+import { readFile } from "fs/promises";
+
+const url = "https://ipfs.io/ipfs/bafybeihkoviema7g3gxyt6la7vd5ho32ictqbilu3wnlo3rs7ewhnp7lly";
+
+(async function main() {
+  // Load the manifest.
+  const manifest = await readFile(`./manifest.json`);
+
+  // Create and configure a TaskExecutor instance.
+  const executor = await TaskExecutor.create({
+    capabilities: ["inet", "manifest-support"],
+    yagnaOptions: { apiKey: "try_golem" },
+    manifest: manifest.toString("base64"),
+  });
+
+  try {
+    await executor.run(async (ctx) => {
+      const result = await ctx.run(`curl ${url} -o /golem/work/example.jpg`);
+
+      console.log((await ctx.run("ls -l")).stdout);
+      if (result.result === "Ok") {
+        console.log("File downloaded!");
+      } else {
+        console.error("Failed to download the file!", result.stderr);
+      }
+    });
+  } catch (err) {
+    console.error("The task failed due to", err);
+  } finally {
+    await executor.end();
+  }
+})();

examples/docs-examples/tutorials/running-parallel-tasks/index.mjs

@@ -19,33 +19,35 @@ async function main(args) {
   const step = Math.floor(keyspace / args.numberOfProviders + 1);
   const range = [...Array(Math.floor(keyspace / step) + 1).keys()].map((i) => i * step);
 
-  const results = executor.map(range, async (ctx, skip = 0) => {
-    const results = await ctx
-      .beginBatch()
-      .run(
-        `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min(
-          keyspace,
-          skip + step,
-        )} -o pass.potfile`,
-      )
-      .run("cat pass.potfile")
-      .end()
-      .catch((err) => console.error(err));
-    if (!results?.[1]?.stdout) return false;
-    return results?.[1]?.stdout.toString().split(":")[1];
-  });
-
-  let password = "";
-  for await (const result of results) {
-    if (result) {
-      password = result;
-      break;
+  const findPasswordInRange = async (skip) => {
+    const password = await executor.run(async (ctx) => {
+      const [, potfileResult] = await ctx
+        .beginBatch()
+        .run(
+          `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${
+            skip + step
+          } -o pass.potfile || true`,
+        )
+        .run("cat pass.potfile || true")
+        .end();
+      if (!potfileResult.stdout) return false;
+      // potfile format is: hash:password
+      return potfileResult.stdout.toString().trim().split(":")[1];
+    });
+    if (!password) {
+      throw new Error(`Cannot find password in range ${skip} - ${skip + step}`);
     }
+    return password;
+  };
+
+  try {
+    const password = await Promise.any(range.map(findPasswordInRange));
+    console.log(`Password found: ${password}`);
+  } catch (err) {
+    console.log(`Password not found`);
+  } finally {
+    await executor.shutdown();
   }
-
-  if (!password) console.log("No password found");
-  else console.log(`Password found: ${password}`);
-  await executor.shutdown();
 }
 
 program

examples/fibonacci/fibonacci.js

@@ -9,12 +9,17 @@ async function main(fiboN = 1, tasksCount = 1, subnetTag, driver, network, debug
     logLevel: debug ? "debug" : "info",
   });
 
-  const data = Array(tasksCount).fill(null);
+  const runningTasks = [];
+  for (let i = 0; i < tasksCount; i++) {
+    runningTasks.push(
+      executor.run(async (ctx) => {
+        const result = await ctx.run("/usr/local/bin/node", ["/golem/work/fibo.js", fiboN.toString()]);
+        console.log(result.stdout);
+      }),
+    );
+  }
 
-  await executor.forEach(data, async (ctx) => {
-    const result = await ctx.run("/usr/local/bin/node", ["/golem/work/fibo.js", fiboN.toString()]);
-    console.log(result.stdout);
-  });
+  await Promise.all(runningTasks);
   await executor.shutdown();
 }
 program