Merge pull request #662 from golemfactory/master_to_beta

Master -> beta

.github/dependabot.yml

@@ -7,5 +7,6 @@ version: 2
 updates:
   - package-ecosystem: "npm" # See documentation for possible values
     directory: "/" # Location of package manifests
+    target-branch: "beta"
     schedule:
       interval: "weekly"

.github/workflows/cypress-nightly.yml

@@ -16,15 +16,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

.github/workflows/examples-nightly.yml

@@ -32,15 +32,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

.github/workflows/goth-nightly.yml

@@ -32,15 +32,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

.github/workflows/release.yml

@@ -55,15 +55,15 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Use random string for subnet
+        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV
 
       - name: Build the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
         run: docker compose -f tests/docker/docker-compose.yml build
 
       - name: Start the docker containers
-        # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2
-        run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
+        # Restart docker to avoid issues with the docker compose down due to improper cleanup of the previous run.
+        run: sudo service docker restart && docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d
 
       - name: Fund the requestor
         # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry.

examples/blender/blender.ts

@@ -35,17 +35,21 @@ async function main(subnetTag: string, driver?: string, network?: string, debug?
     await ctx.uploadFile(`${__dirname}/cubes.blend`, "/golem/resource/scene.blend");
   });
 
-  const results = executor.map<number, string>([0, 10, 20, 30, 40, 50], async (ctx, frame) => {
-    const result = await ctx
-      .beginBatch()
-      .uploadJson(blender_params(frame), "/golem/work/params.json")
-      .run("/golem/entrypoints/run-blender.sh")
-      .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`)
-      .end()
-      .catch((e) => console.error(e));
-    return result?.length ? `output_${frame}.png` : "";
-  });
-  for await (const result of results) console.log(result);
+  const futureResults = [0, 10, 20, 30, 40, 50].map((frame) =>
+    executor.run(async (ctx) => {
+      const result = await ctx
+        .beginBatch()
+        .uploadJson(blender_params(frame), "/golem/work/params.json")
+        .run("/golem/entrypoints/run-blender.sh")
+        .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`)
+        .end()
+        .catch((e) => console.error(e));
+      return result?.length ? `output_${frame}.png` : "";
+    }),
+  );
+  const results = await Promise.all(futureResults);
+  results.forEach((result) => console.log(result));
+
   await executor.end();
 }
 

examples/docs-examples/examples/executing-tasks/action_log.txt

@@ -1 +1 @@
-some action log
+some action log

examples/docs-examples/examples/executing-tasks/before-each.mjs

@@ -12,13 +12,18 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     await ctx.uploadFile("./action_log.txt", "/golem/input/action_log.txt");
   });
 
-  await executor.forEach([1, 2, 3, 4, 5], async (ctx, item) => {
-    await ctx
-      .beginBatch()
-      .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`)
-      .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt")
-      .end();
+  const inputs = [1, 2, 3, 4, 5];
+
+  const futureResults = inputs.map(async (item) => {
+    return await executor.run(async (ctx) => {
+      await ctx
+        .beginBatch()
+        .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`)
+        .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt")
+        .end();
+    });
   });
+  await Promise.all(futureResults);
 
   await executor.end();
 })();

examples/docs-examples/examples/executing-tasks/foreach.mjs

@@ -8,9 +8,11 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const data = [1, 2, 3, 4, 5];
 
-  await executor.forEach(data, async (ctx, item) => {
-    console.log((await ctx.run(`echo "${item}"`)).stdout);
-  });
+  for (const item of data) {
+    await executor.run(async (ctx) => {
+      console.log((await ctx.run(`echo "${item}"`)).stdout);
+    });
+  }
 
   await executor.end();
 })();

examples/docs-examples/examples/executing-tasks/map.mjs

@@ -8,9 +8,14 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const data = [1, 2, 3, 4, 5];
 
-  const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`));
+  const futureResults = data.map(async (item) =>
+    executor.run(async (ctx) => {
+      return await ctx.run(`echo "${item}"`);
+    }),
+  );
 
-  for await (const result of results) console.log(result.stdout);
+  const results = await Promise.all(futureResults);
+  results.forEach((result) => console.log(result.stdout));
 
   await executor.end();
 })();

examples/docs-examples/examples/executing-tasks/max-parallel-tasks.mjs

@@ -9,9 +9,14 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const data = [1, 2, 3, 4, 5];
 
-  const results = executor.map(data, (ctx, item) => ctx.run(`echo "${item}"`));
+  const futureResults = data.map(async (item) =>
+    executor.run(async (ctx) => {
+      return await ctx.run(`echo "${item}"`);
+    }),
+  );
 
-  for await (const result of results) console.log(result.stdout);
+  const results = await Promise.all(futureResults);
+  results.forEach((result) => console.log(result.stdout));
 
   await executor.end();
 })();

examples/docs-examples/examples/selecting-providers/custom-price.mjs

@@ -12,10 +12,10 @@ const myFilter = async (proposal) => {
   let counterIdx = usageVector.findIndex((ele) => ele === "golem.usage.duration_sec");
   let proposedCost = proposal.properties["golem.com.pricing.model.linear.coeffs"][counterIdx];
   costData.push(proposedCost);
-  if (costData.length < 11) return false;
+  if (costData.length < 6) return false;
   else {
     costData.shift();
-    let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 10;
+    let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 5;
     if (proposedCost <= 1.2 * averageProposedCost) decision = true;
     if (decision) {
       console.log(proposedCost, averageProposedCost);

examples/docs-examples/tutorials/running-parallel-tasks/index.mjs

@@ -19,33 +19,35 @@ async function main(args) {
   const step = Math.floor(keyspace / args.numberOfProviders + 1);
   const range = [...Array(Math.floor(keyspace / step) + 1).keys()].map((i) => i * step);
 
-  const results = executor.map(range, async (ctx, skip = 0) => {
-    const results = await ctx
-      .beginBatch()
-      .run(
-        `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${Math.min(
-          keyspace,
-          skip + step,
-        )} -o pass.potfile`,
-      )
-      .run("cat pass.potfile")
-      .end()
-      .catch((err) => console.error(err));
-    if (!results?.[1]?.stdout) return false;
-    return results?.[1]?.stdout.toString().split(":")[1];
-  });
-
-  let password = "";
-  for await (const result of results) {
-    if (result) {
-      password = result;
-      break;
+  const findPasswordInRange = async (skip) => {
+    const password = await executor.run(async (ctx) => {
+      const [, potfileResult] = await ctx
+        .beginBatch()
+        .run(
+          `hashcat -a 3 -m 400 '${args.hash}' '${args.mask}' --skip=${skip} --limit=${
+            skip + step
+          } -o pass.potfile || true`,
+        )
+        .run("cat pass.potfile || true")
+        .end();
+      if (!potfileResult.stdout) return false;
+      // potfile format is: hash:password
+      return potfileResult.stdout.toString().trim().split(":")[1];
+    });
+    if (!password) {
+      throw new Error(`Cannot find password in range ${skip} - ${skip + step}`);
     }
+    return password;
+  };
+
+  try {
+    const password = await Promise.any(range.map(findPasswordInRange));
+    console.log(`Password found: ${password}`);
+  } catch (err) {
+    console.log(`Password not found`);
+  } finally {
+    await executor.end();
   }
-
-  if (!password) console.log("No password found");
-  else console.log(`Password found: ${password}`);
-  await executor.end();
 }
 
 program

examples/fibonacci/fibonacci.js

@@ -9,12 +9,17 @@ async function main(fiboN = 1, tasksCount = 1, subnetTag, driver, network, debug
     logLevel: debug ? "debug" : "info",
   });
 
-  const data = Array(tasksCount).fill(null);
+  const runningTasks = [];
+  for (let i = 0; i < tasksCount; i++) {
+    runningTasks.push(
+      executor.run(async (ctx) => {
+        const result = await ctx.run("/usr/local/bin/node", ["/golem/work/fibo.js", fiboN.toString()]);
+        console.log(result.stdout);
+      }),
+    );
+  }
 
-  await executor.forEach(data, async (ctx) => {
-    const result = await ctx.run("/usr/local/bin/node", ["/golem/work/fibo.js", fiboN.toString()]);
-    console.log(result.stdout);
-  });
+  await Promise.all(runningTasks);
   await executor.end();
 }
 program

examples/simple-usage/map.ts

@@ -4,14 +4,15 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
   const executor = await TaskExecutor.create("golem/alpine:latest");
   const data = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"];
 
-  const results = executor.map<string, string | undefined>(data, async (ctx, x) => {
-    const res = await ctx.run(`echo "${x}"`);
-    return res.stdout?.toString().trim();
-  });
-  const finalOutput: string[] = [];
-  for await (const res of results) {
-    if (res) finalOutput.push(res);
-  }
-  console.log("RESULTS = ", finalOutput.join(", "));
+  const futureResults = data.map((x) =>
+    executor.run(async (ctx) => {
+      const res = await ctx.run(`echo "${x}"`);
+      return res.stdout?.toString().trim();
+    }),
+  );
+
+  const results = await Promise.all(futureResults);
+  console.log(results);
+
   await executor.end();
 })();

examples/ssh/ssh.ts

@@ -12,31 +12,36 @@ async function main(subnetTag, driver, network, count = 2, sessionTimeout = 100,
     payment: { driver, network },
     logLevel: debug ? "debug" : "info",
   });
-  const data = new Array(count).fill(null);
   const appKey = process.env["YAGNA_APPKEY"];
-  await executor.forEach(data, async (ctx) => {
-    const password = crypto.randomBytes(3).toString("hex");
-    const results = await ctx
-      .beginBatch()
-      .run("syslogd")
-      .run("ssh-keygen -A")
-      .run(`echo -e "${password}\n${password}" | passwd`)
-      .run("/usr/sbin/sshd")
-      .end()
-      .catch((e) => console.error(e));
-    if (!results) return;
-    console.log("\n------------------------------------------");
-    console.log(`Connect via ssh to provider "${ctx.provider?.name}" with:`);
-    console.log(
-      `ssh -o ProxyCommand='websocat asyncstdio: ${ctx.getWebsocketUri(
-        22,
-      )} --binary -H=Authorization:"Bearer ${appKey}"' root@${crypto.randomBytes(10).toString("hex")}`,
+  const runningTasks: Promise<void>[] = [];
+  for (let i = 0; i < count; i++) {
+    runningTasks.push(
+      executor.run(async (ctx) => {
+        const password = crypto.randomBytes(3).toString("hex");
+        const results = await ctx
+          .beginBatch()
+          .run("syslogd")
+          .run("ssh-keygen -A")
+          .run(`echo -e "${password}\n${password}" | passwd`)
+          .run("/usr/sbin/sshd")
+          .end()
+          .catch((e) => console.error(e));
+        if (!results) return;
+        console.log("\n------------------------------------------");
+        console.log(`Connect via ssh to provider "${ctx.provider?.name}" with:`);
+        console.log(
+          `ssh -o ProxyCommand='websocat asyncstdio: ${ctx.getWebsocketUri(
+            22,
+          )} --binary -H=Authorization:"Bearer ${appKey}"' root@${crypto.randomBytes(10).toString("hex")}`,
+        );
+        console.log(`Password: ${password}`);
+        console.log("------------------------------------------\n");
+        await new Promise((res) => setTimeout(res, sessionTimeout * 1000));
+        console.log(`Task completed. Session SSH closed after ${sessionTimeout} secs timeout.`);
+      }),
     );
-    console.log(`Password: ${password}`);
-    console.log("------------------------------------------\n");
-    await new Promise((res) => setTimeout(res, sessionTimeout * 1000));
-    console.log(`Task completed. Session SSH closed after ${sessionTimeout} secs timeout.`);
-  });
+  }
+  await Promise.all(runningTasks);
   await executor.end();
 }