fix: dont terminate the executor for the user (#659)

JST-542 * fix: dont terminate the executor for the user * test: fix unit test * test: increase yacat timeout * test: increase timeout * test: increase timeout * test: decrease timeout again after seweryns fix * test: test new example * test: test with old executor * test: test all examples * test: fix jest test * test: add missing error handling * test: code review updates * test: code review update * Revert "test: code review update" This reverts commit da60260. * chore: add missing promise in stream
golemfactory · Nov 23, 2023 · 5c570c3 · 5c570c3
1 parent dbd64a5
commit 5c570c3
Show file tree

Hide file tree

Showing 60 changed files with 672 additions and 424 deletions.
diff --git a/examples/blender/blender.ts b/examples/blender/blender.ts
@@ -21,7 +21,6 @@ const blender_params = (frame) => ({
   WORK_DIR: "/golem/work",
   OUTPUT_DIR: "/golem/output",
 });
-
 async function main(subnetTag: string, driver?: string, network?: string, debug?: boolean, maxParallelTasks?: number) {
   const executor = await TaskExecutor.create({
     subnetTag,
@@ -31,26 +30,32 @@ async function main(subnetTag: string, driver?: string, network?: string, debug?
     maxParallelTasks,
   });
 
-  executor.beforeEach(async (ctx) => {
-    await ctx.uploadFile(`${__dirname}/cubes.blend`, "/golem/resource/scene.blend");
-  });
-
-  const futureResults = [0, 10, 20, 30, 40, 50].map((frame) =>
-    executor.run(async (ctx) => {
-      const result = await ctx
-        .beginBatch()
-        .uploadJson(blender_params(frame), "/golem/work/params.json")
-        .run("/golem/entrypoints/run-blender.sh")
-        .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `${__dirname}/output_${frame}.png`)
-        .end()
-        .catch((e) => console.error(e));
-      return result?.length ? `output_${frame}.png` : "";
-    }),
-  );
-  const results = await Promise.all(futureResults);
-  results.forEach((result) => console.log(result));
+  try {
+    executor.beforeEach(async (ctx) => {
+      await ctx.uploadFile(`${__dirname}/cubes.blend`, "/golem/resource/scene.blend");
+    });
 
-  await executor.end();
+    const futureResults = [0, 10, 20, 30, 40, 50].map((frame) =>
+      executor.run(async (ctx) => {
+        const result = await ctx
+          .beginBatch()
+          .uploadJson(blender_params(frame), "/golem/work/params.json")
+          .run("/golem/entrypoints/run-blender.sh")
+          .downloadFile(
+            `/golem/output/out${frame?.toString().padStart(4, "0")}.png`,
+            `${__dirname}/output_${frame}.png`,
+          )
+          .end();
+        return result?.length ? `output_${frame}.png` : "";
+      }),
+    );
+    const results = await Promise.all(futureResults);
+    results.forEach((result) => console.log(result));
+  } catch (error) {
+    console.error("Computation failed:", error);
+  } finally {
+    await executor.end();
+  }
 }
 
 program

diff --git a/examples/docs-examples/examples/composing-tasks/alert-code.mjs b/examples/docs-examples/examples/composing-tasks/alert-code.mjs
@@ -5,19 +5,23 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => {
-    const res = await ctx
-      .beginBatch()
-      .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
-      .run("node /golem/input/worker.mjs > /golem/input/output.txt")
-      .run("cat /golem/input/output.txt")
-      .downloadFile("/golem/input/output.txt", "./output.txt")
-      .endStream();
+  try {
+    const result = await executor.run(async (ctx) => {
+      const res = await ctx
+        .beginBatch()
+        .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
+        .run("node /golem/input/worker.mjs > /golem/input/output.txt")
+        .run("cat /golem/input/output.txt")
+        .downloadFile("/golem/input/output.txt", "./output.txt")
+        .endStream();
 
-    for await (const chunk of res) {
-      chunk.index == 2 ? console.log(chunk.stdout) : "";
-    }
-  });
-
-  await executor.end();
+      for await (const chunk of res) {
+        if (chunk.index === 2) console.log(chunk.stdout);
+      }
+    });
+  } catch (err) {
+    console.error("Task encountered an error:", err);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/batch-end.mjs b/examples/docs-examples/examples/composing-tasks/batch-end.mjs
@@ -6,19 +6,23 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => {
-    const res = await ctx
-      .beginBatch()
-      .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
-      .run("node /golem/input/worker.mjs > /golem/input/output.txt")
-      .run("cat /golem/input/output.txt")
-      .downloadFile("/golem/input/output.txt", "./output.txt")
-      .end()
-      .catch((error) => console.error(error)); // to be removed and replaced with try & catch
+  try {
+    const result = await executor.run(async (ctx) => {
+      return (
+        await ctx
+          .beginBatch()
+          .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
+          .run("node /golem/input/worker.mjs > /golem/input/output.txt")
+          .run("cat /golem/input/output.txt")
+          .downloadFile("/golem/input/output.txt", "./output.txt")
+          .end()
+      )[2]?.stdout;
+    });
 
-    return res[2]?.stdout;
-  });
-
-  console.log(result);
-  await executor.end();
+    console.log(result);
+  } catch (error) {
+    console.error("Computation failed:", error);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs
@@ -6,17 +6,25 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => {
-    const res = await ctx
-      .beginBatch()
-      .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
-      .run("node /golem/input/worker.mjs > /golem/input/output.txt")
-      .run("cat /golem/input/output.txt")
-      .downloadFile("/golem/input/output.txt", "./output.txt")
-      .endStream();
+  try {
+    const result = await executor.run(async (ctx) => {
+      const res = await ctx
+        .beginBatch()
+        .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
+        .run("node /golem/input/worker.mjs > /golem/input/output.txt")
+        .run("cat /golem/input/output.txt")
+        .downloadFile("/golem/input/output.txt", "./output.txt")
+        .endStream();
 
-    res.on("data", (data) => (data.index == 2 ? console.log(data.stdout) : ""));
-    res.on("error", (error) => console.error(error));
-    res.on("close", () => executor.end());
-  });
+      return new Promise((resolve) => {
+        res.on("data", (result) => console.log(result));
+        res.on("error", (error) => console.error(error));
+        res.once("close", resolve);
+      });
+    });
+  } catch (error) {
+    console.error("Computation failed:", error);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs b/examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs
@@ -6,19 +6,23 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => {
-    const res = await ctx
-      .beginBatch()
-      .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
-      .run("node /golem/input/worker.mjs > /golem/input/output.txt")
-      .run("cat /golem/input/output.txt")
-      .downloadFile("/golem/input/output.txt", "./output.txt")
-      .endStream();
+  try {
+    const result = await executor.run(async (ctx) => {
+      const res = await ctx
+        .beginBatch()
+        .uploadFile("./worker.mjs", "/golem/input/worker.mjs")
+        .run("node /golem/input/worker.mjs > /golem/input/output.txt")
+        .run("cat /golem/input/output.txt")
+        .downloadFile("/golem/input/output.txt", "./output.txt")
+        .endStream();
 
-    for await (const chunk of res) {
-      console.log(chunk.stdout);
-    }
-  });
-
-  await executor.end();
+      for await (const chunk of res) {
+        console.log(chunk.stdout);
+      }
+    });
+  } catch (err) {
+    console.error("An error occurred:", err);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs b/examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs
@@ -6,15 +6,19 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => {
-    await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs");
-    await ctx.run("node /golem/input/worker.mjs > /golem/input/output.txt");
-    const result = await ctx.run("cat /golem/input/output.txt");
-    await ctx.downloadFile("/golem/input/output.txt", "./output.txt");
-    return result.stdout;
-  });
-
-  console.log(result);
+  try {
+    const result = await executor.run(async (ctx) => {
+      await ctx.uploadFile("./worker.mjs", "/golem/input/worker.mjs");
+      await ctx.run("node /golem/input/worker.mjs > /golem/input/output.txt");
+      const result = await ctx.run("cat /golem/input/output.txt");
+      await ctx.downloadFile("/golem/input/output.txt", "./output.txt");
+      return result.stdout;
+    });
 
-  await executor.end();
+    console.log(result);
+  } catch (err) {
+    console.error("An error occurred:", err);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/single-command.cjs b/examples/docs-examples/examples/composing-tasks/single-command.cjs
@@ -6,8 +6,12 @@ const { TaskExecutor } = require("@golem-sdk/golem-js");
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout);
-  await executor.end();
-
-  console.log("Task result:", result);
+  try {
+    const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout);
+    console.log("Task result:", result);
+  } catch (err) {
+    console.error("Task failed:", err);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/single-command.mjs b/examples/docs-examples/examples/composing-tasks/single-command.mjs
@@ -6,8 +6,12 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout);
-  await executor.end();
-
-  console.log("Task result:", result);
+  try {
+    const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout);
+    console.log("Task result:", result);
+  } catch (err) {
+    console.error("An error occurred:", err);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/composing-tasks/single-command.ts b/examples/docs-examples/examples/composing-tasks/single-command.ts
@@ -6,8 +6,12 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const result = await executor.run<string | unknown>(async (ctx) => (await ctx.run("node -v")).stdout);
-  await executor.end();
-
-  console.log("Task result:", result);
+  try {
+    const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout);
+    console.log("Task result:", result);
+  } catch (err) {
+    console.error("Error while running the task:", err);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/executing-tasks/before-each.mjs b/examples/docs-examples/examples/executing-tasks/before-each.mjs
@@ -14,16 +14,20 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
   const inputs = [1, 2, 3, 4, 5];
 
-  const futureResults = inputs.map(async (item) => {
-    return await executor.run(async (ctx) => {
-      await ctx
-        .beginBatch()
-        .run(`echo ` + `'processing item: ` + item + `' >> /golem/input/action_log.txt`)
-        .downloadFile("/golem/input/action_log.txt", "./output_" + ctx.provider.name + ".txt")
-        .end();
+  try {
+    const futureResults = inputs.map(async (item) => {
+      return await executor.run(async (ctx) => {
+        await ctx
+          .beginBatch()
+          .run(`echo 'processing item: ${item}' >> /golem/input/action_log.txt`)
+          .downloadFile("/golem/input/action_log.txt", `./output_${ctx.provider.name}.txt`)
+          .end();
+      });
     });
-  });
-  await Promise.all(futureResults);
-
-  await executor.end();
+    await Promise.all(futureResults);
+  } catch (error) {
+    console.error("A critical error occurred:", error);
+  } finally {
+    await executor.end();
+  }
 })();
diff --git a/examples/docs-examples/examples/executing-tasks/foreach.mjs b/examples/docs-examples/examples/executing-tasks/foreach.mjs
@@ -6,13 +6,17 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const data = [1, 2, 3, 4, 5];
+  try {
+    const data = [1, 2, 3, 4, 5];
 
-  for (const item of data) {
-    await executor.run(async (ctx) => {
-      console.log((await ctx.run(`echo "${item}"`)).stdout);
-    });
+    for (const item of data) {
+      await executor.run(async (ctx) => {
+        console.log((await ctx.run(`echo "${item}"`)).stdout);
+      });
+    }
+  } catch (err) {
+    console.error("An error occurred during execution", err);
+  } finally {
+    await executor.end();
   }
-
-  await executor.end();
 })();
diff --git a/examples/docs-examples/examples/executing-tasks/map.mjs b/examples/docs-examples/examples/executing-tasks/map.mjs
@@ -6,16 +6,20 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
     yagnaOptions: { apiKey: "try_golem" },
   });
 
-  const data = [1, 2, 3, 4, 5];
+  try {
+    const data = [1, 2, 3, 4, 5];
 
-  const futureResults = data.map(async (item) =>
-    executor.run(async (ctx) => {
-      return await ctx.run(`echo "${item}"`);
-    }),
-  );
+    const futureResults = data.map(async (item) =>
+      executor.run(async (ctx) => {
+        return await ctx.run(`echo "${item}"`);
+      }),
+    );
 
-  const results = await Promise.all(futureResults);
-  results.forEach((result) => console.log(result.stdout));
-
-  await executor.end();
+    const results = await Promise.all(futureResults);
+    results.forEach((result) => console.log(result.stdout));
+  } catch (err) {
+    console.error("An error occurred:", err);
+  } finally {
+    await executor.end();
+  }
 })();