Merge pull request #603 from golemfactory/beta

Beta -> master

.github/workflows/cypress-nightly.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   cypress-tests:
-    runs-on: goth2
+    runs-on: [goth2, ubuntu-22.10]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -34,6 +34,7 @@ jobs:
           npm run build
 
       - name: Configure python
+        continue-on-error: true
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/examples-nightly.yml

@@ -21,7 +21,7 @@ jobs:
         run: echo "::set-output name=matrix::{\"include\":[{\"branch\":\"master\"}]}"
 
   goth-tests:
-    runs-on: goth2
+    runs-on: [goth2, ubuntu-22.10]
     needs: prepare-matrix-master-only
     strategy:
       matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }}
@@ -46,6 +46,7 @@ jobs:
           npm install ts-node
 
       - name: Configure python
+        continue-on-error: true
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/goth-nightly.yml

@@ -21,7 +21,7 @@ jobs:
         run: echo "::set-output name=matrix::{\"include\":[{\"branch\":\"master\"}]}"
 
   goth-tests:
-    runs-on: goth2
+    runs-on: [goth2, ubuntu-22.10]
     needs: prepare-matrix-master-only
     strategy:
       matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }}
@@ -44,6 +44,7 @@ jobs:
           npm run build
 
       - name: Configure python
+        continue-on-error: true
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"

.github/workflows/release.yml

@@ -81,6 +81,8 @@ jobs:
         run: |
           npm install
           npm run build
+          npm install --prefix examples
+          npm install ts-node
 
       - name: Configure python
         uses: actions/setup-python@v4

examples/docs-examples/examples/selecting-providers/custom-price.mjs

@@ -16,7 +16,7 @@ const myFilter = async (proposal) => {
   else {
     costData.shift();
     let averageProposedCost = costData.reduce((part, x) => part + x, 0) / 10;
-    if (proposedCost <= averageProposedCost) decision = true;
+    if (proposedCost <= 1.2 * averageProposedCost) decision = true;
     if (decision) {
       console.log(proposedCost, averageProposedCost);
     }

examples/docs-examples/examples/selecting-providers/whitelist.mjs

@@ -5,7 +5,14 @@ import { TaskExecutor, ProposalFilters } from "@golem-sdk/golem-js";
  * which only allows offers from a provider whose ID is in the array
  */
 
-const whiteListIds = ["0x3fc1d65ddc5258dc8807df30a29f71028e965e1b", "0x4506550de84d207f3ab90add6336f68119015836"];
+const whiteListIds = [
+  "0x3a21c608925ddbc745afab6375d1f5e77283538e",
+  "0xd79f83f1108d1fcbe0cf57e13b452305eb38a325",
+  "0x677c5476f3b0e1f03d5c3abd2e2e2231e36fddde",
+  "0x06c03165aaa676680b9d02c1f3ee846c3806fec7",
+  "0x17ec8597ff92c3f44523bdc65bf0f1be632917ff", // goth provider-1:
+  "0x63fc2ad3d021a4d7e64323529a55a9442c444da0", // goth provider-2:
+];
 console.log("Will accept only proposals from:");
 for (let i = 0; i < whiteListIds.length; i++) {
   console.log(whiteListIds[i]);

examples/docs-examples/examples/working-with-images/tag.mjs

@@ -3,7 +3,7 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 (async () => {
   const executor = await TaskExecutor.create({
     package: "golem/alpine:latest",
-    yagnaOptions: { appKey: "try_golem" },
+    yagnaOptions: { apiKey: "try_golem" },
   });
 
   const result = await executor.run(async (ctx) => (await ctx.run("node -v")).stdout);

src/task/batch.spec.ts

@@ -1,4 +1,4 @@
-import { DownloadFile, Run, UploadData, UploadFile } from "../script";
+import { DownloadFile, Run, Transfer, UploadData, UploadFile } from "../script";
 import { Batch } from "./batch";
 import { NullStorageProvider } from "../storage";
 import { ActivityMock } from "../../tests/mock/activity.mock";
@@ -28,6 +28,16 @@ describe("Batch", () => {
       });
     });
 
+    describe("transfer()", () => {
+      it("should add transfer file command", async () => {
+        expect(batch.transfer("http://golem.network/test.txt", "/golem/file.txt")).toBe(batch);
+        const cmd = batch["script"]["commands"][0] as Transfer;
+        expect(cmd).toBeInstanceOf(Transfer);
+        expect(cmd["from"]).toBe("http://golem.network/test.txt");
+        expect(cmd["to"]).toBe("/golem/file.txt");
+      });
+    });
+
     describe("uploadFile()", () => {
       it("should add upload file command", async () => {
         expect(batch.uploadFile("/tmp/file.txt", "/golem/file.txt")).toBe(batch);

src/task/batch.ts

@@ -1,4 +1,4 @@
-import { DownloadFile, Run, Script, UploadFile } from "../script";
+import { DownloadFile, Run, Script, Transfer, UploadFile } from "../script";
 import { Activity, Result } from "../activity";
 import { StorageProvider } from "../storage/provider";
 import { Logger, sleep } from "../utils";
@@ -44,6 +44,11 @@ export class Batch {
     return this;
   }
 
+  transfer(from: string, to: string): Batch {
+    this.script.add(new Transfer(from, to));
+    return this;
+  }
+
   uploadFile(src: string, dst: string): Batch {
     this.script.add(new UploadFile(this.storageProvider, src, dst));
     return this;

src/task/work.spec.ts

@@ -1,7 +1,7 @@
 import { Batch, WorkContext } from "./index";
 import { LoggerMock, YagnaMock } from "../../tests/mock";
 import { ActivityStateEnum, ResultState } from "../activity";
-import { DownloadData, DownloadFile, Run, Script, UploadData, UploadFile } from "../script";
+import { DownloadData, DownloadFile, Run, Script, Transfer, UploadData, UploadFile } from "../script";
 import { ActivityMock } from "../../tests/mock/activity.mock";
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
@@ -46,6 +46,19 @@ describe("Work Context", () => {
       });
     });
 
+    describe("transfer()", () => {
+      it("should execute transfer command", async () => {
+        const result = ActivityMock.createResult({ stdout: "Ok" });
+        runSpy.mockImplementation((cmd) => {
+          expect(cmd).toBeInstanceOf(Transfer);
+          expect(cmd["from"]).toBe("http://golem.network/test.txt");
+          expect(cmd["to"]).toBe("/golem/work/test.txt");
+          return Promise.resolve(result);
+        });
+        expect(await context.transfer("http://golem.network/test.txt", "/golem/work/test.txt")).toBe(result);
+      });
+    });
+
     describe("uploadFile()", () => {
       it("should execute upload file command", async () => {
         const result = ActivityMock.createResult();
@@ -177,7 +190,7 @@ describe("Work Context", () => {
 
   describe("getWebsocketUri()", () => {
     it("should throw error if there is no network node", () => {
-      expect(context["networkNode"]).toBeUndefined();
+      expect(() => context.getIp()).toThrow(new Error("There is no network in this work context"));
     });
 
     it("should return websocket URI", () => {
@@ -190,6 +203,19 @@ describe("Work Context", () => {
     });
   });
 
+  describe("getIp()", () => {
+    it("should throw error if there is no network node", () => {
+      expect(() => context.getIp()).toThrow(new Error("There is no network in this work context"));
+    });
+
+    it("should return ip address of provider vpn network node", () => {
+      (context as any)["networkNode"] = {
+        ip: "192.168.0.2",
+      };
+      expect(context.getIp()).toEqual("192.168.0.2");
+    });
+  });
+
   describe("beginBatch()", () => {
     it("should create a batch object", () => {
       const o = {};

src/task/work.ts

@@ -8,6 +8,7 @@ import {
   Run,
   Script,
   Start,
+  Transfer,
   UploadData,
   UploadFile,
 } from "../script";
@@ -134,6 +135,17 @@ export class WorkContext {
     return this.runOneCommand(run, runOptions);
   }
 
+  /**
+   * Generic transfer command, requires the user to provide a publicly readable transfer source
+   *
+   * @param from - publicly available resource for reading. Supported protocols: file, http, ftp or gftp
+   * @param to - file path
+   * @param options Additional run options.
+   */
+  async transfer(from: string, to: string, options?: CommandOptions): Promise<Result> {
+    return this.runOneCommand(new Transfer(from, to), options);
+  }
+
   async uploadFile(src: string, dst: string, options?: CommandOptions): Promise<Result> {
     return this.runOneCommand(new UploadFile(this.storageProvider, src, dst), options);
   }
@@ -185,7 +197,12 @@ export class WorkContext {
 
   getWebsocketUri(port: number): string {
     if (!this.networkNode) throw new Error("There is no network in this work context");
-    return this.networkNode?.getWebsocketUri(port);
+    return this.networkNode.getWebsocketUri(port);
+  }
+
+  getIp(): string {
+    if (!this.networkNode) throw new Error("There is no network in this work context");
+    return this.networkNode.ip.toString();
   }
 
   async getState(): Promise<ActivityStateEnum> {

tests/e2e/tasks.spec.ts

@@ -1,6 +1,7 @@
 import { LoggerMock } from "../mock";
 import { readFileSync } from "fs";
 import { TaskExecutor } from "../../src";
+import fs from "fs";
 const logger = new LoggerMock(false);
 
 describe("Task Executor", function () {
@@ -153,4 +154,30 @@ describe("Task Executor", function () {
     expect(result).toEqual("Ok");
     expect(readFileSync(`${process.env.GOTH_GFTP_VOLUME || ""}new_test.json`, "utf-8")).toEqual('{"test":"1234"}');
   });
+
+  it("should run transfer file via http", async () => {
+    executor = await TaskExecutor.create({
+      package: "golem/alpine:latest",
+      logger,
+    });
+    const result = await executor.run(async (ctx) => {
+      const res = await ctx.transfer(
+        "http://registry.golem.network/download/a2bb9119476179fac36149723c3ad4474d8d135e8d2d2308eb79907a6fc74dfa",
+        "/golem/work/alpine.gvmi",
+      );
+      return res.result;
+    });
+    expect(result).toEqual("Ok");
+  });
+
+  it("should get ip address", async () => {
+    executor = await TaskExecutor.create({
+      package: "golem/alpine:latest",
+      capabilities: ["vpn"],
+      networkIp: "192.168.0.0/24",
+      logger,
+    });
+    const result = await executor.run(async (ctx) => ctx.getIp());
+    expect(["192.168.0.2", "192.168.0.3"]).toContain(result);
+  });
 });

tests/examples/examples.json

@@ -1,7 +1,15 @@
 [
-  { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs" },
-  { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs" },
-  { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs" },
+  { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs", "noGoth": true },
+  {
+    "cmd": "node",
+    "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-chunks.mjs",
+    "noGoth": true
+  },
+  {
+    "cmd": "node",
+    "path": "examples/docs-examples/examples/composing-tasks/batch-endstream-forawait.mjs",
+    "noGoth": true
+  },
   { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/multiple-run-prosaic.mjs" },
   { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.mjs" },
   { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.cjs" },
@@ -15,7 +23,7 @@
 
   { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/custom-price.mjs", "noGoth": true },
   { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/demand.mjs" },
-  { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs", "noGoth": true },
+  { "cmd": "node", "path": "examples/docs-examples/examples/selecting-providers/whitelist.mjs" },
 
   { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/downloading-file.mjs" },
   { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-file.mjs" },
@@ -33,7 +41,11 @@
   { "cmd": "node", "path": "examples/docs-examples/examples/working-with-images/tag.mjs" },
 
   { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-end.mjs" },
-  { "cmd": "node", "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs" },
+  {
+    "cmd": "node",
+    "path": "examples/docs-examples/examples/working-with-results/multi-command-endstream.mjs",
+    "noGoth": true
+  },
   {
     "cmd": "node",
     "path": "examples/docs-examples/examples/working-with-results/multi-command-fail.mjs",

tests/examples/examples.test.ts

@@ -21,7 +21,7 @@ type Example = {
   skip?: boolean;
 };
 
-async function test(cmd: string, path: string, args: string[] = [], timeout = 120) {
+async function test(cmd: string, path: string, args: string[] = [], timeout = 180) {
   const file = basename(path);
   const cwd = dirname(path);
   const spawnedExample = spawn(cmd, [file, ...args], { cwd });