Merge pull request #564 from golemfactory/feature/JST-291/fix-issues-…

…exposed-by-examples-for-presentation

JST-291: Fix issues exposed by examples for presentation

.github/workflows/ci.yml

@@ -39,3 +39,5 @@ jobs:
           npm run lint
           npm run test:unit
           npm run build
+          npm install --prefix examples
+          npm run --prefix examples lint:ts

.github/workflows/release.yml

@@ -42,7 +42,8 @@ jobs:
           npm run format:check
           npm run lint
           npm run test:unit
-          npm run build
+          npm install --prefix examples
+          npm run --prefix examples lint:ts
 
   run-integration-and-e2e-tests:
     name: Run integration and E2E tests

README.md

@@ -1,5 +1,28 @@
 # Golem JavaScript API
 
+## Table of contents
+
+<!-- TOC -->
+
+- [Golem JavaScript API](#golem-javascript-api)
+  - [Table of contents](#table-of-contents)
+  - [What's Golem and `golem-js`?](#whats-golem-and-golem-js)
+  - [Golem application development](#golem-application-development)
+    - [Installation](#installation)
+    - [Building](#building)
+    - [Usage](#usage)
+      - [Node.js context](#nodejs-context)
+      - [Web Browser context](#web-browser-context)
+    - [Testing](#testing)
+    - [Running unit tests](#running-unit-tests)
+    - [Running E2E tests](#running-e2e-tests)
+      - [NodeJS](#nodejs)
+      - [Cypress](#cypress)
+    - [Contributing](#contributing)
+  - [Controlling interactions and costs](#controlling-interactions-and-costs)
+  - [See also](#see-also)
+  <!-- TOC -->
+
 ![GitHub](https://img.shields.io/github/license/golemfactory/golem-js)
 ![npm](https://img.shields.io/npm/v/@golem-sdk/golem-js)
 ![node-current](https://img.shields.io/node/v/@golem-sdk/golem-js)
@@ -10,13 +33,18 @@
 
 ## What's Golem and `golem-js`?
 
-**[The Golem Network](https://golem.network)** fosters a global group of creators building ambitious software solutions that will shape the technological landscape of future generations by accessing computing resources across the platform. Golem Network is an accessible, reliable, open access and censorship-resistant protocol, democratizing access to digital resources and connecting users through a flexible, open-source platform.
+**[The Golem Network](https://golem.network)** fosters a global group of creators building ambitious software solutions
+that will shape the technological landscape of future generations by accessing computing resources across the platform.
+Golem Network is an accessible, reliable, open access and censorship-resistant protocol, democratizing access to digital
+resources and connecting users through a flexible, open-source platform.
 
-**@golem-sdk/golem-js** is the JavaScript API that allows developers to connect to their Golem nodes and manage their distributed, computational loads through Golem Network.
+**@golem-sdk/golem-js** is the JavaScript API that allows developers to connect to their Golem nodes and manage their
+distributed, computational loads through Golem Network.
 
 ## Golem application development
 
-For a detailed introduction to using Golem and `@golem-sdk/golem-js` to run your tasks on Golem [please consult our quickstart section](https://docs.golem.network/creators/javascript/quickstart/).
+For a detailed introduction to using Golem and `@golem-sdk/golem-js` to run your tasks on
+Golem [please consult our quickstart section](https://docs.golem.network/creators/javascript/quickstart/).
 
 ### Installation
 
@@ -68,7 +96,8 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
 
 ![hello_web](https://user-images.githubusercontent.com/26308335/217530424-a1dd4487-f95f-43e6-a91b-7106b6f30802.gif)
 
-For more detailed usage examples and tutorials, see the [Java Script API section of the Golem Network Docs](https://docs.golem.network/creators/javascript/)
+For more detailed usage examples and tutorials, see
+the [Java Script API section of the Golem Network Docs](https://docs.golem.network/creators/javascript/)
 
 ### Testing
 
@@ -84,9 +113,13 @@ yarn test:unit
 
 ### Running E2E tests
 
-Both test cases for the NodeJS environment and the browser (cypress) require preparation of a test environment of the Golem Network with providers and all the necessary infrastructure. [Goth](https://github.com/golemfactory/goth) framework is used for this purpose.
+Both test cases for the NodeJS environment and the browser (cypress) require preparation of a test environment of the
+Golem Network with Providers and all the necessary infrastructure. [Goth](https://github.com/golemfactory/goth)
+framework is used for this purpose.
 
-To enable E2E testing, you need to ensure that `python -m goth` is executable. Therefore, you must first install [Goth](https://github.com/golemfactory/goth) according to the instructions described in the readme of the project.
+To enable E2E testing, you need to ensure that `python -m goth` is executable. Therefore, you must first
+install [Goth](https://github.com/golemfactory/goth) according to the instructions described in the readme of the
+project.
 
 #### NodeJS
 
@@ -114,12 +147,51 @@ yarn lint
 yarn format
 ```
 
+## Controlling interactions and costs
+
+The Golem Network provides an open marketplace where anyone can join as a Provider and supply the network with their
+computing power. In return for their service, they are billing Requestors (users of this SDK) according to the pricing
+that they define. As a Requestor, you might want to:
+
+- control the limit price so that you're not going to over-spend your funds
+- control the interactions with the providers if you have a list of the ones which you like or the ones which you would
+  like to avoid
+
+To make this easy, we provided you with a set of predefined market proposal filters, which you can combine to implement
+your own market strategy. For example:
+
+```typescript
+import { TaskExecutor, ProposalFilters } from "@golem-sdk/golem-js";
+
+const executor = await TaskExecutor.create({
+  // What do you want to run
+  package: "golem/alpine:3.18.2",
+
+  // How much you wish to spend
+  budget: 0.5,
+  proposalFilter: ProposalFilters.limitPriceFilter({
+    start: 1,
+    cpuPerSec: 1 / 3600,
+    envPerSec: 1 / 3600,
+  }),
+
+  // Where you want to spend
+  payment: {
+    network: "polygon",
+  },
+});
+```
+
+To learn more about other filters, please check the [API reference of the market/strategy module](https://docs.golem.network/docs/golem-js/reference/modules/market_strategy)
+
 ## See also
 
 - [Golem](https://golem.network), a global, open-source, decentralized supercomputer that anyone can access.
 - Learn what you need to know to set up your Golem requestor node:
   - [Requestor development: a quick primer](https://handbook.golem.network/requestor-tutorials/flash-tutorial-of-requestor-development)
   - [Quick start](https://docs.golem.network/creators/javascript/quickstart/)
-- Have a look at the most important concepts behind any Golem application: [Golem application fundamentals](https://handbook.golem.network/requestor-tutorials/golem-application-fundamentals)
-- Learn about preparing your own Docker-like images for the [VM runtime](https://handbook.golem.network/requestor-tutorials/vm-runtime)
+- Have a look at the most important concepts behind any Golem
+  application: [Golem application fundamentals](https://handbook.golem.network/requestor-tutorials/golem-application-fundamentals)
+- Learn about preparing your own Docker-like images for
+  the [VM runtime](https://handbook.golem.network/requestor-tutorials/vm-runtime)
 - Write your own app with [JavaScript API](https://docs.golem.network/creators/javascript/high-level/task-model/)

examples/package.json

@@ -18,7 +18,8 @@
     "fibonacci": "node ./fibonacci/fibonacci.js",
     "ssh": "ts-node ./ssh/ssh.ts",
     "tag": "ts-node ./simple-usage/tag.ts",
-    "web": "node ./web/app.mjs"
+    "web": "node ./web/app.mjs",
+    "lint:ts": "tsc --project tsconfig.json --noEmit"
   },
   "author": "GolemFactory <[email protected]>",
   "license": "LGPL-3.0",

examples/simple-usage/map.ts

@@ -4,9 +4,9 @@ import { TaskExecutor } from "@golem-sdk/golem-js";
   const executor = await TaskExecutor.create("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae");
   const data = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"];
 
-  const results = executor.map<string, string>(data, async (ctx, x) => {
+  const results = executor.map<string, string | undefined>(data, async (ctx, x) => {
     const res = await ctx.run(`echo "${x}"`);
-    return res.stdout?.trim();
+    return res.stdout?.toString().trim();
   });
   const finalOutput: string[] = [];
   for await (const res of results) {

examples/strategy/customProviderFilter.ts

@@ -1,10 +1,10 @@
-import { TaskExecutor, ProposalDTO } from "@golem-sdk/golem-js";
+import { ProposalFilter, TaskExecutor } from "@golem-sdk/golem-js";
 
 /**
  * Example demonstrating how to write a custom proposal filter.
  * In this case the proposal must include VPN access and must not be from "bad-provider"
  */
-const myFilter = async (proposal: ProposalDTO) => {
+const myFilter: ProposalFilter = async (proposal) => {
   return (
     proposal.provider.name !== "bad-provider" || !proposal.properties["golem.runtime.capabilities"]?.includes("vpn")
   );

examples/yacat/yacat.ts

@@ -15,7 +15,7 @@ async function main(args) {
   });
   const keyspace = await executor.run<number>(async (ctx) => {
     const result = await ctx.run(`hashcat --keyspace -a 3 ${args.mask} -m 400`);
-    return parseInt(result.stdout || "");
+    return parseInt(result.stdout?.toString().trim() || "");
   });
 
   if (!keyspace) throw new Error(`Cannot calculate keyspace`);
@@ -34,7 +34,7 @@ async function main(args) {
       .run("cat pass.potfile || true")
       .end();
     if (!results?.[1]?.stdout) return false;
-    return results?.[1]?.stdout.split(":")[1];
+    return results?.[1]?.stdout.toString().trim().split(":")[1];
   });
 
   let password = "";

src/activity/activity.ts

@@ -1,4 +1,4 @@
-import { Result, StreamingBatchEvent } from "./results";
+import { Result, ResultState, StreamingBatchEvent } from "./results";
 import EventSource from "eventsource";
 import { Readable } from "stream";
 import { Logger } from "../utils";
@@ -188,17 +188,17 @@ export class Activity {
             // This will ignore "incompatibility" between ExeScriptCommandResultResultEnum and ResultState, which both
             // contain exactly the same entries, however TSC refuses to compile it as it assumes the former is dynamicaly
             // computed.
-            const { data: results }: { data: Result[] } = (await api.control.getExecBatchResults(
+            const { data: rawExecBachResults } = await api.control.getExecBatchResults(
               activityId,
               batchId,
               undefined,
               activityExecuteTimeout / 1000,
               {
                 timeout: 0,
               },
-            )) as unknown as { data: Result[] };
+            );
             retryCount = 0;
-            const newResults = results.slice(lastIndex + 1);
+            const newResults = rawExecBachResults.map((rawResult) => new Result(rawResult)).slice(lastIndex + 1);
             if (Array.isArray(newResults) && newResults.length) {
               newResults.forEach((result) => {
                 this.push(result);
@@ -333,15 +333,19 @@ export class Activity {
   private parseEventToResult(msg: string, batchSize: number): Result {
     try {
       const event: StreamingBatchEvent = JSON.parse(msg);
-      return {
+      return new Result({
         index: event.index,
         eventDate: event.timestamp,
-        result: event?.kind?.finished ? (event?.kind?.finished?.return_code === 0 ? "Ok" : "Error") : undefined,
+        result: event?.kind?.finished
+          ? event?.kind?.finished?.return_code === 0
+            ? ResultState.Ok
+            : ResultState.Error
+          : ResultState.Error,
         stdout: event?.kind?.stdout,
         stderr: event?.kind?.stderr,
         message: event?.kind?.finished?.message,
         isBatchFinished: event.index + 1 >= batchSize && Boolean(event?.kind?.finished),
-      } as Result;
+      });
     } catch (error) {
       throw new Error(`Cannot parse ${msg} as StreamingBatchEvent`);
     }

src/activity/results.test.ts

@@ -0,0 +1,39 @@
+import { Result, ResultState } from "./results";
+
+describe("Results", () => {
+  describe("converting output to JSON", () => {
+    describe("positive cases", () => {
+      test("produces JSON when the stdout contains correct data", () => {
+        const result = new Result({
+          index: 0,
+          result: ResultState.Ok,
+          stdout: '{ "value": 55 }\n',
+          stderr: null,
+          message: null,
+          isBatchFinished: true,
+          eventDate: "2023-08-29T09:23:52.305095307Z",
+        });
+
+        expect(result.getOutputAsJson()).toEqual({
+          value: 55,
+        });
+      });
+    });
+
+    describe("negative cases", () => {
+      test("throws an error when stdout does not contain nice JSON", () => {
+        const result = new Result({
+          index: 0,
+          result: ResultState.Ok,
+          stdout: "not json\n",
+          stderr: null,
+          message: null,
+          isBatchFinished: true,
+          eventDate: "2023-08-29T09:23:52.305095307Z",
+        });
+
+        expect(() => result.getOutputAsJson()).toThrow("Failed to parse output to JSON!");
+      });
+    });
+  });
+});

src/activity/results.ts

@@ -1,29 +1,71 @@
-export enum ResultState {
-  OK = "Ok",
-  ERROR = "Error",
-}
+import { ExeScriptCommandResultResultEnum } from "ya-ts-client/dist/ya-activity/src/models/exe-script-command-result";
+
+export import ResultState = ExeScriptCommandResultResultEnum;
 
 /**
  * @hidden
  */
-export interface Result<T = unknown> {
+// FIXME: Make the `data` field Uint8Array and update the rest of the code
+// eslint-disable-next-line
+export interface ResultData<T = any> {
   /** Index of script command */
   index: number;
   /** The datetime of the event on which the result was received */
   eventDate: string;
   /** If is success */
   result: ResultState;
   /** stdout of script command */
-  stdout?: string;
+  stdout?: string | ArrayBuffer | null;
   /** stderr of script command */
-  stderr?: string;
+  stderr?: string | ArrayBuffer | null;
   /** an error message if the result is not successful */
-  message?: string;
+  message?: string | null;
   /** Is batch of already finished */
   isBatchFinished?: boolean;
+
+  /** In case the command was related to upload or download, this will contain the transferred data */
   data?: T;
 }
 
+// FIXME: Make the `data` field Uint8Array and update the rest of the code
+// eslint-disable-next-line
+export class Result<TData = any> implements ResultData<TData> {
+  index: number;
+  eventDate: string;
+  result: ResultState;
+  stdout?: string | ArrayBuffer | null;
+  stderr?: string | ArrayBuffer | null;
+  message?: string | null;
+  isBatchFinished?: boolean;
+  data?: TData;
+
+  constructor(props: ResultData) {
+    this.index = props.index;
+    this.eventDate = props.eventDate;
+    this.result = props.result;
+    this.stdout = props.stdout;
+    this.stderr = props.stderr;
+    this.message = props.message;
+    this.isBatchFinished = props.isBatchFinished;
+    this.data = props.data;
+  }
+
+  /**
+   * Helper method making JSON-like output results more accessible
+   */
+  public getOutputAsJson<Output = object>(): Output {
+    if (!this.stdout) {
+      throw new Error("Can't convert Result output to JSON, because the output is missing!");
+    }
+
+    try {
+      return JSON.parse(this.stdout.toString().trim());
+    } catch (err) {
+      throw new Error(`Failed to parse output to JSON! Output: "${this.stdout.toString()}". Error: ${err}`);
+    }
+  }
+}
+
 export interface StreamingBatchEvent {
   batch_id: string;
   index: number;

src/agreement/service.ts

@@ -2,7 +2,7 @@ import Bottleneck from "bottleneck";
 import { Logger } from "../utils";
 import { Agreement, AgreementOptions, AgreementStateEnum } from "./agreement";
 import { AgreementServiceConfig } from "./config";
-import { Proposal, ProposalDTO } from "../market/proposal";
+import { Proposal } from "../market/proposal";
 import sleep from "../utils/sleep";
 
 export interface AgreementDTO {
@@ -12,7 +12,7 @@ export interface AgreementDTO {
 
 export class AgreementCandidate {
   agreement?: AgreementDTO;
-  constructor(readonly proposal: ProposalDTO) {}
+  constructor(readonly proposal: Proposal) {}
 }
 
 export type AgreementSelector = (candidates: AgreementCandidate[]) => Promise<AgreementCandidate>;

src/executor/config.ts

@@ -7,7 +7,7 @@ const DEFAULTS = Object.freeze({
   payment: { driver: "erc20", network: "goerli" },
   budget: 1.0,
   subnetTag: "public",
-  logLevel: LogLevel.info,
+  logLevel: LogLevel.Info,
   basePath: "http://127.0.0.1:7465",
   maxParallelTasks: 5,
   taskTimeout: 1000 * 60 * 5, // 5 min,