diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fa4664dc..68adb1f3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: name: Build and unit-test on supported platforms and NodeJS versions strategy: matrix: - node-version: [16.x, 18.x, 20.x] + node-version: [18.x, 20.x] os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/cypress-nightly.yml b/.github/workflows/cypress-nightly.yml index c2ed3acee..ca7253bf4 100644 --- a/.github/workflows/cypress-nightly.yml +++ b/.github/workflows/cypress-nightly.yml @@ -10,63 +10,37 @@ on: jobs: cypress-tests: - runs-on: [goth2, ubuntu-22.10] + runs-on: goth2 steps: - name: Checkout uses: actions/checkout@v3 - - name: Configure node.js - uses: actions/setup-node@v3 - with: - node-version: 18 + - name: Use random string for subnet + run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - - name: Install browsers and graphic environment - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential - sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt install --allow-downgrades -y ./google-chrome-stable_current_amd64.deb + - name: Build the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml build - - name: Build the SDK - run: | - npm install - npm run build + - name: Start the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - - name: Configure python - continue-on-error: true - uses: actions/setup-python@v4 - with: - python-version: "3.10" + - name: Fund the requestor + # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. + run: sleep 4 && docker exec -t docker-requestor-1 /bin/sh -c "/golem-js/tests/docker/fundRequestor.sh" - - name: Install goth + - name: Install and build the SDK in the docker container run: | - pip install goth - rm -rf ../goth/assets - python -m goth create-assets ../goth/assets - sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml - sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile - - - name: Cleanup Docker - if: always() - run: | - c=$(docker ps -q) && [[ $c ]] && docker kill $c - docker system prune -af - - - name: Log in to GitHub Docker repository - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run build && ./node_modules/.bin/cypress install" - name: Run web server run: | - cd examples/web - node app.mjs & + docker exec -t -d docker-requestor-1 /bin/sh -c "cd /golem-js/examples/web && node app.mjs" - name: Run test suite - env: - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - npm run test:cypress -- --browser chrome + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm run test:cypress -- --browser chromium" - name: Upload test logs uses: actions/upload-artifact@v2 @@ -75,11 +49,6 @@ jobs: name: cypress-logs path: .cypress - # Only relevant for self-hosted runners - - name: Remove test logs - if: always() - run: rm -rf .cypress - - name: Cleanup Docker if: always() run: | diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 4833088ca..a5daacefd 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -21,7 +21,7 @@ jobs: run: echo "::set-output name=matrix::{\"include\":[{\"branch\":\"master\"}]}" goth-tests: - runs-on: [goth2, ubuntu-22.10] + runs-on: goth2 needs: prepare-matrix-master-only strategy: matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }} @@ -31,61 +31,28 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Configure node.js - uses: actions/setup-node@v3 - with: - node-version: 18 + - name: Use random string for subnet + run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - - name: Build golem-js - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential - npm install - npm run build - npm install --prefix examples - npm install ts-node + - name: Build the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml build - - name: Configure python - continue-on-error: true - uses: actions/setup-python@v4 - with: - python-version: "3.10" + - name: Start the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - - name: Install goth - run: | - pip install goth>=0.15.3 - rm -rf ../goth/assets - python -m goth create-assets ../goth/assets - sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc21"/mg' ../goth/assets/goth-config.yml - sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile + - name: Fund the requestor + # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. + run: sleep 10 && docker exec -t docker-requestor-1 /bin/sh -c "/golem-js/tests/docker/fundRequestor.sh" - - name: Cleanup Docker - if: always() + - name: Install and build the SDK in the docker container run: | - c=$(docker ps -q) && [[ $c ]] && docker kill $c - docker system prune -af + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run build" - - name: Log in to GitHub Docker repository - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin - - - name: Run test suite - env: - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run the Examples tests run: | - npm run test:examples - - - name: Upload test logs - uses: actions/upload-artifact@v2 - if: always() - with: - name: goth-logs - path: /tmp/goth-tests - - # Only relevant for self-hosted runners - - name: Remove test logs - if: always() - run: rm -rf /tmp/goth-tests + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm install --prefix examples && npm install ts-node && npm run test:examples -- --exitOnError" - name: Cleanup Docker if: always() diff --git a/.github/workflows/goth-nightly.yml b/.github/workflows/goth-nightly.yml index 1106a7b26..ca10cb93d 100644 --- a/.github/workflows/goth-nightly.yml +++ b/.github/workflows/goth-nightly.yml @@ -21,7 +21,7 @@ jobs: run: echo "::set-output name=matrix::{\"include\":[{\"branch\":\"master\"}]}" goth-tests: - runs-on: [goth2, ubuntu-22.10] + runs-on: goth2 needs: prepare-matrix-master-only strategy: matrix: ${{ fromJson(needs.prepare-matrix-master-only.outputs.matrix-json) }} @@ -31,64 +31,27 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Configure node.js - uses: actions/setup-node@v3 - with: - node-version: 18 + - name: Use random string for subnet + run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - - name: Build golem-js - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential - npm install - npm run build - - - name: Configure python - continue-on-error: true - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install goth - run: | - pip install goth - rm -rf ../goth/assets - python -m goth create-assets ../goth/assets - sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml + - name: Build the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml build - - name: Install websocat and sshpass - run: | - sudo wget https://github.com/vi/websocat/releases/download/v1.9.0/websocat_linux64 -O /usr/local/bin/websocat - sudo chmod +x /usr/local/bin/websocat - sudo apt-get install sshpass - - - name: Cleanup Docker - if: always() - run: | - c=$(docker ps -q) && [[ $c ]] && docker kill $c - docker system prune -af + - name: Start the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - - name: Log in to GitHub Docker repository - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin + - name: Fund the requestor + # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. + run: sleep 10 && docker exec -t docker-requestor-1 /bin/sh -c "/golem-js/tests/docker/fundRequestor.sh" - - name: Run test suite - env: - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install and build the SDK in the docker container run: | - npm run test:e2e + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run build" - - name: Upload test logs - uses: actions/upload-artifact@v2 - if: always() - with: - name: goth-logs - path: /tmp/goth-tests - - # Only relevant for self-hosted runners - - name: Remove test logs - if: always() - run: rm -rf /tmp/goth-tests + - name: Start the e2e test + run: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run test:e2e -- --reporters github-actions --reporters summary" - name: Cleanup Docker if: always() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6f8dc4dce..ef1b52405 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ jobs: name: Build and unit-test on supported platforms and NodeJS versions strategy: matrix: - node-version: [16.x, 18.x, 20.x] + node-version: [18.x, 20.x] os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} @@ -54,131 +54,53 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Configure NodeJS - uses: actions/setup-node@v3 - with: - # other versions are tested beforehand, so we can keep it short - node-version: 16 - - - name: Install packages required to set-up Goth - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential - - - name: Install browsers and graphic environment for Cypress tests - run: | - sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt install -y ./google-chrome-stable_current_amd64.deb + - name: Use random string for subnet + run: echo "YAGNA_SUBNET=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8 ; echo '')" >> $GITHUB_ENV - - name: Install websocat and sshpass (required by some tests) - run: | - sudo wget https://github.com/vi/websocat/releases/download/v1.9.0/websocat_linux64 -O /usr/local/bin/websocat - sudo chmod +x /usr/local/bin/websocat - sudo apt-get install sshpass - - - name: Build the SDK - run: | - npm install - npm run build - npm install --prefix examples - npm install ts-node + - name: Build the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml build - - name: Configure python - uses: actions/setup-python@v4 - with: - python-version: "3.9" + - name: Start the docker containers + # Use a random string to avoid other providers on the same subnet which might cause tests to fail because it expects only providers named provider-1 and provider-2 + run: docker compose -f tests/docker/docker-compose.yml down && docker compose -f tests/docker/docker-compose.yml up -d - #region Goth Setup - - name: Install Goth - run: | - pip install goth - rm -rf ../goth/assets - python -m goth create-assets ../goth/assets - sed -Ezi 's/(use\-proxy:\s)(True)/\1False/mg' ../goth/assets/goth-config.yml - sed -Ezi 's/(use\-prerelease:\s)(false)/\1true\n release-tag: "0.13.0-rc10"/mg' ../goth/assets/goth-config.yml - sed -i '/^ENTRYPOINT/i ENV YAGNA_AUTOCONF_APPKEY=try_golem' ../goth/assets/docker/yagna-goth-deb.Dockerfile + - name: Fund the requestor + # Use a funding script which will retry funding the requestor 3 times, else it exits with error. The faucet is not reliable and sometimes fails to fund the requestor, thus the retry. + run: sleep 4 && docker exec -t docker-requestor-1 /bin/sh -c "/golem-js/tests/docker/fundRequestor.sh" - - name: Cleanup Docker - if: always() + - name: Install and build the SDK in the docker container run: | - c=$(docker ps -q) && [[ $c ]] && docker kill $c - docker system prune -af + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm i && npm run build && ./node_modules/.bin/cypress install && npm install --prefix examples && npm install ts-node" - - name: Log in to GitHub Docker repository - run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin - #endregion - - #region E2E test execution - - name: Run the E2E tests using Goth - env: - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - npm run test:e2e - - - name: Upload test logs - uses: actions/upload-artifact@v2 - if: always() - with: - name: goth-logs - path: /tmp/goth-tests - - # Only relevant for self-hosted runners - - name: Remove test logs - if: always() - run: rm -rf /tmp/goth-tests - #endregion + - name: Start the e2e test + run: docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm run test:e2e" #region Cypress test execution - name: Run web server run: | - cd examples/web - node app.mjs & + docker exec -t -d docker-requestor-1 /bin/sh -c "cd /golem-js/examples/web && node app.mjs" - name: Run test suite - env: - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - npm run test:cypress -- --browser chrome + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm run test:cypress -- --browser chromium" - - name: Upload test logs - uses: actions/upload-artifact@v2 - if: always() - with: - name: cypress-logs - path: .cypress - - # Only relevant for self-hosted runners - - name: Remove test logs - if: always() - run: rm -rf .cypress - #endregion - - #region Examples test execution - - name: Run the Examples tests using Goth - env: - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run the Examples tests run: | - npm run test:examples + docker exec -t docker-requestor-1 /bin/sh -c "cd /golem-js && npm run test:examples -- --exitOnError" - name: Upload test logs uses: actions/upload-artifact@v2 if: always() with: - name: goth-logs - path: /tmp/goth-tests - - # Only relevant for self-hosted runners - - name: Remove test logs - if: always() - run: rm -rf /tmp/goth-tests + name: cypress-logs + path: .cypress - name: Cleanup Docker if: always() run: | c=$(docker ps -q) && [[ $c ]] && docker kill $c docker system prune -af - #endregion release: name: Release the SDK to NPM and GitHub diff --git a/README.md b/README.md index 2da5a0352..abf13775f 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,25 @@ -- [Golem JavaScript API](#golem-javascript-api) - - [Table of contents](#table-of-contents) - - [What's Golem and `golem-js`?](#whats-golem-and-golem-js) - - [Golem application development](#golem-application-development) - - [Installation](#installation) - - [Building](#building) - - [Usage](#usage) - - [Node.js context](#nodejs-context) - - [Web Browser context](#web-browser-context) - - [Testing](#testing) - - [Running unit tests](#running-unit-tests) - - [Running E2E tests](#running-e2e-tests) - - [NodeJS](#nodejs) - - [Cypress](#cypress) - - [Contributing](#contributing) - - [Controlling interactions and costs](#controlling-interactions-and-costs) - - [See also](#see-also) - +- [Table of contents](#table-of-contents) +- [What's Golem and `golem-js`?](#whats-golem-and-golem-js) +- [Golem application development](#golem-application-development) + - [Installation](#installation) + - [Building](#building) + - [Usage](#usage) + - [Node.js context](#nodejs-context) + - [Web Browser context](#web-browser-context) + - [Testing](#testing) + - [Running unit tests](#running-unit-tests) + - [Running E2E tests](#running-e2e-tests) + - [NodeJS](#execute-the-e2e-tests) + - [Cypress](#execute-the-cypress-tests) + - [Contributing](#contributing) +- [Controlling interactions and costs](#controlling-interactions-and-costs) + - [Limit price limits to filter out offers that are too expensive](#limit-price-limits-to-filter-out-offers-that-are-too-expensive) + - [Work with reliable providers](#work-with-reliable-providers) +- [See also](#see-also) + ![GitHub](https://img.shields.io/github/license/golemfactory/golem-js) ![npm](https://img.shields.io/npm/v/@golem-sdk/golem-js) @@ -114,27 +115,69 @@ yarn test:unit ### Running E2E tests Both test cases for the NodeJS environment and the browser (cypress) require preparation of a test environment of the -Golem Network with Providers and all the necessary infrastructure. [Goth](https://github.com/golemfactory/goth) -framework is used for this purpose. +Golem Network with Providers and all the necessary infrastructure. -To enable E2E testing, you need to ensure that `python -m goth` is executable. Therefore, you must first -install [Goth](https://github.com/golemfactory/goth) according to the instructions described in the readme of the -project. +#### Prerequisites -#### NodeJS +1. Ensure you have `docker` and `docker-compose` installed in your system. +2. Your Linux environment should have nested virtualization enabled. -```bash -npm run test:e2e -# or -yarn test:e2e +#### Test Environment Preparation + +Follow these steps to prepare your test environment: + +##### Build Docker Containers + +First, build the Docker containers using the `docker-compose.yml` file located under `tests/docker`. + +Execute this command to build the Docker containers: + + docker-compose -f tests/docker/docker-compose.yml build + +##### Start Docker Containers + +Then, launch the Docker containers you've just built using the same `docker-compose.yml` file. + +Execute this command to start the Docker containers: + + docker-compose -f tests/docker/docker-compose.yml down && docker-compose -f tests/docker/docker-compose.yml up -d + +##### Fund the Requestor + +The next step is to fund the requestor. + + docker exec -t docker_requestor_1 /bin/sh -c "/golem-js/tests/docker/fundRequestor.sh" + +##### Install and Build the SDK + +Finally, install and build the golem-js SDK in the Docker container + +Run this chain of commands to install and build the SDK and prepare cypress. + +```docker +docker exec -t docker_requestor_1 /bin/sh -c "cd /golem-js && npm i && npm run build && ./node_modules/.bin/cypress install" ``` -#### Cypress +#### Execute the E2E Tests -```bash -npm run test:cypress -# or -yarn test:cypress +With your test environment set up, you can now initiate the E2E tests. Run the following command to start: + +```docker +docker exec -t docker_requestor_1 /bin/sh -c "cd /golem-js && npm run test:e2e" +``` + +#### Execute the cypress Tests + +First make sure that the webserver that's used for testing is running, by running the command + +```docker +docker exec -t -d docker_requestor_1 /bin/sh -c "cd /golem-js/examples/web && node app.mjs" +``` + +Now you're ready to start the cypress tests by running the command + +```docker +docker exec -t docker_requestor_1 /bin/sh -c "cd /golem-js && npm run test:cypress -- --browser chromium" ``` ### Contributing @@ -158,7 +201,9 @@ that they define. As a Requestor, you might want to: like to avoid To make this easy, we provided you with a set of predefined market proposal filters, which you can combine to implement -your own market strategy. For example: +your own market strategy. + +### Limit price limits to filter out offers that are too expensive ```typescript import { TaskExecutor, ProposalFilters } from "@golem-sdk/golem-js"; @@ -184,6 +229,41 @@ const executor = await TaskExecutor.create({ To learn more about other filters, please check the [API reference of the market/strategy module](https://docs.golem.network/docs/golem-js/reference/modules/market_strategy) +### Work with reliable providers + +The `getHealthyProvidersWhiteList` helper will provide you with a list of Provider ID's that were checked with basic health-checks. Using this whitelist will increase the chance of working with a reliable provider. Please note, that you can also build up your own list of favourite providers and use it in a similar fashion. + +```typescript +import { TaskExecutor, ProposalFilters, MarketHelpers } from "@golem-sdk/golem-js"; + +// Prepare the price filter +const acceptablePrice = ProposalFilters.limitPriceFilter({ + start: 1, + cpuPerSec: 1 / 3600, + envPerSec: 1 / 3600, +}); + +// Collect the whitelist +const verifiedProviders = await MarketHelpers.getHealthyProvidersWhiteList(); + +// Prepare the whitelist filter +const whiteList = ProposalFilters.whiteListProposalIdsFilter(verifiedProviders); + +const executor = await TaskExecutor.create({ + // What do you want to run + package: "golem/alpine:3.18.2", + + // How much you wish to spend + budget: 0.5, + proposalFilter: async (proposal) => (await acceptablePrice(proposal)) && (await whiteList(proposal)), + + // Where you want to spend + payment: { + network: "polygon", + }, +}); +``` + ## See also - [Golem](https://golem.network), a global, open-source, decentralized supercomputer that anyone can access. diff --git a/cypress.config.ts b/cypress.config.ts index f76baf63e..4ce809e49 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -1,9 +1,4 @@ import { defineConfig } from "cypress"; -import { Goth } from "./tests/goth/goth"; -import { resolve } from "path"; - -const gothConfig = resolve("../goth/assets/goth-config.yml"); -const goth = new Goth(gothConfig); export default defineConfig({ fileServerFolder: "examples/web", @@ -11,7 +6,7 @@ export default defineConfig({ fixturesFolder: "tests/cypress/fixtures", videosFolder: ".cypress/video", screenshotsFolder: ".cypress/screenshots", - defaultCommandTimeout: 90000, + defaultCommandTimeout: 180000, experimentalInteractiveRunEvents: true, chromeWebSecurity: false, video: true, @@ -20,14 +15,10 @@ export default defineConfig({ supportFile: "tests/cypress/support/e2e.ts", specPattern: "tests/cypress/ui/**/*.cy.ts", setupNodeEvents(on, config) { - on("after:run", async () => { - await goth.end(); - }); return new Promise(async (res) => { - const { apiKey, basePath, subnetTag } = await goth.start(); - config.env.YAGNA_APPKEY = apiKey; - config.env.YAGNA_API_BASEPATH = basePath; - config.env.YAGNA_SUBNET = subnetTag; + config.env.YAGNA_APPKEY = process.env.YAGNA_APPKEY; + config.env.YAGNA_API_BASEPATH = process.env.YAGNA_API_URL; + config.env.YAGNA_SUBNET = process.env.YAGNA_SUBNET; res(config); }); }, diff --git a/examples/docs-examples/examples/composing-tasks/alert-code.mjs b/examples/docs-examples/examples/composing-tasks/alert-code.mjs new file mode 100644 index 000000000..dde33f6a8 --- /dev/null +++ b/examples/docs-examples/examples/composing-tasks/alert-code.mjs @@ -0,0 +1,23 @@ +import { TaskExecutor } from "@golem-sdk/golem-js"; +(async () => { + const executor = await TaskExecutor.create({ + package: "529f7fdaf1cf46ce3126eb6bbcd3b213c314fe8fe884914f5d1106d4", + yagnaOptions: { apiKey: "try_golem" }, + }); + + const result = await executor.run(async (ctx) => { + const res = await ctx + .beginBatch() + .uploadFile("./worker.mjs", "/golem/input/worker.mjs") + .run("node /golem/input/worker.mjs > /golem/input/output.txt") + .run("cat /golem/input/output.txt") + .downloadFile("/golem/input/output.txt", "./output.txt") + .endStream(); + + for await (const chunk of res) { + chunk.index == 2 ? console.log(chunk.stdout) : ""; + } + }); + + await executor.end(); +})(); diff --git a/examples/docs-examples/examples/selecting-providers/custom-price.mjs b/examples/docs-examples/examples/selecting-providers/custom-price.mjs index b3e785812..d4a604006 100644 --- a/examples/docs-examples/examples/selecting-providers/custom-price.mjs +++ b/examples/docs-examples/examples/selecting-providers/custom-price.mjs @@ -31,6 +31,7 @@ const myFilter = async (proposal) => { package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", proposalFilter: myFilter, yagnaOptions: { apiKey: "try_golem" }, + startupTimeout: 60_000, }); await executor.run(async (ctx) => console.log((await ctx.run(`echo "This task is run on ${ctx.provider.id}"`)).stdout, ctx.provider.id), diff --git a/examples/docs-examples/examples/selecting-providers/whitelist.mjs b/examples/docs-examples/examples/selecting-providers/whitelist.mjs index 26bdcd6fe..120ad157a 100644 --- a/examples/docs-examples/examples/selecting-providers/whitelist.mjs +++ b/examples/docs-examples/examples/selecting-providers/whitelist.mjs @@ -1,31 +1,24 @@ import { TaskExecutor, ProposalFilters } from "@golem-sdk/golem-js"; /** - * Example demonstrating how to use the predefined filter `whiteListProposalIdsFilter`, - * which only allows offers from a provider whose ID is in the array + * Example demonstrating how to use the predefined filter `whiteListProposalNamesFilter`, + * which only allows offers from a provider whose name is in the array */ -const whiteListIds = [ - "0x3a21c608925ddbc745afab6375d1f5e77283538e", - "0xd79f83f1108d1fcbe0cf57e13b452305eb38a325", - "0x677c5476f3b0e1f03d5c3abd2e2e2231e36fddde", - "0x06c03165aaa676680b9d02c1f3ee846c3806fec7", - "0x17ec8597ff92c3f44523bdc65bf0f1be632917ff", // goth provider-1: - "0x63fc2ad3d021a4d7e64323529a55a9442c444da0", // goth provider-2: -]; +const whiteListNames = ["provider-2", "fractal_01_3.h", "sharkoon_379_0.h", "fractal_01_1.h", "sharkoon_379_1.h"]; console.log("Will accept only proposals from:"); -for (let i = 0; i < whiteListIds.length; i++) { - console.log(whiteListIds[i]); +for (let i = 0; i < whiteListNames.length; i++) { + console.log(whiteListNames[i]); } (async function main() { const executor = await TaskExecutor.create({ package: "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", - proposalFilter: ProposalFilters.whiteListProposalIdsFilter(whiteListIds), + proposalFilter: ProposalFilters.whiteListProposalNamesFilter(whiteListNames), yagnaOptions: { apiKey: "try_golem" }, }); await executor.run(async (ctx) => - console.log((await ctx.run(`echo "This task is run on ${ctx.provider.id}"`)).stdout, ctx.provider.id), + console.log((await ctx.run(`echo "This task is run on ${ctx.provider.name}"`)).stdout, ctx.provider.name), ); await executor.end(); })(); diff --git a/examples/docs-examples/quickstarts/retrievable-task/task.mjs b/examples/docs-examples/quickstarts/retrievable-task/task.mjs new file mode 100644 index 000000000..830ace4f5 --- /dev/null +++ b/examples/docs-examples/quickstarts/retrievable-task/task.mjs @@ -0,0 +1,23 @@ +import { GolemNetwork, JobState } from "@golem-sdk/golem-js"; + +const golem = new GolemNetwork({ + yagnaOptions: { apiKey: "try_golem" }, +}); +await golem.init(); +const job = await golem.createJob(async (ctx) => { + const response = await ctx.run("echo 'Hello, Golem!'"); + return response.stdout; +}); + +let state = await job.fetchState(); +while (state === JobState.Pending || state === JobState.New) { + console.log("Job is still running..."); + await new Promise((resolve) => setTimeout(resolve, 1000)); + state = await job.fetchState(); +} + +console.log("Job finished with state:", state); +const result = await job.fetchResults(); +console.log("Job results:", result); + +await golem.close(); diff --git a/package.json b/package.json index 3324690ab..2733d5b46 100644 --- a/package.json +++ b/package.json @@ -27,10 +27,8 @@ "test": "npm run test:unit && npm run test:e2e", "test:unit": "jest --config tests/unit/jest.config.json", "test:e2e": "jest --config tests/e2e/jest.config.json tests/e2e/**.spec.ts --runInBand --forceExit", - "test:e2e:no-goth": "jest tests/e2e/**.spec.ts --testTimeout=180000 --runInBand --forceExit", "test:cypress": "cypress run", "test:examples": "ts-node --project tsconfig.spec.json tests/examples/examples.test.ts", - "test:examples:no-goth": "ts-node --project tsconfig.spec.json tests/examples/examples.test.ts --no-goth", "lint": "npm run lint:ts && npm run lint:ts:tests && npm run lint:eslint", "lint:ts": "tsc --project tsconfig.json --noEmit", "lint:ts:tests": "tsc --project tests/tsconfig.json --noEmit", @@ -45,10 +43,9 @@ "author": "GolemFactory ", "license": "LGPL-3.0", "engines": { - "node": ">=16.0.0" + "node": ">=18.0.0" }, "dependencies": { - "@rauschma/stringio": "^1.4.0", "axios": "^1.1.3", "bottleneck": "^2.19.5", "collect.js": "^4.34.3", @@ -65,6 +62,7 @@ "devDependencies": { "@commitlint/cli": "^17.7.1", "@commitlint/config-conventional": "^17.7.0", + "@johanblumenberg/ts-mockito": "^1.0.39", "@rollup/plugin-alias": "^5.0.0", "@rollup/plugin-commonjs": "^25.0.3", "@rollup/plugin-json": "^6.0.0", @@ -88,6 +86,7 @@ "jest": "^29.6.2", "prettier": "^3.0.0", "rollup": "^3.26.3", + "rollup-plugin-filesize": "^10.0.0", "rollup-plugin-ignore": "^1.0.10", "rollup-plugin-polyfill-node": "^0.12.0", "rollup-plugin-visualizer": "^5.9.0", diff --git a/rollup.config.mjs b/rollup.config.mjs index 57dd47c5f..31e85f961 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -7,6 +7,7 @@ import typescript from "@rollup/plugin-typescript"; import nodePolyfills from "rollup-plugin-polyfill-node"; import pkg from "./package.json" assert { type: "json" }; import ignore from "rollup-plugin-ignore"; +import filesize from "rollup-plugin-filesize"; /** * Looking for plugins? @@ -41,8 +42,9 @@ export default [ commonjs(), nodePolyfills(), json(), // Required because one our dependencies (bottleneck) loads its own 'version.json' - typescript({ tsconfig: "./tsconfig.json" }), + typescript({ tsconfig: "./tsconfig.json", exclude: ["**/__tests__", "**/*.test.ts"] }), terser({ keep_classnames: true }), + filesize({ reporter: [sizeValidator, "boxen"] }), ], }, // NodeJS @@ -52,6 +54,15 @@ export default [ { file: pkg.main, format: "cjs", sourcemap: true }, { file: pkg.module, format: "es", sourcemap: true }, ], - plugins: [typescript({ tsconfig: "./tsconfig.json" })], + plugins: [ + typescript({ tsconfig: "./tsconfig.json", exclude: ["**/__tests__", "**/*.test.ts"] }), + filesize({ reporter: [sizeValidator, "boxen"] }), + ], }, ]; + +function sizeValidator(options, bundle, { bundleSize }) { + if (parseInt(bundleSize) === 0) { + throw new Error(`Something went wrong while building. Bundle size = ${bundleSize}`); + } +} diff --git a/src/executor/config.ts b/src/executor/config.ts index c553bdeef..c0d3d61bc 100644 --- a/src/executor/config.ts +++ b/src/executor/config.ts @@ -14,6 +14,7 @@ const DEFAULTS = Object.freeze({ taskTimeout: 1000 * 60 * 5, // 5 min, maxTaskRetries: 3, enableLogging: true, + startupTimeout: 1000 * 30, // 30 sec }); /** @@ -34,6 +35,7 @@ export class ExecutorConfig { readonly maxTaskRetries: number; readonly activityExecuteTimeout?: number; readonly jobStorage: JobStorage; + readonly startupTimeout: number; constructor(options: ExecutorOptions & ActivityOptions) { const processEnv = !runtimeContextChecker.isBrowser @@ -83,5 +85,6 @@ export class ExecutorConfig { this.eventTarget = options.eventTarget || new EventTarget(); this.maxTaskRetries = options.maxTaskRetries ?? DEFAULTS.maxTaskRetries; this.jobStorage = options.jobStorage || new InMemoryJobStorage(); + this.startupTimeout = options.startupTimeout ?? DEFAULTS.startupTimeout; } } diff --git a/src/executor/executor.test.ts b/src/executor/executor.test.ts new file mode 100644 index 000000000..d247338d3 --- /dev/null +++ b/src/executor/executor.test.ts @@ -0,0 +1,64 @@ +import { MarketService } from "../market/"; +import { AgreementPoolService } from "../agreement/"; +import { TaskService } from "../task/"; +import { TaskExecutor } from "./executor"; +import { sleep } from "../utils"; +import { LoggerMock } from "../../tests/mock"; + +jest.mock("../market/service"); +jest.mock("../agreement/service"); +jest.mock("../network/service"); +jest.mock("../task/service"); +jest.mock("../storage/gftp"); +jest.mock("../utils/yagna/yagna"); + +const serviceRunSpy = jest.fn().mockImplementation(() => Promise.resolve()); +jest.spyOn(MarketService.prototype, "run").mockImplementation(serviceRunSpy); +jest.spyOn(AgreementPoolService.prototype, "run").mockImplementation(serviceRunSpy); +jest.spyOn(TaskService.prototype, "run").mockImplementation(serviceRunSpy); + +jest.mock("../payment/service", () => { + return { + PaymentService: jest.fn().mockImplementation(() => { + return { + config: { payment: { network: "test" } }, + createAllocation: jest.fn(), + run: serviceRunSpy, + end: jest.fn(), + }; + }), + }; +}); + +describe("Task Executor", () => { + const logger = new LoggerMock(); + const yagnaOptions = { apiKey: "test" }; + beforeEach(() => { + jest.clearAllMocks(); + logger.clear(); + }); + + describe("init()", () => { + it("should run all set services", async () => { + const executor = await TaskExecutor.create({ package: "test", logger, yagnaOptions }); + expect(serviceRunSpy).toHaveBeenCalledTimes(4); + expect(executor).toBeDefined(); + await executor.end(); + }); + it("should handle a critical error if startup timeout is reached", async () => { + const executor = await TaskExecutor.create({ package: "test", startupTimeout: 0, logger, yagnaOptions }); + jest + .spyOn(MarketService.prototype, "getProposalsCount") + .mockImplementation(() => ({ confirmed: 0, initial: 0, rejected: 0 })); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const handleErrorSpy = jest.spyOn(executor as any, "handleCriticalError").mockImplementation((error) => { + expect((error as Error).message).toEqual( + "Could not start any work on Golem. Processed 0 initial proposals from yagna, filters accepted 0. Check your demand if it's not too restrictive or restart yagna.", + ); + }); + await sleep(10, true); + expect(handleErrorSpy).toHaveBeenCalled(); + await executor.end(); + }); + }); +}); diff --git a/src/executor/executor.ts b/src/executor/executor.ts index eeb2eb637..4f5a7c728 100644 --- a/src/executor/executor.ts +++ b/src/executor/executor.ts @@ -54,6 +54,24 @@ export type ExecutorOptions = { * For more details see {@link JobStorage}. Defaults to a simple in-memory storage. */ jobStorage?: JobStorage; + /** + * Do not install signal handlers for SIGINT, SIGTERM, SIGBREAK, SIGHUP. + * + * By default, TaskExecutor will install those and terminate itself when any of those signals is received. + * This is to make sure proper shutdown with completed invoice payments. + * + * Note: If you decide to set this to `true`, you will be responsible for proper shutdown of task executor. + */ + skipProcessSignals?: boolean; + /** + * Timeout for waiting for at least one offer from the market. + * This parameter (set to 30 sec by default) will throw an error when executing `TaskExecutor.run` + * if no offer from the market is accepted before this time. + * You can set a slightly higher time in a situation where your parameters such as proposalFilter + * or minimum hardware requirements are quite restrictive and finding a suitable provider + * that meets these criteria may take a bit longer. + */ + startupTimeout?: number; } & Omit & MarketOptions & TaskServiceOptions & @@ -91,8 +109,11 @@ export class TaskExecutor { private isRunning = true; private configOptions: ExecutorOptions; private isCanceled = false; + private startupTimeoutId?: NodeJS.Timeout; private yagna: Yagna; + private signalHandler = (signal: string) => this.cancel(signal); + /** * Create a new Task Executor * @description Factory Method that create and initialize an instance of the TaskExecutor @@ -205,14 +226,16 @@ export class TaskExecutor { this.logger?.debug("Initializing task executor services..."); const allocations = await this.paymentService.createAllocation(); - this.marketService.run(taskPackage, allocations).catch((e) => this.handleCriticalError(e)); - this.agreementPoolService.run().catch((e) => this.handleCriticalError(e)); - this.paymentService.run().catch((e) => this.handleCriticalError(e)); + await Promise.all([ + this.marketService.run(taskPackage, allocations).then(() => this.setStartupTimeout()), + this.agreementPoolService.run(), + this.paymentService.run(), + this.networkService?.run(), + this.statsService.run(), + this.storageProvider?.init(), + ]).catch((e) => this.handleCriticalError(e)); this.taskService.run().catch((e) => this.handleCriticalError(e)); - this.networkService?.run().catch((e) => this.handleCriticalError(e)); - this.statsService.run().catch((e) => this.handleCriticalError(e)); - this.storageProvider?.init().catch((e) => this.handleCriticalError(e)); - if (runtimeContextChecker.isNode) this.handleCancelEvent(); + if (runtimeContextChecker.isNode) this.installSignalHandlers(); this.options.eventTarget.dispatchEvent(new Events.ComputationStarted()); this.logger?.info( `Task Executor has started using subnet: ${this.options.subnetTag}, network: ${this.paymentService.config.payment.network}, driver: ${this.paymentService.config.payment.driver}`, @@ -223,9 +246,10 @@ export class TaskExecutor { * Stop all executor services and shut down executor instance */ async end() { - if (runtimeContextChecker.isNode) this.removeCancelEvent(); + if (runtimeContextChecker.isNode) this.removeSignalHandlers(); if (!this.isRunning) return; this.isRunning = false; + clearTimeout(this.startupTimeoutId); if (!this.configOptions.storageProvider) await this.storageProvider?.close(); await this.networkService?.end(); await Promise.all([this.taskService.end(), this.agreementPoolService.end(), this.marketService.end()]); @@ -449,18 +473,24 @@ export class TaskExecutor { this.end().catch((e) => this.logger?.error(e)); } - private handleCancelEvent() { - terminatingSignals.forEach((event) => process.on(event, () => this.cancel(event))); + private installSignalHandlers() { + if (this.configOptions.skipProcessSignals) return; + terminatingSignals.forEach((event) => { + process.on(event, this.signalHandler); + }); } - private removeCancelEvent() { - terminatingSignals.forEach((event) => process.removeAllListeners(event)); + private removeSignalHandlers() { + if (this.configOptions.skipProcessSignals) return; + terminatingSignals.forEach((event) => { + process.removeListener(event, this.signalHandler); + }); } public async cancel(reason?: string) { try { if (this.isCanceled) return; - if (runtimeContextChecker.isNode) this.removeCancelEvent(); + if (runtimeContextChecker.isNode) this.removeSignalHandlers(); const message = `Executor has interrupted by the user. Reason: ${reason}.`; this.logger?.warn(`${message}. Stopping all tasks...`); this.isCanceled = true; @@ -480,4 +510,28 @@ export class TaskExecutor { if (costsSummary.length) this.logger?.table?.(costsSummary); this.logger?.info(`Total Cost: ${costs.total} Total Paid: ${costs.paid}`); } + + /** + * Sets a timeout for waiting for offers from the market. + * If at least one offer is not confirmed during the set timeout, + * a critical error will be reported and the entire process will be interrupted. + */ + private setStartupTimeout() { + this.startupTimeoutId = setTimeout(() => { + const proposalsCount = this.marketService.getProposalsCount(); + if (proposalsCount.confirmed === 0) { + const hint = + proposalsCount.initial === 0 && proposalsCount.confirmed === 0 + ? "Check your demand if it's not too restrictive or restart yagna." + : proposalsCount.initial === proposalsCount.rejected + ? "All off proposals got rejected." + : "Check your proposal filters if they are not too restrictive."; + this.handleCriticalError( + new Error( + `Could not start any work on Golem. Processed ${proposalsCount.initial} initial proposals from yagna, filters accepted ${proposalsCount.confirmed}. ${hint}`, + ), + ); + } + }, this.options.startupTimeout); + } } diff --git a/src/index.ts b/src/index.ts index 1eaa3ea4e..ea0ebfd89 100755 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ export { } from "./storage"; export { ActivityStateEnum, Result } from "./activity"; export { AgreementCandidate, AgreementSelectors } from "./agreement"; -export { ProposalFilters, ProposalFilter } from "./market"; +export { ProposalFilters, ProposalFilter, MarketHelpers } from "./market"; export { Package, PackageOptions } from "./package"; export { PaymentFilters } from "./payment"; export { Events, BaseEvent, EventType } from "./events"; diff --git a/src/market/helpers.test.ts b/src/market/helpers.test.ts new file mode 100644 index 000000000..bd061f6b6 --- /dev/null +++ b/src/market/helpers.test.ts @@ -0,0 +1,55 @@ +import { MockPropertyPolicy, imock, instance, when } from "@johanblumenberg/ts-mockito"; + +import { getHealthyProvidersWhiteList } from "./helpers"; + +const mockFetch = jest.spyOn(global, "fetch"); +const response = imock(); + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe("Market Helpers", () => { + describe("Getting public healthy providers whitelist", () => { + describe("Positive cases", () => { + test("Will return the list returned by the endpoint", async () => { + // Given + when(response.json()).thenResolve(["0xAAA", "0xBBB"]); + mockFetch.mockResolvedValue(instance(response)); + + // When + const data = await getHealthyProvidersWhiteList(); + + // Then + expect(data).toEqual(["0xAAA", "0xBBB"]); + }); + }); + + describe("Negative cases", () => { + test("It throws an error when the response from the API will not be a successful one (fetch -> response.ok)", async () => { + // Given + const mockResponse = imock(MockPropertyPolicy.StubAsProperty); + when(mockResponse.ok).thenReturn(false); + when(mockResponse.text()).thenResolve("{error:'test'}"); + mockFetch.mockResolvedValue(instance(mockResponse)); + + // When, Then + await expect(() => getHealthyProvidersWhiteList()).rejects.toThrow( + "Failed to download healthy provider whitelist due to an error: Error: Request to download healthy provider whitelist failed: {error:'test'}", + ); + }); + + test("It throws an error when executing of fetch will fail for any reason", async () => { + // Given + mockFetch.mockImplementation(() => { + throw new Error("Something went wrong really bad!"); + }); + + // When, Then + await expect(() => getHealthyProvidersWhiteList()).rejects.toThrow( + "Failed to download healthy provider whitelist due to an error: Error: Something went wrong really bad!", + ); + }); + }); + }); +}); diff --git a/src/market/helpers.ts b/src/market/helpers.ts new file mode 100644 index 000000000..6553fde9b --- /dev/null +++ b/src/market/helpers.ts @@ -0,0 +1,22 @@ +/** + * Helps to obtain a whitelist of providers which were health-tested. + * + * Important: This helper requires internet access to function properly. + * + * @return An array with Golem Node IDs of the whitelisted providers. + */ +export async function getHealthyProvidersWhiteList(): Promise { + try { + const response = await fetch("https://provider-health.golem.network/v1/provider-whitelist"); + + if (response.ok) { + return response.json(); + } else { + const body = await response.text(); + + throw new Error(`Request to download healthy provider whitelist failed: ${body}`); + } + } catch (err) { + throw new Error(`Failed to download healthy provider whitelist due to an error: ${err}`); + } +} diff --git a/src/market/index.ts b/src/market/index.ts index 806bf9987..5ea8eca87 100644 --- a/src/market/index.ts +++ b/src/market/index.ts @@ -4,3 +4,4 @@ export { Proposal, ProposalDetails } from "./proposal"; export { MarketDecoration } from "./builder"; export { DemandConfig } from "./config"; export * as ProposalFilters from "./strategy"; +export * as MarketHelpers from "./helpers"; diff --git a/src/market/service.ts b/src/market/service.ts index 438c3ad34..d62663235 100644 --- a/src/market/service.ts +++ b/src/market/service.ts @@ -1,11 +1,10 @@ -import { Logger, sleep } from "../utils"; +import { YagnaApi, Logger, sleep } from "../utils"; import { Package } from "../package"; import { Proposal } from "./proposal"; import { AgreementPoolService } from "../agreement"; import { Allocation } from "../payment"; import { Demand, DemandEvent, DemandEventType, DemandOptions } from "./demand"; import { MarketConfig } from "./config"; -import { YagnaApi } from "../utils/yagna/yagna"; export type ProposalFilter = (proposal: Proposal) => Promise | boolean; @@ -28,6 +27,11 @@ export class MarketService { private logger?: Logger; private taskPackage?: Package; private maxResubscribeRetries = 5; + private proposalsCount = { + initial: 0, + confirmed: 0, + rejected: 0, + }; constructor( private readonly agreementPoolService: AgreementPoolService, @@ -53,10 +57,18 @@ export class MarketService { this.logger?.debug("Market Service has been stopped"); } + getProposalsCount() { + return this.proposalsCount; + } private async createDemand(): Promise { if (!this.taskPackage || !this.allocation) throw new Error("The service has not been started correctly."); this.demand = await Demand.create(this.taskPackage, this.allocation, this.yagnaApi, this.options); this.demand.addEventListener(DemandEventType, this.demandEventListener.bind(this)); + this.proposalsCount = { + initial: 0, + confirmed: 0, + rejected: 0, + }; this.logger?.debug(`New demand has been created (${this.demand.id})`); return true; } @@ -72,7 +84,10 @@ export class MarketService { if (proposal.isInitial()) this.processInitialProposal(proposal); else if (proposal.isDraft()) this.processDraftProposal(proposal); else if (proposal.isExpired()) this.logger?.debug(`Proposal hes expired ${proposal.id}`); - else if (proposal.isRejected()) this.logger?.debug(`Proposal hes rejected ${proposal.id}`); + else if (proposal.isRejected()) { + this.proposalsCount.rejected++; + this.logger?.debug(`Proposal hes rejected ${proposal.id}`); + } } private async resubscribeDemand() { @@ -92,6 +107,7 @@ export class MarketService { private async processInitialProposal(proposal: Proposal) { if (!this.allocation) throw new Error("The service has not been started correctly."); this.logger?.debug(`New proposal has been received (${proposal.id})`); + this.proposalsCount.initial++; try { const { result: isProposalValid, reason } = await this.isProposalValid(proposal); if (isProposalValid) { @@ -101,6 +117,7 @@ export class MarketService { .catch((e) => this.logger?.debug(`Unable to respond proposal ${proposal.id}. ${e}`)); this.logger?.debug(`Proposal has been responded (${proposal.id})`); } else { + this.proposalsCount.rejected++; this.logger?.debug(`Proposal has been rejected (${proposal.id}). Reason: ${reason}`); } } catch (error) { @@ -122,6 +139,7 @@ export class MarketService { private async processDraftProposal(proposal: Proposal) { await this.agreementPoolService.addProposal(proposal); + this.proposalsCount.confirmed++; this.logger?.debug( `Proposal has been confirmed with provider ${proposal.issuerId} and added to agreement pool (${proposal.id})`, ); diff --git a/src/market/strategy.ts b/src/market/strategy.ts index d441f3fc3..821953fd2 100644 --- a/src/market/strategy.ts +++ b/src/market/strategy.ts @@ -42,7 +42,7 @@ export type PriceLimits = { */ export const limitPriceFilter = (priceLimits: PriceLimits) => async (proposal: Proposal) => { return ( - proposal.pricing.cpuSec < priceLimits.cpuPerSec && + proposal.pricing.cpuSec <= priceLimits.cpuPerSec && proposal.pricing.envSec <= priceLimits.envPerSec && proposal.pricing.start <= priceLimits.start ); diff --git a/src/payment/payments.ts b/src/payment/payments.ts index 785fb4bc5..e0ba001fb 100644 --- a/src/payment/payments.ts +++ b/src/payment/payments.ts @@ -61,6 +61,7 @@ export class Payments extends EventTarget { { timeout: 0 }, ); for (const event of invoiceEvents) { + if (!this.isRunning) return; if (event.eventType !== "InvoiceReceivedEvent") continue; const invoice = await Invoice.create(event["invoiceId"], this.yagnaApi, { ...this.options }).catch( (e) => @@ -95,6 +96,7 @@ export class Payments extends EventTarget { ) .catch(() => ({ data: [] })); for (const event of debitNotesEvents) { + if (!this.isRunning) return; if (event.eventType !== "DebitNoteReceivedEvent") continue; const debitNote = await DebitNote.create(event["debitNoteId"], this.yagnaApi, { ...this.options }).catch( (e) => diff --git a/src/payment/service.ts b/src/payment/service.ts index 331d64596..cca5f4b3d 100644 --- a/src/payment/service.ts +++ b/src/payment/service.ts @@ -76,7 +76,7 @@ export class PaymentService { clearTimeout(timeoutId); } this.isRunning = false; - this.payments?.unsubscribe().catch((error) => this.logger?.warn(error)); + await this.payments?.unsubscribe().catch((error) => this.logger?.warn(error)); this.payments?.removeEventListener(PaymentEventType, this.subscribePayments.bind(this)); await this.allocation?.release().catch((error) => this.logger?.warn(error)); this.logger?.info("Allocation has been released"); @@ -127,10 +127,13 @@ export class PaymentService { `Invoice has been rejected for provider ${agreement.provider.name}. Reason: ${reason.message}`, ); } - this.agreementsDebitNotes.delete(invoice.agreementId); - this.agreementsToPay.delete(invoice.agreementId); } catch (error) { this.logger?.error(`Invoice failed from provider ${invoice.providerId}. ${error}`); + } finally { + // Until we implement a re-acceptance mechanism for unsuccessful acceptances, + // we no longer have to wait for the invoice during an unsuccessful attempt. + this.agreementsDebitNotes.delete(invoice.agreementId); + this.agreementsToPay.delete(invoice.agreementId); } } diff --git a/src/storage/gftp.ts b/src/storage/gftp.ts index 099d62246..2281c4c7a 100644 --- a/src/storage/gftp.ts +++ b/src/storage/gftp.ts @@ -1,13 +1,11 @@ -import { StorageProvider, StorageProviderDataCallback } from "./provider"; -import { Logger, runtimeContextChecker } from "../utils"; +import { StorageProvider } from "./provider"; +import { Logger, runtimeContextChecker, sleep } from "../utils"; import path from "path"; import fs from "fs"; -import { chomp, chunksToLinesAsync, streamEnd, streamWrite } from "@rauschma/stringio"; import cp from "child_process"; export class GftpStorageProvider implements StorageProvider { - private gftpServerProcess; - private reader; + private gftpServerProcess?: cp.ChildProcess; /** * All published URLs to be release on close(). @@ -16,6 +14,12 @@ export class GftpStorageProvider implements StorageProvider { private publishedUrls = new Set(); private isInitialized = false; + private reader?: AsyncIterableIterator; + /** + * lock against parallel writing to stdin in gftp process + * @private + */ + private lock = false; constructor(private logger?: Logger) { if (runtimeContextChecker.isBrowser) { @@ -52,9 +56,7 @@ export class GftpStorageProvider implements StorageProvider { this.gftpServerProcess?.stdout?.setEncoding("utf-8"); this.gftpServerProcess?.stderr?.setEncoding("utf-8"); - - this.gftpServerProcess.stdout.on("data", (data) => this.logger?.debug(`GFTP server stdout: ${data}`)); - this.gftpServerProcess.stderr.on("data", (data) => this.logger?.error(`GFTP server stderr: ${data}`)); + this.reader = this.gftpServerProcess?.stdout?.iterator(); }); } @@ -70,17 +72,12 @@ export class GftpStorageProvider implements StorageProvider { return file_name; } - private getGftpServerProcess() { - return this.gftpServerProcess; - } - async receiveFile(path: string): Promise { const { url } = await this.jsonrpc("receive", { output_file: path }); return url; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - receiveData(callback: StorageProviderDataCallback): Promise { + receiveData(): Promise { throw new Error("receiveData is not implemented in GftpStorageProvider"); } @@ -102,8 +99,7 @@ export class GftpStorageProvider implements StorageProvider { return url; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - release(urls: string[]): Promise { + release(): Promise { // NOTE: Due to GFTP's handling of file Ids (hashes), all files with same content will share IDs, so releasing // one might break transfer of another one. Therefore, we release all files on close(). return Promise.resolve(undefined); @@ -119,21 +115,21 @@ export class GftpStorageProvider implements StorageProvider { async close() { await this.releaseAll(); - const stream = this.getGftpServerProcess(); - if (stream) await streamEnd(this.getGftpServerProcess().stdin); + this.gftpServerProcess?.kill(); } private async jsonrpc(method: string, params: object = {}) { if (!this.isInitiated()) await this.init(); - if (!this.reader) this.reader = this.readStream(this.getGftpServerProcess().stdout); + while (this.lock) await sleep(100, true); + this.lock = true; const paramsStr = JSON.stringify(params); const query = `{"jsonrpc": "2.0", "id": "1", "method": "${method}", "params": ${paramsStr}}\n`; let valueStr = ""; - await streamWrite(this.getGftpServerProcess().stdin, query); try { - const { value } = await this.reader.next(); + this.gftpServerProcess?.stdin?.write(query); + const value = (await this.reader?.next())?.value; if (!value) throw "Unable to get GFTP command result"; - const { result } = JSON.parse(value as string); + const { result } = JSON.parse(value); valueStr = value; if (result === undefined) throw value; return result; @@ -141,12 +137,8 @@ export class GftpStorageProvider implements StorageProvider { throw Error( `Error while obtaining response to JSONRPC. query: ${query} value: ${valueStr} error: ${JSON.stringify(error)}`, ); - } - } - - async *readStream(readable) { - for await (const line of chunksToLinesAsync(readable)) { - yield chomp(line); + } finally { + this.lock = false; } } diff --git a/tests/docker/Provider.Dockerfile b/tests/docker/Provider.Dockerfile new file mode 100644 index 000000000..81ee1c94d --- /dev/null +++ b/tests/docker/Provider.Dockerfile @@ -0,0 +1,41 @@ +ARG UBUNTU_VERSION=22.04 +ARG YA_CORE_VERSION=0.12.3 +ARG YA_WASI_VERSION=0.2.2 +ARG YA_VM_VERSION=0.3.0 + +FROM ubuntu:${UBUNTU_VERSION} +ARG YA_CORE_VERSION +ARG YA_WASI_VERSION +ARG YA_VM_VERSION +ARG YA_DIR_INSTALLER=/ya-installer +ARG YA_DIR_BIN=/usr/bin +ARG YA_DIR_PLUGINS=/lib/yagna/plugins +COPY /data-node/ya-provider/ /root/.local/share/ya-provider/ +RUN apt-get update -q \ + && apt-get install -q -y --no-install-recommends \ + wget \ + apt-transport-https \ + ca-certificates \ + xz-utils \ + curl \ + python3 \ + && apt-get remove --purge -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p ${YA_DIR_PLUGINS} \ + && mkdir ${YA_DIR_INSTALLER} \ + && cd ${YA_DIR_INSTALLER} \ + && wget -q "https://github.com/golemfactory/yagna/releases/download/v${YA_CORE_VERSION}/golem-provider-linux-v${YA_CORE_VERSION}.tar.gz" \ + && wget -q "https://github.com/golemfactory/ya-runtime-wasi/releases/download/v${YA_WASI_VERSION}/ya-runtime-wasi-linux-v${YA_WASI_VERSION}.tar.gz" \ + && wget -q "https://github.com/golemfactory/ya-runtime-vm/releases/download/v${YA_VM_VERSION}/ya-runtime-vm-linux-v${YA_VM_VERSION}.tar.gz" \ + && tar -zxvf golem-provider-linux-v${YA_CORE_VERSION}.tar.gz \ + && tar -zxvf ya-runtime-wasi-linux-v${YA_WASI_VERSION}.tar.gz \ + && tar -zxvf ya-runtime-vm-linux-v${YA_VM_VERSION}.tar.gz \ + && find golem-provider-linux-v${YA_CORE_VERSION} -executable -type f -exec cp {} ${YA_DIR_BIN} \; \ + && cp -R golem-provider-linux-v${YA_CORE_VERSION}/plugins/* ${YA_DIR_PLUGINS} \ + && cp -R ya-runtime-wasi-linux-v${YA_WASI_VERSION}/* ${YA_DIR_PLUGINS} \ + && cp -R ya-runtime-vm-linux-v${YA_VM_VERSION}/* ${YA_DIR_PLUGINS} \ + && rm -Rf ${YA_DIR_INSTALLER} +COPY ./configureProvider.py /configureProvider.py + +CMD ["bash", "-c", "python3 /configureProvider.py && golemsp run --payment-network testnet"] diff --git a/tests/docker/Requestor.Dockerfile b/tests/docker/Requestor.Dockerfile new file mode 100644 index 000000000..b64167fdd --- /dev/null +++ b/tests/docker/Requestor.Dockerfile @@ -0,0 +1,44 @@ +ARG UBUNTU_VERSION=22.04 +ARG YA_CORE_VERSION=0.13.0-rc10 + +FROM node:18 +ARG YA_CORE_VERSION +ARG YA_DIR_INSTALLER=/ya-installer +ARG YA_DIR_BIN=/usr/bin +RUN apt-get update -q \ + && apt-get install -q -y --no-install-recommends \ + wget \ + apt-transport-https \ + ca-certificates \ + xz-utils \ + curl \ + sshpass \ + python3 \ + libgtk2.0-0 \ + libgtk-3-0 \ + libgbm-dev \ + libnotify-dev \ + libgconf-2-4 \ + libnss3 \ + libxss1 \ + libasound2 \ + libxtst6 \ + xauth \ + xvfb \ + chromium \ + && apt-get remove --purge -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir ${YA_DIR_INSTALLER} \ + && cd ${YA_DIR_INSTALLER} \ + && wget -q "https://github.com/golemfactory/yagna/releases/download/pre-rel-v${YA_CORE_VERSION}/golem-requestor-linux-pre-rel-v${YA_CORE_VERSION}.tar.gz" \ + && tar -zxvf golem-requestor-linux-pre-rel-v${YA_CORE_VERSION}.tar.gz \ + && find golem-requestor-linux-pre-rel-v${YA_CORE_VERSION} -executable -type f -exec cp {} ${YA_DIR_BIN} \; \ + && rm -Rf ${YA_DIR_INSTALLER} \ + && wget -O ${YA_DIR_BIN}/websocat "https://github.com/vi/websocat/releases/download/v1.12.0/websocat_max.x86_64-unknown-linux-musl" \ + && chmod +x ${YA_DIR_BIN}/websocat + + +COPY ./startRequestor.sh /startRequestor.sh + +CMD ["bash", "-c", "/startRequestor.sh"] diff --git a/tests/docker/configureProvider.py b/tests/docker/configureProvider.py new file mode 100644 index 000000000..ccd452297 --- /dev/null +++ b/tests/docker/configureProvider.py @@ -0,0 +1,20 @@ +import os +import json + + +def update_globals(file_path, node_name): + try: + with open(file_path, 'r+') as f: + data = json.load(f) + data['node_name'] = node_name + f.seek(0) + print(data) + json.dumps(data, f) + f.truncate() + print(f"Provider node name configured to {node_name}") + except Exception as e: + print(f"Error occurred: {str(e)}") + + +update_globals(os.path.expanduser( + '/root/.local/share/ya-provider/globals.json'), os.environ.get('NODE_NAME')) diff --git a/tests/docker/data-node/ya-provider/globals.json b/tests/docker/data-node/ya-provider/globals.json new file mode 100644 index 000000000..83261dce1 --- /dev/null +++ b/tests/docker/data-node/ya-provider/globals.json @@ -0,0 +1,5 @@ +{ + "node_name": "node-name", + "subnet": "public", + "account": "0x797cE3Aa8dc255D13E48F31A6B23fe18b5924940" +} diff --git a/tests/docker/data-node/ya-provider/hardware.json b/tests/docker/data-node/ya-provider/hardware.json new file mode 100644 index 000000000..51f668a89 --- /dev/null +++ b/tests/docker/data-node/ya-provider/hardware.json @@ -0,0 +1,10 @@ +{ + "active": "default", + "profiles": { + "default": { + "cpu_threads": 1, + "mem_gib": 2.0, + "storage_gib": 10.0 + } + } +} diff --git a/tests/docker/data-node/ya-provider/presets.json b/tests/docker/data-node/ya-provider/presets.json new file mode 100644 index 000000000..367c0199f --- /dev/null +++ b/tests/docker/data-node/ya-provider/presets.json @@ -0,0 +1,35 @@ +{ + "active": ["wasmtime", "vm"], + "presets": [ + { + "name": "default", + "exeunit-name": "wasmtime", + "pricing-model": "linear", + "usage-coeffs": { + "cpu": 0.100000001, + "duration": 0.0, + "initial": 0.0 + } + }, + { + "name": "vm", + "exeunit-name": "vm", + "pricing-model": "linear", + "usage-coeffs": { + "cpu": 0.100000001, + "duration": 0.0, + "initial": 0.0 + } + }, + { + "name": "wasmtime", + "exeunit-name": "wasmtime", + "pricing-model": "linear", + "usage-coeffs": { + "cpu": 0.100000001, + "duration": 0.0, + "initial": 0.0 + } + } + ] +} diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml new file mode 100644 index 000000000..aa38e11f1 --- /dev/null +++ b/tests/docker/docker-compose.yml @@ -0,0 +1,69 @@ +version: "3.5" +services: + provider-1: + build: + context: . + dockerfile: Provider.Dockerfile + image: provider:latest + restart: always + deploy: + replicas: 6 + volumes: + - /etc/localtime:/etc/localtime:ro + - /root/.local/share/yagna/ + devices: + - /dev/kvm:/dev/kvm + healthcheck: + test: ["CMD-SHELL", "curl -s -o /dev/null -w '%{http_code}' http://localhost:7465 | grep -q 401"] + interval: 10s + timeout: 5s + retries: 1 + start_period: 40s + environment: + - NODE_NAME=provider-1 + - SUBNET=${YAGNA_SUBNET:-golemjstest} + provider-2: + build: + context: . + dockerfile: Provider.Dockerfile + image: provider:latest + restart: always + deploy: + replicas: 6 + volumes: + - /etc/localtime:/etc/localtime:ro + - /root/.local/share/yagna/ + devices: + - /dev/kvm:/dev/kvm + healthcheck: + test: ["CMD-SHELL", "curl -s -o /dev/null -w '%{http_code}' http://localhost:7465 | grep -q 401"] + interval: 10s + timeout: 5s + retries: 1 + start_period: 40s + environment: + - NODE_NAME=provider-2 + - SUBNET=${YAGNA_SUBNET:-golemjstest} + requestor: + build: + context: . + dockerfile: Requestor.Dockerfile + image: requestor:latest + restart: always + volumes: + - /etc/localtime:/etc/localtime:ro + - /root/.local/share/yagna/ + - ../../:/golem-js + environment: + - YAGNA_AUTOCONF_APPKEY=try_golem + - YAGNA_API_URL=http://0.0.0.0:7465 + - GSB_URL=tcp://0.0.0.0:7464 + - YAGNA_SUBNET=${YAGNA_SUBNET:-golemjstest} + - YAGNA_APPKEY=try_golem + + healthcheck: + test: ["CMD-SHELL", "curl -s -o /dev/null -w '%{http_code}' http://localhost:7465 | grep -q 401"] + interval: 10s + timeout: 5s + retries: 1 + start_period: 40s diff --git a/tests/docker/fundRequestor.sh b/tests/docker/fundRequestor.sh new file mode 100755 index 000000000..e1c24e1fb --- /dev/null +++ b/tests/docker/fundRequestor.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for i in {1..3}; do + yagna payment fund && exit 0 +done + +echo "yagna payment fund failed" >&2 +exit 1 \ No newline at end of file diff --git a/tests/docker/startRequestor.sh b/tests/docker/startRequestor.sh new file mode 100755 index 000000000..d77cf281f --- /dev/null +++ b/tests/docker/startRequestor.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +get_funds_from_faucet() { + echo "Sending request to the faucet" + yagna payment fund +} +echo "Starting Yagna" +yagna service run --api-allow-origin="*" diff --git a/tests/e2e/_setup.ts b/tests/e2e/_setup.ts deleted file mode 100644 index de9ff4749..000000000 --- a/tests/e2e/_setup.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Goth } from "../goth/goth"; -import { resolve } from "path"; - -const timeoutPromise = (seconds: number) => - new Promise((_resolve, reject) => { - setTimeout( - () => reject(new Error(`The timeout was reached and the racing promise has rejected after ${seconds} seconds`)), - seconds * 1000, - ); - }); - -export default async function setUpGoth() { - const gothConfig = resolve("../goth/assets/goth-config.yml"); - globalThis.__GOTH = new Goth(gothConfig); - - // Start Goth, but don't wait for an eternity - return await Promise.race([globalThis.__GOTH.start(), timeoutPromise(180)]); -} diff --git a/tests/e2e/_teardown.ts b/tests/e2e/_teardown.ts deleted file mode 100644 index 994401c50..000000000 --- a/tests/e2e/_teardown.ts +++ /dev/null @@ -1,3 +0,0 @@ -export default async function tearDownGoth() { - await globalThis.__GOTH.end(); -} diff --git a/tests/e2e/blender.spec.ts b/tests/e2e/blender.spec.ts index 1741c6c8e..2f29a54db 100644 --- a/tests/e2e/blender.spec.ts +++ b/tests/e2e/blender.spec.ts @@ -23,40 +23,44 @@ const blenderParams = (frame) => ({ }); describe("Blender rendering", function () { - it("should render images by blender", async () => { - const executor = await TaskExecutor.create({ - package: "golem/blender:latest", - logger, - }); - - executor.beforeEach(async (ctx) => { - const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/cubes.blend"); - await ctx.uploadFile(sourcePath, "/golem/resource/scene.blend"); - }); - - const data = [0, 10, 20, 30, 40, 50]; - - const results = executor.map(data, async (ctx, frame) => { - const result = await ctx - .beginBatch() - .uploadJson(blenderParams(frame), "/golem/work/params.json") - .run("/golem/entrypoints/run-blender.sh") - .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `output_${frame}.png`) - .end() - .catch((error) => ctx.rejectResult(error.toString())); - return result ? `output_${frame}.png` : ""; - }); - - const expectedResults = data.map((d) => `output_${d}.png`); - - for await (const result of results) { - expect(expectedResults).toContain(result); - } - - for (const file of expectedResults) { - expect(fs.existsSync(`${process.env.GOTH_GFTP_VOLUME || ""}${file}`)).toEqual(true); - } - - await executor.end(); - }); + it( + "should render images by blender", + async () => { + const executor = await TaskExecutor.create({ + package: "golem/blender:latest", + logger, + }); + + executor.beforeEach(async (ctx) => { + const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/cubes.blend"); + await ctx.uploadFile(sourcePath, "/golem/resource/scene.blend"); + }); + + const data = [0, 10, 20, 30, 40, 50]; + + const results = executor.map(data, async (ctx, frame) => { + const result = await ctx + .beginBatch() + .uploadJson(blenderParams(frame), "/golem/work/params.json") + .run("/golem/entrypoints/run-blender.sh") + .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `output_${frame}.png`) + .end() + .catch((error) => console.error(error.toString())); + return result ? `output_${frame}.png` : ""; + }); + + const expectedResults = data.map((d) => `output_${d}.png`); + + for await (const result of results) { + expect(expectedResults).toContain(result); + } + + for (const file of expectedResults) { + expect(fs.existsSync(`${process.env.GOTH_GFTP_VOLUME || ""}${file}`)).toEqual(true); + } + + await executor.end(); + }, + 1000 * 240, + ); }); diff --git a/tests/e2e/express.spec.ts b/tests/e2e/express.spec.ts index f8fa13a28..d9ec6259e 100644 --- a/tests/e2e/express.spec.ts +++ b/tests/e2e/express.spec.ts @@ -29,7 +29,7 @@ describe("Express", function () { image: "golem/blender:latest", demand: { minMemGib: 1, - minStorageGib: 2, + minStorageGib: 1, minCpuThreads: 1, minCpuCores: 1, }, @@ -39,7 +39,7 @@ describe("Express", function () { "/golem/resource/scene.blend", ); }, - enableLogging: false, + enableLogging: true, }); await network.init(); }); @@ -126,40 +126,44 @@ describe("Express", function () { return app; }; - it("starts the express server", async function () { - const app = getServer(); - app.get("/network-status", (req, res) => { - res.json({ isInitialized: network.isInitialized() }); - }); + it( + "starts the express server", + async function () { + const app = getServer(); + app.get("/network-status", (req, res) => { + res.json({ isInitialized: network.isInitialized() }); + }); - const response = await supertest(app).post("/render-scene").send({ frame: 0 }); - expect(response.status).toEqual(200); - expect(response.body.jobId).toBeDefined(); - - const jobId = response.body.jobId; - let statusResponse = await supertest(app).get(`/job/${jobId}/status`); - expect(statusResponse.status).toEqual(200); - expect(statusResponse.body.status).toEqual(JobState.New); - - await new Promise((resolve) => { - const interval = setInterval(async () => { - const statusResponse = await supertest(app).get(`/job/${jobId}/status`); - if (statusResponse.body.status === JobState.Done) { - clearInterval(interval); - resolve(undefined); - } - }, 500); - }); + const response = await supertest(app).post("/render-scene").send({ frame: 0 }); + expect(response.status).toEqual(200); + expect(response.body.jobId).toBeDefined(); + + const jobId = response.body.jobId; + let statusResponse = await supertest(app).get(`/job/${jobId}/status`); + expect(statusResponse.status).toEqual(200); + expect(statusResponse.body.status).toEqual(JobState.New); + + await new Promise((resolve) => { + const interval = setInterval(async () => { + const statusResponse = await supertest(app).get(`/job/${jobId}/status`); + if (statusResponse.body.status === JobState.Done) { + clearInterval(interval); + resolve(undefined); + } + }, 500); + }); - statusResponse = await supertest(app).get(`/job/${jobId}/status`); - expect(statusResponse.status).toEqual(200); - expect(statusResponse.body.status).toEqual(JobState.Done); + statusResponse = await supertest(app).get(`/job/${jobId}/status`); + expect(statusResponse.status).toEqual(200); + expect(statusResponse.body.status).toEqual(JobState.Done); - const resultResponse = await supertest(app).get(`/job/${jobId}/result`); - expect(resultResponse.status).toEqual(200); - expect(resultResponse.body).toEqual( - "Job completed successfully! See your result at http://localhost:3001/results/EXPRESS_SPEC_output_0.png", - ); - expect(fs.existsSync(`${process.env.GOTH_GFTP_VOLUME || ""}EXPRESS_SPEC_output_0.png`)).toEqual(true); - }); + const resultResponse = await supertest(app).get(`/job/${jobId}/result`); + expect(resultResponse.status).toEqual(200); + expect(resultResponse.body).toEqual( + "Job completed successfully! See your result at http://localhost:3001/results/EXPRESS_SPEC_output_0.png", + ); + expect(fs.existsSync(`EXPRESS_SPEC_output_0.png`)).toEqual(true); + }, + 1000 * 240, + ); }); diff --git a/tests/e2e/gftp.spec.ts b/tests/e2e/gftp.spec.ts index 3e82e0161..d57fafd9c 100644 --- a/tests/e2e/gftp.spec.ts +++ b/tests/e2e/gftp.spec.ts @@ -2,43 +2,46 @@ import { TaskExecutor } from "../../src"; import { LoggerMock } from "../mock"; import fs from "fs"; -const logger = new LoggerMock(); +const logger = new LoggerMock(false); describe("GFTP transfers", function () { - it("should upload and download big files simultaneously", async () => { - const executor = await TaskExecutor.create({ - package: "golem/alpine:latest", - logger, - }); - - executor.beforeEach(async (ctx) => { - const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/eiffel.blend"); - await ctx.uploadFile(sourcePath, "/golem/work/eiffel.blend"); - }); - - const data = [0, 1, 2, 3, 4, 5]; - - const results = executor.map(data, async (ctx, frame) => { - const result = await ctx - .beginBatch() - .run("ls -Alh /golem/work/eiffel.blend") - .downloadFile(`/golem/work/eiffel.blend`, `copy_${frame}.blend`) - .end() - .catch((error) => ctx.rejectResult(error.toString())); - return result ? `copy_${frame}.blend` : ""; - }); - - const expectedResults = data.map((d) => `copy_${d}.blend`); - - for await (const result of results) { - expect(expectedResults).toContain(result); - } - - for (const file of expectedResults) { - const path = `${process.env.GOTH_GFTP_VOLUME || ""}${file}`; - expect(fs.existsSync(path)).toEqual(true); - } - - await executor.end(); - }); + it( + "should upload and download big files simultaneously", + async () => { + const executor = await TaskExecutor.create({ + package: "golem/alpine:latest", + logger, + }); + + executor.beforeEach(async (ctx) => { + const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/eiffel.blend"); + await ctx.uploadFile(sourcePath, "/golem/work/eiffel.blend"); + }); + + const data = [0, 1, 2, 3, 4, 5]; + + const results = executor.map(data, async (ctx, frame) => { + const result = await ctx + .beginBatch() + .run("ls -Alh /golem/work/eiffel.blend") + .downloadFile(`/golem/work/eiffel.blend`, `copy_${frame}.blend`) + .end() + .catch((error) => console.error(error.toString())); + return result ? `copy_${frame}.blend` : ""; + }); + + const expectedResults = data.map((d) => `copy_${d}.blend`); + + for await (const result of results) { + expect(expectedResults).toContain(result); + } + + for (const file of expectedResults) { + expect(fs.existsSync(file)).toEqual(true); + } + + await executor.end(); + }, + 1000 * 240, + ); }); diff --git a/tests/e2e/jest.config.json b/tests/e2e/jest.config.json index ae00deaa5..8eb5939a8 100644 --- a/tests/e2e/jest.config.json +++ b/tests/e2e/jest.config.json @@ -1,8 +1,6 @@ { "preset": "ts-jest", "testEnvironment": "node", - "globalSetup": "/_setup.ts", - "globalTeardown": "/_teardown.ts", "setupFilesAfterEnv": ["/_setupLogging.ts"], "testTimeout": 180000 } diff --git a/tests/examples/examples.json b/tests/examples/examples.json index 688cc6718..c2176632e 100644 --- a/tests/examples/examples.json +++ b/tests/examples/examples.json @@ -1,4 +1,5 @@ [ + { "cmd": "node", "path": "examples/docs-examples/quickstarts/retrievable-task/task.mjs", "noGoth": true }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/batch-end.mjs", "noGoth": true }, { "cmd": "node", @@ -14,6 +15,7 @@ { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/composing-tasks/single-command.cjs" }, { "cmd": "ts-node", "path": "examples/docs-examples/examples/composing-tasks/single-command.ts" }, + { "cmd": "ts-node", "path": "examples/docs-examples/examples/composing-tasks/alert-code.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/before-each.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/executing-tasks/foreach.mjs" }, @@ -29,8 +31,6 @@ { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/sending-data/uploading-json.mjs" }, - { "cmd": "node", "path": "examples/docs-examples/examples/switching-to-mainnet/run-on-polygon.mjs", "noGoth": true }, - { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/download-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-file.mjs" }, { "cmd": "node", "path": "examples/docs-examples/examples/transferring-data/upload-json.mjs" }, @@ -62,6 +62,6 @@ "cmd": "node", "path": "examples/docs-examples/tutorials/running-parallel-tasks/index.mjs", "args": ["--mask", "?a?a", "--hash", "$P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0"], - "timeout": 500 + "timeout": 1000 } ] diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts index e61e534b2..3e30cfe3e 100644 --- a/tests/examples/examples.test.ts +++ b/tests/examples/examples.test.ts @@ -1,15 +1,8 @@ import { spawn } from "child_process"; import { dirname, basename, resolve } from "path"; -import { Goth } from "../goth/goth"; import chalk from "chalk"; import testExamples from "./examples.json"; -const noGoth = process.argv[2] === "--no-goth"; -const gothConfig = resolve("../goth/assets/goth-config.yml"); -const gothStartingTimeout = 180; -const goth = new Goth(gothConfig); - -const examples = !noGoth ? testExamples.filter((e) => !e?.noGoth) : testExamples; const criticalLogsRegExp = [/Task *. timeot/, /Task *. has been rejected/, /ERROR: TypeError/, /ERROR: Error/gim]; type Example = { @@ -17,11 +10,12 @@ type Example = { path: string; args?: string[]; timeout?: number; - noGoth?: boolean; skip?: boolean; }; -async function test(cmd: string, path: string, args: string[] = [], timeout = 180) { +const exitOnError = process.argv.includes("--exitOnError"); + +async function test(cmd: string, path: string, args: string[] = [], timeout = 360) { const file = basename(path); const cwd = dirname(path); const spawnedExample = spawn(cmd, [file, ...args], { cwd }); @@ -62,16 +56,6 @@ async function test(cmd: string, path: string, args: string[] = [], timeout = 18 async function testAll(examples: Example[]) { const failedTests = new Set(); const skippedTests = new Set(); - if (!noGoth) - await Promise.race([ - goth.start(), - new Promise((res, rej) => - setTimeout( - () => rej(new Error(`The Goth starting timeout was reached after ${gothStartingTimeout} seconds`)), - gothStartingTimeout * 1000, - ), - ), - ]); for (const example of examples) { try { console.log(chalk.yellow(`\n---- Starting test: "${example.path}" ----\n`)); @@ -84,10 +68,13 @@ async function testAll(examples: Example[]) { } } catch (error) { console.log(chalk.bgRed.white(" FAIL "), chalk.red(error)); + if (exitOnError) { + console.log(chalk.bold.red(`\nExiting due to error in: "${example.path}"\n`)); + process.exit(1); + } failedTests.add(example.path); } } - if (!noGoth) await goth.end().catch((error) => console.error(error)); console.log( chalk.bold.yellow("\n\nTESTS RESULTS: "), chalk.bgGreen.black(` ${examples.length - failedTests.size - skippedTests.size} passed `), @@ -102,4 +89,4 @@ async function testAll(examples: Example[]) { process.exit(failedTests.size > 0 ? 1 : 0); } -testAll(examples).then(); +testAll(testExamples).then(); diff --git a/tests/goth/goth.ts b/tests/goth/goth.ts deleted file mode 100644 index 8fea42ca4..000000000 --- a/tests/goth/goth.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { ChildProcess, spawn } from "child_process"; - -type EnvironmentSettings = { apiKey: string; basePath: string; subnetTag: string; gsbUrl: string; path: string }; - -export class Goth { - private gothProcess?: ChildProcess; - - constructor(private readonly gothConfig) {} - - async start(): Promise { - return new Promise((resolve, reject) => { - const startTime = Date.now(); - console.log("\x1b[33mStarting goth process..."); - console.log("\x1b[33mRun command:\x1b[0m \x1b[36m", `python -m goth start ${this.gothConfig}`); - this.gothProcess = spawn("python", ["-m", "goth", "start", this.gothConfig], { - env: { ...process.env, PYTHONUNBUFFERED: "1" }, - }); - this.gothProcess.on("spawn", () => console.log("Goth spawned successfully")); - this.gothProcess?.stdout?.setEncoding("utf-8"); - this.gothProcess?.stderr?.setEncoding("utf-8"); - - this.gothProcess?.stdout?.on("data", (data) => { - const regexp = - /YAGNA_APPKEY=(\w+) YAGNA_API_URL=(http:\/\/127\.0{0,3}\.0{0,3}.0{0,2}1:\d+) GSB_URL=(tcp:\/\/\d+\.\d+\.\d+\.\d+:\d+) PATH=(.*) YAGNA_SUBNET=(\w+)/g; - const results = Array.from(data?.toString()?.matchAll(regexp) || [])?.pop(); - const apiKey = results?.[1]; - const basePath = results?.[2]; - const gsbUrl = results?.[3]; - const path = results?.[4]?.split(":")?.shift(); - const subnetTag = results?.[5]; - if (apiKey) { - process.env["YAGNA_APPKEY"] = apiKey; - process.env["YAGNA_API_URL"] = basePath; - process.env["GSB_URL"] = gsbUrl; - process.env["PATH"] = `${path}:${process.env["PATH"]}`; - process.env["YAGNA_SUBNET"] = subnetTag; - - const settings = { apiKey, basePath, subnetTag, gsbUrl, path }; - - console.log( - `\x1b[33mGoth has been successfully started in ${((Date.now() - startTime) / 1000).toFixed( - 0, - )}s. Resulting settings:`, - settings, - ); - - resolve(settings); - } - }); - this.gothProcess?.stderr?.on("data", (data) => { - if (data.toString().match(/error/)) reject(data); - const regexp = /\[requestor] Gftp volume ([a-zA-Z0-9/_]*)/g; - const results = Array.from(data?.toString()?.matchAll(regexp) || [])?.pop(); - const gftpVolume = results?.[1]; - if (gftpVolume) process.env["GOTH_GFTP_VOLUME"] = gftpVolume + "/out/"; - console.log("\x1b[33m[goth]\x1b[0m " + data.replace(/[\n\t\r]/g, "")); - }); - this.gothProcess.on("error", (error) => reject("Failed to spawn Goth" + error.toString())); - this.gothProcess.on("close", (code) => console.info(`Goth process exit with code ${code}`)); - this.gothProcess.on("exit", (code) => console.info(`Goth process exit with code ${code}`)); - }); - } - - async end() { - this.gothProcess?.kill("SIGINT"); - return new Promise((resolve) => { - this.gothProcess?.on("close", () => { - this.gothProcess?.stdout?.removeAllListeners(); - this.gothProcess?.stderr?.removeAllListeners(); - this.gothProcess?.removeAllListeners(); - console.log(`\x1b[33mGoth has been terminated`); - resolve(); - }); - }); - } -}