Add nightly workflows for goth tests (#179) (#182)

.github/workflows/goth-nightly.yml

@@ -0,0 +1,93 @@
+name: Goth nightly
+
+on:
+  schedule:
+    # run this workflow every day at 2:00 AM UTC
+    - cron: '0 2 * * *'
+
+jobs:
+  goth-tests:
+    name: Run integration tests
+    runs-on: goth
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: 'b0.3'
+
+      - name: Configure node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.18.3'
+
+      - name: Build yajsapi
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y build-essential
+          npm install
+          npm run-script build
+          cd examples
+          npm install
+          sed -i 's,require("yajsapi");,require("'$([ -f ../dist/index.js ] && echo "../../dist/index.js" || echo "../../dist/yajsapi/index.js")'");,' blender/blender.js
+
+      - name: Configure python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8.0'
+
+      - name: Configure poetry
+        uses: Gr1N/setup-poetry@v4
+        with:
+          poetry-version: 1.1.6
+          working-directory: 'tests/goth'
+
+      - name: Install dependencies
+        run: |
+          cd tests/goth/
+          poetry env use python3.8
+          poetry install
+
+      - name: Disconnect Docker containers from default network
+        continue-on-error: true
+        run: |
+          docker network inspect docker_default
+          sudo apt-get install -y jq
+          docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default
+
+      - name: Remove Docker containers
+        continue-on-error: true
+        run: docker rm -f $(docker ps -a -q)
+
+      - name: Log in to GitHub Docker repository
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
+
+      - name: Run test suite
+        env:
+          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cd tests/goth
+          poetry run poe goth-assets
+          poetry run poe goth-tests
+
+      - name: Upload test logs
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+           name: goth-logs
+           path: /tmp/goth-tests
+
+      # Only relevant for self-hosted runners
+      - name: Remove test logs
+        if: always()
+        run: rm -rf /tmp/goth-tests
+
+      # Only relevant for self-hosted runners
+      - name: Remove poetry virtual env
+        if: always()
+        # Python version below should agree with the version set up by this job.
+        # In future we'll be able to use the `--all` flag here to remove envs for
+        # all Python versions (https://github.com/python-poetry/poetry/issues/3208).
+        run: |
+          cd tests/goth/
+          poetry env remove python3.8 

.github/workflows/integration-tests.yml

@@ -9,14 +9,17 @@ on:
     branches:
       - master
       - b0.*
+  schedule:
+    # run this workflow every day at 2:30 AM UTC
+    - cron: '30 2 * * *'
 
 jobs:
 
   goth-tests:
     name: Run integration tests
     runs-on: goth
-    steps:
 
+    steps:
       - name: Checkout
         uses: actions/checkout@v2
 
@@ -43,24 +46,36 @@ jobs:
       - name: Configure poetry
         uses: Gr1N/setup-poetry@v4
         with:
-          poetry-version: 1.1.4
+          poetry-version: 1.1.6
+          working-directory: 'tests/goth'
 
       - name: Install dependencies
         run: |
           cd tests/goth/
           poetry env use python3.8
           poetry install
 
+      - name: Disconnect Docker containers from default network
+        continue-on-error: true
+        run: |
+          docker network inspect docker_default
+          sudo apt-get install -y jq
+          docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default
+
+      - name: Remove Docker containers
+        continue-on-error: true
+        run: docker rm -f $(docker ps -a -q)
+
       - name: Log in to GitHub Docker repository
         run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
 
       - name: Run test suite
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          cd tests/goth/
-          poetry run python -m goth create-assets assets
-          poetry run pytest -svx .
+          cd tests/goth
+          poetry run poe goth-assets
+          poetry run poe goth-tests
 
       - name: Upload test logs
         uses: actions/upload-artifact@v2
@@ -82,4 +97,4 @@ jobs:
         # all Python versions (https://github.com/python-poetry/poetry/issues/3208).
         run: |
           cd tests/goth/
-          poetry env remove python3.8 
+          poetry env remove python3.8 

tests/goth/pyproject.toml

@@ -20,6 +20,17 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.8.0"
-pytest = "^5.4.3"
-pytest-asyncio = "^0.12.0"
-goth = "^0.2.0"
+pytest = "^6.2"
+pytest-asyncio = "^0.14"
+goth = "^0.2"
+
+[tool.poetry.dev-dependencies]
+black = "^20.8b1"
+mypy = "^0.782"
+poethepoet = "^0.8.0"
+
+[tool.poe.tasks]
+codestyle = "black --check --diff ."
+goth-assets = "python -m goth create-assets assets"
+goth-tests = "pytest -svx ."
+typecheck = "mypy ."

tests/goth/test_run_blender.py

@@ -62,16 +62,22 @@ async def assert_all_invoices_accepted(output_lines: EventStream[str]):
         m = re.search("Agreement confirmed by provider '([^']*)'", line)
         if m:
             prov_name = m.group(1)
-            logger.debug("assert_all_invoices_accepted: adding provider '%s'", prov_name)
+            logger.debug(
+                "assert_all_invoices_accepted: adding provider '%s'", prov_name
+            )
             unpaid_agreement_providers.add(prov_name)
         m = re.search("Accepted invoice from '([^']*)'", line)
         if m:
             prov_name = m.group(1)
-            logger.debug("assert_all_invoices_accepted: adding invoice for '%s'", prov_name)
+            logger.debug(
+                "assert_all_invoices_accepted: adding invoice for '%s'", prov_name
+            )
             unpaid_agreement_providers.remove(prov_name)
 
     if unpaid_agreement_providers:
-        raise AssertionError(f"Unpaid agreements for: {','.join(unpaid_agreement_providers)}")
+        raise AssertionError(
+            f"Unpaid agreements for: {','.join(unpaid_agreement_providers)}"
+        )
 
 
 @pytest.mark.asyncio
@@ -107,7 +113,9 @@ async def test_run_blender(
             all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent)
             all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
 
-            await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ", timeout=10)
+            await cmd_monitor.wait_for_pattern(
+                ".*Received proposals from 2 ", timeout=10
+            )
             logger.info("Received proposals")
 
             await cmd_monitor.wait_for_pattern(".*Agreement proposed ", timeout=10)