Merge pull request #429 from aai-institute/fix/reduce-execution-time-…

…of-tests

Reduce execution time of tests

.github/workflows/main.yaml

@@ -38,6 +38,15 @@ jobs:
         pre-commit run --all --show-diff-on-failure
         python build_scripts/run_pylint.py | (pylint-json2html -f jsonextended -o pylint.html)
       shell: bash
+    - name: Generate mypy cache key
+      id: generate-mypy-cache-key
+      run: |
+        MYPY_VERSION=$(mypy --version | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+        echo "key=mypy-$MYPY_VERSION-${{ env.pythonLocation }}" >> $GITHUB_OUTPUT
+    - uses: actions/cache@v3
+      with:
+        path: .mypy_cache
+        key: ${{ steps.generate-mypy-cache-key.outputs.key }}
     - name: Check Type Hints
       run: mypy src/
   docs:
@@ -57,40 +66,32 @@ jobs:
         pandoc-version: ${{ env.PANDOC_VERSION }}
     - name: Build Docs
       run: mkdocs build
-  base-tests:
+  group-tests:
     strategy:
+      fail-fast: false
       matrix:
         python_version: ["3.8", "3.9", "3.10"]
-    name: Run base tests
+        group_number: [1, 2, 3, 4]
+    name: Run Tests - Python ${{ matrix.python_version }} - Group ${{ matrix.group_number }}
     uses: ./.github/workflows/run-tests-workflow.yaml
     with:
-      tests_to_run: base
-      python_version: ${{ matrix.python_version }}
-    needs: [code-quality]
-  torch-tests:
-    strategy:
-      matrix:
-        python_version: ["3.8", "3.9", "3.10"]
-    name: Run tests that use PyTorch
-    uses: ./.github/workflows/run-tests-workflow.yaml
-    with:
-      tests_to_run: torch
+      split_size: 4
+      group_number: ${{ matrix.group_number }}
       python_version: ${{ matrix.python_version }}
     needs: [code-quality]
   notebook-tests:
     strategy:
       matrix:
         python_version: ["3.8", "3.9", "3.10"]
-    name: Run notebook tests
-    uses: ./.github/workflows/run-tests-workflow.yaml
+    name: Run Notebook tests -  Python ${{ matrix.python_version }}
+    uses: ./.github/workflows/run-notebook-tests-workflow.yaml
     with:
-      tests_to_run: notebooks
       python_version: ${{ matrix.python_version }}
     needs: [code-quality]
   push-docs-and-release-testpypi:
     name: Push Docs and maybe release Package to TestPyPI
     runs-on: ubuntu-latest
-    needs: [docs, base-tests, torch-tests, notebook-tests]
+    needs: [docs, group-tests, notebook-tests]
     if: ${{ github.ref == 'refs/heads/develop' }}
     concurrency:
       group: publish

.github/workflows/run-notebook-tests-workflow.yaml

@@ -0,0 +1,34 @@
+name: Run Notebook Tests
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        description: "Determines which Python version to use"
+        type: string
+        required: true
+
+env:
+  PY_COLORS: 1
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Setup Python ${{ inputs.python_version }}
+      uses: ./.github/actions/python
+      with:
+        python_version: ${{ inputs.python_version }}
+    - name: Cache Tox Directory for Tests
+      uses: actions/cache@v3
+      with:
+        key: tox-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('tox.ini', 'requirements.txt') }}-${{ inputs.python_version }}
+        path: .tox
+    - name: Set up memcached
+      uses: niden/actions-memcached@v7
+    - name: Test Notebooks
+      run: tox -e notebook-tests
+      continue-on-error: true

.github/workflows/run-tests-workflow.yaml

@@ -3,8 +3,12 @@ name: Run Tests
 on:
   workflow_call:
     inputs:
-      tests_to_run:
-        description: "Determines which tests to run using Tox. Should be one of: base, torch, notebooks"
+      split_size:
+        description: "Determines the number of groups into which the tests should be split"
+        type: string
+        default: 4
+      group_number:
+        description: "Determines which which group of tests to run. Can be 1, 2, ..., split_size"
         type: string
         required: true
       python_version:
@@ -29,17 +33,16 @@ jobs:
     - name: Cache Tox Directory for Tests
       uses: actions/cache@v3
       with:
-        key: tox-${{ github.ref }}-${{ runner.os }}-${{ hashFiles('tox.ini', 'requirements.txt') }}
+        key: tox-${{ runner.os }}-${{ github.ref }}-${{ hashFiles('tox.ini', 'requirements.txt') }}-${{ inputs.python_version }}
         path: .tox
     - name: Set up memcached
       uses: niden/actions-memcached@v7
-    - name: Test Base Modules
-      run: tox -e base
-      if: inputs.tests_to_run == 'base'
-    - name: Test Modules that rely on PyTorch
-      run: tox -e torch
-      if: inputs.tests_to_run == 'torch'
-    - name: Test Notebooks
-      run: tox -e notebooks
-      if: inputs.tests_to_run == 'notebooks'
-      continue-on-error: true
+    - name: Test Group ${{ inputs.group_number }}
+      run: tox -e tests -- --slow-tests --splits ${{ inputs.split_size }} --group ${{ inputs.group_number }}
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./coverage.xml
+        env_vars: OS,PYTHON
+        verbose: false

.gitignore

@@ -144,3 +144,6 @@ data/models/
 
 # Docs
 docs_build
+
+# pytest-profiling
+prof/