run-tests.yml

name: 🧪 QGIS tests

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  push:
    branches:
      - master
      - release-**
      - queued_ltr_backports
    paths:
    - 'src/**'
    - 'external/**'
    - 'python/**'
    - 'tests/**'
    - 'CMakeLists.txt'
    - '.github/workflows/run-tests.yml'
    - '.docker/**'
    - '.ci/**'
  pull_request:
    branches:
      - master
      - release-**
      - queued_ltr_backports
    # paths cannot be filtered on this workflow on pull request as it is a required one in the branch protection
    # feature request and hacks: https://github.community/t/feature-request-conditional-required-checks/16761

jobs:
  build:
    env:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      QGIS_WORKSPACE: ${{ github.workspace }} # used in docker compose
      RUN_FLAKY_TESTS: ${{ contains( github.event.pull_request.labels.*.name, 'run flaky tests') }}

    runs-on: ubuntu-latest
    continue-on-error: ${{ matrix.experimental }}

    strategy:
      matrix:
        include:
          - distro-version: '22.04'
            qt-version: 5
            run-tests: true
            docker-tag-suffix: ''
            with-qt6: OFF
            with-qt5: ON
            with-3d: ON
            with-quick: ON
            with-clazy: ON
            with-grass7: OFF
            with-grass8: OFF
            with-compile-commands: ON
            # LD_PRELOAD: /lib/x86_64-linux-gnu/libSegFault.so
            experimental: false

          - distro-version: '38'
            qt-version: 6
            run-tests: true
            docker-tag-suffix: ''
            with-qt6: ON
            with-qt5: OFF
            with-3d: ON
            with-quick: ON
            with-clazy: OFF
            with-grass7: OFF
            with-grass8: ON
            with-compile-commands: OFF
            LD_PRELOAD: ''
            experimental: false


      fail-fast: false

    outputs:
      compile_outcome: ${{ steps.compile.outcome }}
      tests_failing: ${{ steps.tests.outputs.TESTS_FAILING }}
      cdash_url: ${{ steps.tests.outputs.CDASH_URL }}
      runners_outcome: ${{ steps.runners.outcome }}

    steps:

      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
        run: |
          # Be aware that these instructions are duplicated in run-tests job
          DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')${{ matrix.docker-tag-suffix }}
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}"
          echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
          echo "QT_VERSION=${{ matrix.qt-version }}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo DOCKER_TAG: ${DOCKER_TAG}
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}
          echo QT_VERSION: ${QT_VERSION}

      - name: Build Docker Container with Build Environment
        id: docker-build
        uses: whoan/docker-build-with-cache-action@v8
        with:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          registry: ghcr.io
          image_name: qgis3-build-deps-${{ matrix.distro-version }}
          dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          build_extra_args: "--build-arg=DISTRO_VERSION=${{ matrix.distro-version }}"
          push_git_tag: true
          push_image_and_stages: on:push
          pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }}

      - name: Tag image
        run: |
          docker tag ${DOCKER_IMAGE} qgis3-build-deps
        env:
          DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }}

      - name: Push Qt5 deps image to Docker hub
        if: ${{ github.repository_owner == 'qgis' && github.event_name != 'pull_request' && matrix.qt-version == '5' }}
        run: |
          docker tag qgis3-build-deps qgis/qgis3-build-deps:${DOCKER_TAG}
          docker login -u="${DOCKER_USERNAME}" -p="${DOCKER_PASSWORD}"
          docker push "qgis/qgis3-build-deps:${DOCKER_TAG}"

      - name: Prepare build cache for pull request
        uses: pat-s/always-upload-cache@v3.0.11
        if: github.event_name == 'pull_request'
        with:
          path: /home/runner/QGIS/.ccache
          key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.actor }}-${{ github.head_ref }}-${{ github.sha }}
          # The head_ref or source branch of the pull request in a workflow run.
          # The base_ref or target branch of the pull request in a workflow run.
          restore-keys: |
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.actor }}-${{ github.head_ref }}-
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/${{ github.base_ref }}-
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/master-

      - name: Prepare build cache for branch/tag
        # use a fork of actions/cache@v2 to upload cache even when the build or test failed
        uses: pat-s/always-upload-cache@v3.0.11
        if: github.event_name != 'pull_request'
        with:
          path: /home/runner/QGIS/.ccache
          # The branch or tag ref that triggered the workflow run. For branches this in the format refs/heads/<branch_name>, and for tags it is refs/tags/<tag_name>
          key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref }}-
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-refs/heads/master-

      - name: Compile QGIS
        id: compile
        run: |
          docker run -t --name qgis_container \
                     -v $(pwd):/root/QGIS \
                     -v /home/runner/QGIS/.ccache:/root/.ccache \
                     --env-file .docker/docker-variables.env \
                     --env PUSH_TO_CDASH=true \
                     --env WITH_QT5=${{ matrix.with-qt5 }} \
                     --env BUILD_WITH_QT6=${{ matrix.with-qt6 }} \
                     --env WITH_QUICK=${{ matrix.with-quick }} \
                     --env WITH_3D=${{ matrix.with-3d }} \
                     --env WITH_GRASS7=${{ matrix.with-grass7 }} \
                     --env WITH_GRASS8=${{ matrix.with-grass8 }} \
                     --env LD_PRELOAD=${{ matrix.LD_PRELOAD }} \
                     --env WITH_CLAZY=${{ matrix.with-clazy }} \
                     --env WITH_COMPILE_COMMANDS=${{ matrix.with-compile-commands }} \
                     qgis3-build-deps \
                     /root/QGIS/.docker/docker-qgis-build.sh

      - name: Push artifact
        id: push_artifact
        if: ${{ matrix.run-tests }}
        run: tar --exclude='*.o' -cvzf build.tgz build

      - uses: actions/upload-artifact@v3
        if: ${{ matrix.run-tests }}
        with:
          name: build-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}.tgz
          path: ./build.tgz
          retention-days: 1

#      - name: Test QGIS runners
#        id: runners
#        if: ${{ matrix.distro-version == env.DEFAULT_DISTRO_VERSION }}
#        run: |
#          docker run -d --name qgis-testing-environment \
#                     -v $(pwd):/root/QGIS \
#                     -v $(pwd)/tests/src/python:/tests_directory \
#                     -v $(pwd)/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
#                     -v $(pwd)/.docker/qgis_resources/supervisor:/etc/supervisor \
#                     -e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
#                     -e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
#                     -e DISPLAY=:99 \
#                     qgis_image \
#                     /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
#          # Wait for xvfb to finish starting
#          printf "Waiting for the docker...🐳..."
#          sleep 10
#          echo " done 🥩"
#
#          declare -A testrunners
#          # Passing cases:
#          testrunners["test_testrunner.run_passing"]=0
#          testrunners["test_testrunner.run_skipped_and_passing"]=0
#          # Failing cases:
#          testrunners["test_testrunner"]=1
#          testrunners["test_testrunner.run_all"]=1
#          testrunners["test_testrunner.run_failing"]=1
#          set +e # do not exit on error
#          # Run tests in the docker
#          for i in "${!testrunners[@]}"
#          do
#            echo "::group::docker_test_runner_${i}"
#            echo "test ${i}..."
#            docker exec -i qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
#            [[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
#            echo "::endgroup::"
#          done
#          set -e # switch back
#          docker stop qgis-testing-environment


#  tests-report-comment:
#    name: Write tests report in a comment
#    needs: build
#    runs-on: ubuntu-latest
#    if: always() && ( needs.build.result == 'failure' || needs.build.result == 'success' ) && github.event_name == 'pull_request'
#    steps:
#      - name: Find Comment
#        uses: peter-evans/find-comment@v1
#        id: find-comment
#        with:
#          issue-number: ${{ github.event.pull_request.number }}
#          comment-author: 'github-actions[bot]'
#          body-includes: Tests report
#
#      - name: Create comment
#        if: ${{ steps.find-comment.outputs.comment-id == 0 }}
#        uses: peter-evans/create-or-update-comment@v1
#        id: create-comment
#        with:
#          issue-number: ${{ github.event.pull_request.number }}
#          body: |
#            **Tests report**
#
#      - name: Process
#        id: process-vars
#        env:
#          COMMENT_FOUND: ${{ steps.find-comment.outputs.comment-id }}
#          COMMENT_CREATED: ${{ steps.create-comment.outputs.comment-id }}
#          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
#          CDASH_URL: ${{ needs.build.outputs.cdash_url }}
#          COMPILE_OUTCOME: ${{ needs.build.outputs.compile_outcome }}
#          TESTS_FAILING: ${{ needs.build.outputs.tests_failing }}
#          RUNNERS_OUTCOME: ${{ needs.build.outputs.runners_outcome }}
#        run: |
#          echo "::set-output name=COMMENT_ID::"$([[ "${COMMENT_FOUND}" -eq "0" ]] && echo ${COMMENT_CREATED} || echo ${COMMENT_FOUND})
#          if [[ ${COMPILE_OUTCOME} != "success" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :scream: compilation failed"
#          elif [[ ${TESTS_FAILING} != "0" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :fire: ${TESTS_FAILING} unit-tests are failing ${CDASH_URL}"
#          elif [[ ${RUNNERS_OUTCOME} != "success" ]]; then
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :broken_heart: QGIS runners test failed"
#          else
#            echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :sunglasses: unit-tests succeeded"
#          fi
#
#      - name: Update comment
#        uses: peter-evans/create-or-update-comment@v1
#        with:
#          comment-id: ${{ steps.process-vars.outputs.COMMENT_ID }}
#          edit-mode: append
#          body: ${{ steps.process-vars.outputs.COMMENT_BODY }}

  run-tests:
    name: Run tests
    env:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
      QGIS_WORKSPACE: ${{ github.workspace }} # used in docker compose

    runs-on: ubuntu-latest

    needs: build
    if: always()

    strategy:
      matrix:
        qt-version: [5, 6]
        test-batch: [ALL_BUT_PROVIDERS, POSTGRES, HANA]

        include:
          - qt-version: 5
            distro-version: 22.04
            docker-target: binary-only

          - qt-version: 6
            distro-version: 38
            docker-target: single

          - qt-version: 5
            distro-version: 22.04
            test-batch: ORACLE
            docker-target: binary-for-oracle

        exclude:
          - qt-version: 6
            test-batch: HANA

          - qt-version: 6
            test-batch: POSTGRES

      fail-fast: false

    steps:

      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
        run: |
          # Be aware that these instructions are duplicated in build job
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}_${{ matrix.test-batch }}"
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
          echo "QT_VERSION=${{ matrix.qt-version }}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}

      - name: Build Docker Container with Testing Environment
        id: docker-build
        uses: whoan/docker-build-with-cache-action@v8
        with:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          registry: ghcr.io
          image_name: qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only
          dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          build_extra_args: "--target ${{ matrix.docker-target }} --build-arg=DISTRO_VERSION=${{ matrix.distro-version }}"
          push_git_tag: true
          push_image_and_stages: on:push
          pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }}

      - name: Tag image
        run: |
          docker tag ${DOCKER_IMAGE} qgis3-build-deps-binary-image
        env:
          DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }}

      - name: Print disk space
        run: |
          echo "DOCKER IMAGES"
          docker images
          echo "DF -H"
          sudo df -h

      - name: Download build artifact
        uses: actions/download-artifact@v3
        with:
          name: build-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}.tgz
          path: .

      - name: Extract build artifact
        run: |
          tar xvzf build.tgz
          rm -Rf build.tgz

      - name: Print disk space
        run: |
          echo "DF -H"
          sudo df -h

      - name: Run tests
        id: tests
        env:
          TEST_BATCH: ${{matrix.test-batch}}
          QGIS_COMMON_GIT_DIR: ${{ github.workspace }}
          GITHUB_SHA: ${{ github.sha }}
        run: |
          DOCKERFILE=$( ( [[ ${{ matrix.test-batch }} == "ORACLE"   ]] && echo "docker-compose-testing-oracle.yml" ) \
                     || ( [[ ${{ matrix.test-batch }} == "POSTGRES" ]] && echo "docker-compose-testing-postgres.yml" ) \
                     || echo "docker-compose-testing.yml" )
          [[ ${{ matrix.test-batch }} == "ORACLE" ]] && sudo rm -rf /usr/share/dotnet/sdk
          echo "TEST_BATCH=$TEST_BATCH"
          echo "DOCKERFILE=$DOCKERFILE"
          mkdir -p /tmp/webdav_tests && chmod 777 /tmp/webdav_tests
          mkdir -p /tmp/minio_tests/test-bucket && chmod -R 777 /tmp/minio_tests
          docker-compose -f .docker/$DOCKERFILE run -e GITHUB_SHA=$GITHUB_SHA qgis-deps /root/QGIS/.docker/docker-qgis-test.sh $TEST_BATCH

      - name: Fix permissions on test report
        if: ${{ failure() }}
        run: |
          sudo chmod -R 777 qgis_test_report

      - name: Dump report contents
        if: ${{ failure() }}
        run: |
          MD_REPORT_FILE="qgis_test_report/summary.md"; [ -f "$MD_REPORT_FILE" ] && cat "$MD_REPORT_FILE" || true

      - name: Save PR number to test report
        if: ${{ failure() }}
        run: |
          echo ${{ github.event.number }} | tee qgis_test_report/pr_number
          echo ${{ github.event.pull_request.head.sha }} | tee qgis_test_report/git_commit

      - name: Archive test results report
        if: ${{ failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: test-results-qt${{ matrix.qt-version }}
          path: qgis_test_report

  clang-tidy:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    needs: build

    strategy:
      matrix:
        include:
          - distro-version: '22.04'
            qt-version: 5

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Build Docker Container with Testing Environment
        id: docker-build
        uses: whoan/docker-build-with-cache-action@v8
        with:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
          registry: ghcr.io
          image_name: qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only
          dockerfile: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          build_extra_args: "--build-arg=DISTRO_VERSION=${{ matrix.distro-version }}"
          push_git_tag: true
          push_image_and_stages: on:push
          pull_image_and_stages: ${{ github.event_name != 'workflow_dispatch' }}

      - name: Tag image
        run: |
          docker tag ${DOCKER_IMAGE} qgis3-build-deps
        env:
          DOCKER_IMAGE: ${{ steps.docker-build.outputs.FULL_IMAGE_NAME }}

      - name: Download build artifact
        uses: actions/download-artifact@v3
        with:
          name: build-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}.tgz
          path: .

      - name: Extract build artifact
        run: |
          tar xvzf build.tgz
          rm -Rf build.tgz

      - name: Run Clang-Tidy
        run: |
          docker run -t --name qgis_container \
          -v $(pwd):/root/QGIS \
          -v /home/runner/QGIS/.ccache:/root/.ccache \
          --env-file .docker/docker-variables.env \
          qgis3-build-deps \
          /root/QGIS/.docker/docker-qgis-clangtidy.sh