pr.yaml

name: Galaxy Tool Linting and Tests for push and PR
on: [pull_request]
env:
  GALAXY_FORK: galaxyproject
  GALAXY_BRANCH: release_23.1
  MAX_CHUNKS: 4
jobs:
  # the setup job does two things:
  # 1. cache the pip cache and .planemo
  # 2. determine the list of changed repositories
  # it produces one artifact which contains
  # - a file with the latest SHA from the chosen branch of the Galaxy repo
  # - a file containing the list of changed repositories
  # which are needed in subsequent steps.
  setup:
    name: Setup cache and determine changed repositories
    runs-on: self-hosted
    outputs:
      galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
      repository-list: ${{ steps.discover.outputs.repository-list }}
      tool-list: ${{ steps.discover.outputs.tool-list }}
      chunk-count: ${{ steps.discover.outputs.chunk-count }}
      chunk-list: ${{ steps.discover.outputs.chunk-list }}
    strategy:
      matrix:
        python-version: ['3.7']
    steps:
    - name: Print github context properties
      run: |
        echo 'event: ${{ github.event_name }}'
        echo 'sha: ${{ github.sha }}'
        echo 'ref: ${{ github.ref }}'
        echo 'head_ref: ${{ github.head_ref }}'
        echo 'base_ref: ${{ github.base_ref }}'
        echo 'event.before: ${{ github.event.before }}'
        echo 'event.after: ${{ github.event.after }}'
    - name: Determine latest commit in the Galaxy repo
      id: get-galaxy-sha
      run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)"
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}-${{ secrets.CACHE_VERSION }}
    - name: Cache .planemo
      uses: actions/cache@v3
      id: cache-planemo
      with:
        path: ~/.planemo
        key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}-${{ secrets.CACHE_VERSION }}
    # Install the `wheel` package so that when installing other packages which
    # are not available as wheels, pip will build a wheel for them, which can be cached.
    - name: Install wheel
      run: pip install wheel
    - name: Install jq
      run: sudo apt-get install -yq jq libjq1
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
      uses: galaxyproject/planemo-ci-action@v1
      id: discover
      with:
        create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        max-chunks: ${{ env.MAX_CHUNKS }}
        python-version: ${{ matrix.python-version }}
    - name: Show commit range
      run: echo '${{ steps.discover.outputs.commit-range }}'
    - name: Show repository list
      run: echo '${{ steps.discover.outputs.repository-list }}'
    - name: Show tool list
      run: echo '${{ steps.discover.outputs.tool-list }}'
    - name: Show chunks
      run: |
        echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})'

  # Planemo lint the changed repositories
  lint:
    name: Lint tool-list
    needs: setup
    if: needs.setup.outputs.repository-list != ''
    runs-on: self-hosted
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8']
    steps:
    # checkout the repository
    # and use it as the current working directory
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }}-${{ secrets.CACHE_VERSION }}
    - name: Install jq
      run: sudo apt-get install -yq jq libjq1
    - name: Planemo lint
      uses: galaxyproject/planemo-ci-action@v1
      id: lint
      with:
        mode: lint
        repository-list: ${{ needs.setup.outputs.repository-list }}
        tool-list: ${{ needs.setup.outputs.tool-list }}

  # Planemo test the changed repositories, each chunk creates an artifact
  # containing HTML and JSON reports for the executed tests
  test:
    name: Test tools
    # This job runs on Linux
    runs-on: self-hosted
    needs: setup
    if: needs.setup.outputs.repository-list != ''
    strategy:
      fail-fast: false
      matrix:
        chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }}
        python-version: ['3.7']
    services:
      postgres:
        image: postgres:11
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        ports:
          - 5432:5432
    steps:
    # checkout the repository
    # and use it as the current working directory
    - uses: actions/checkout@v3
      with:
        fetch-depth: 1
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }}-${{ secrets.CACHE_VERSION }}
    - name: Cache .planemo
      uses: actions/cache@v3
      id: cache-planemo
      with:
        path: ~/.planemo
        key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }}-${{ secrets.CACHE_VERSION }}
    - name: Install jq
      run: sudo apt-get install -yq jq libjq1
    - name: Get test-data for changed repos
      run: |
        for repo in ${{ needs.setup.outputs.repository-list }}; do
          pushd $repo
          if [ -f get_test_data.sh ]; then
            mkdir -p test-data
            chmod u+x get_test_data.sh
            ./get_test_data.sh
          fi
          popd
        done
    - name: Planemo test
      uses: galaxyproject/planemo-ci-action@v1
      id: test
      with:
        mode: test
        repository-list: ${{ needs.setup.outputs.repository-list }}
        galaxy-fork: ${{ env.GALAXY_FORK }}
        galaxy-branch: ${{ env.GALAXY_BRANCH }}
        chunk: ${{ matrix.chunk }}
        chunk-count: ${{ needs.setup.outputs.chunk-count }}
        html-report: true
    - uses: actions/upload-artifact@v3
      with:
        name: 'Tool test output ${{ matrix.chunk }}'
        path: upload

  # - combine the results of the test chunks (which will never fail due
  #   to `|| true`) and create a global test report as json and html which
  #   is provided as artifact
  # - check if any tool test actually failed (by lookup in the combined json)
  #   and fail this step if this is the case
  combine_outputs:
    name: Combine chunked test results
    needs: [setup, test]
    strategy:
      matrix:
        python-version: ['3.7']
    # This job runs on Linux
    runs-on: self-hosted
    steps:
    - uses: actions/download-artifact@v3
      with:
        path: artifacts
    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Cache .cache/pip
      uses: actions/cache@v3
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }}-${{ secrets.CACHE_VERSION }}
    - name: Combine outputs
      uses: galaxyproject/planemo-ci-action@v1
      id: combine
      with:
        mode: combine
        html-report: true
    - uses: actions/upload-artifact@v3
      with:
        name: 'All tool test results'
        path: upload
    - name: Check outputs
      uses: galaxyproject/planemo-ci-action@v1
      id: check
      with:
        mode: check