release-test.yml

name: release-tests

# Run automated part of the tests specified by the release specs (see
# https://github.com/RIOT-OS/Release-Specs/) on `native` and IoT-LAB boards.
#
# It is split into `native` tests and IoT-LAB tests, as the tests are
# resource-wise disjunct and thus can be run parallel. For each IoT-LAB board
# test an IoT-LAB experiment is started using the `rici` account.
#
# This workflow is run periodically on master with a cron job, on new pushed
# release candidates tags and release tags, and can be triggered via workflow
# dispatch.

on:
  schedule:
    - cron: '0 3 * * 6'
  push:
    # Run on all new release candidates and release tags
    tags:
      - '[0-9][0-9][0-9][0-9].[0-9][0-9]-RC[0-9]*'
      - '[0-9][0-9][0-9][0-9].[0-9][0-9]'
      - '[0-9][0-9][0-9][0-9].[0-9][0-9].*'
  workflow_dispatch:
    inputs:
      riot_version:
        description: 'RIOT version to checkout'
        required: true
        default: 'master'
      release_specs_version:
        description: 'Release-Specs version to checkout'
        required: true
        default: 'master'
      docker_version:
        description: 'riot/riotbuild docker image version'
        required: true
        default: 'latest'

env:
  DOCKER_MAKE_ARGS: -j
# split up native and IoT-LAB tasks to parallelize somewhat and prevent
# to hit Github Limit of 6h per job, can't use matrix because there seems to be
# some sync happening and the longer running job is killed.
jobs:
  native-tasks:
    runs-on: ubuntu-latest
    timeout-minutes: 360
    steps:
    - name: Generate .riotgithubtoken
      run: |
        echo '${{ secrets.RIOT_CI_ACCESS_TOKEN }}' > ~/.riotgithubtoken
    - name: Checkout Release-Specs
      uses: actions/checkout@v2
      with:
        repository: RIOT-OS/Release-Specs
        path: Release-Specs
        fetch-depth: 1
        ref: ${{ github.event.inputs.release_specs_version }}
    - name: Checkout RIOT
      uses: actions/checkout@v2
      with:
        repository: RIOT-OS/RIOT
        path: RIOT
        fetch-depth: 1
        ref: ${{ github.event.inputs.riot_version }}
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.x
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install tox junit2html
    - name: Pull riotbuild docker image
      run: |
        DOCKER_VERSION="${{ github.event.inputs.docker_version }}"
        if [ -z "$DOCKER_VERSION" ]; then
          DOCKER_VERSION="latest"
        fi
        docker pull riot/riotbuild:$DOCKER_VERSION
    - name: Create TAP interfaces
      run: |
        sudo RIOT/dist/tools/tapsetup/tapsetup -c 11
    - name: Run release tests
      timeout-minutes: 350
      run: |
        RIOTBASE="$GITHUB_WORKSPACE/RIOT"
        TOX_ARGS=""
        if ! echo ${{ github.event.inputs.riot_version }} | \
              grep -q "[0-9]\{4\}.[0-9]\{2\}-RC[0-9]\+"; then
          TOX_ARGS+="--non-RC "
        fi

        cd Release-Specs
        # definition in env does not work since $GITHUB_WORKSPACE seems not to
        # be accessible
        sudo \
          BUILD_IN_DOCKER=1 \
          DOCKER_MAKE_ARGS=${DOCKER_MAKE_ARGS} \
          DOCKER_ENV_VARS=USEMODULE \
          GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \
          GITHUB_RUN_ID=${GITHUB_RUN_ID} \
          GITHUB_SERVER_URL=${GITHUB_SERVER_URL} \
          RIOTBASE=${RIOTBASE} \
          $(which tox) -- ${TOX_ARGS} -m "not iotlab_creds"
    - name: junit2html and XML deploy
      if: always()
      run: |
        DATE=$(date +"%Y-%m-%d-%H-%M-%S")
        if echo ${{ github.event.inputs.riot_version }} | \
              grep -q "[0-9]\{4\}.[0-9]\{2\}"; then
          VER=${{ github.event.inputs.riot_version }}
        else
          VER=$(git -C $GITHUB_WORKSPACE/RIOT rev-parse --short HEAD)
        fi
        mkdir test-reports/
        junit2html $GITHUB_WORKSPACE/Release-Specs/test-report.xml \
          test-reports/test-report-native-$VER-$DATE.html
        cp $GITHUB_WORKSPACE/Release-Specs/test-report.xml \
          test-reports/test-report-native-$VER-$DATE.xml
    - uses: actions/upload-artifact@v2
      if: always()
      with:
        name: Test Reports
        path: test-reports/*
  iotlab-tasks:
    runs-on: ubuntu-latest
    timeout-minutes: 360
    steps:
    - name: Generate .riotgithubtoken
      run: |
        echo '${{ secrets.RIOT_CI_ACCESS_TOKEN }}' > ~/.riotgithubtoken
    - name: Checkout Release-Specs
      uses: actions/checkout@v2
      with:
        repository: RIOT-OS/Release-Specs
        path: Release-Specs
        fetch-depth: 1
        ref: ${{ github.event.inputs.release_specs_version }}
    - name: Checkout RIOT
      uses: actions/checkout@v2
      with:
        repository: RIOT-OS/RIOT
        path: RIOT
        fetch-depth: 1
        ref: ${{ github.event.inputs.riot_version }}
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.x
    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install tox junit2html
    - name: Pull riotbuild docker image
      run: |
        DOCKER_VERSION="${{ github.event.inputs.docker_version }}"
        if [ -z "$DOCKER_VERSION" ]; then
          DOCKER_VERSION="latest"
        fi
        docker pull riot/riotbuild:$DOCKER_VERSION
    - name: Setup IoT-LAB credentials
      run: |
        echo '${{ secrets.IOTLABRC }}' > ~/.iotlabrc
    - name: Setup SSH agent
      uses: webfactory/ssh-agent@v0.4.0
      with:
        ssh-private-key: ${{ secrets.IOTLAB_PRIVATE_KEY }}
    - name: Fetch host key from IoT-LAB saclay site
      run: |
        IOTLAB_USER=$(cat ~/.iotlabrc | cut -f1 -d:)
        ssh -oStrictHostKeyChecking=accept-new \
          ${IOTLAB_USER}@saclay.iot-lab.info exit
    - name: Run release tests
      timeout-minutes: 350
      run: |
        # definition in env does not work since $GITHUB_WORKSPACE seems not to
        # be accessible
        export RIOTBASE="$GITHUB_WORKSPACE/RIOT"
        TOX_ARGS=""
        if ! echo ${{ github.event.inputs.riot_version }} | \
              grep -q "[0-9]\{4\}.[0-9]\{2\}-RC[0-9]\+"; then
          TOX_ARGS+="--non-RC "
        fi

        cd Release-Specs
        BUILD_IN_DOCKER=1 \
          DOCKER_ENV_VARS=USEMODULE \
          $(which tox) -- ${TOX_ARGS} -m "iotlab_creds"
    - name: junit2html and XML deploy
      if: always()
      run: |
        DATE=$(date +"%Y-%m-%d-%H-%M-%S")
        if echo ${{ github.event.inputs.riot_version }} | \
              grep -q "[0-9]\{4\}.[0-9]\{2\}"; then
          VER=${{ github.event.inputs.riot_version }}
        else
          VER=$(git -C $GITHUB_WORKSPACE/RIOT rev-parse --short HEAD)
        fi
        mkdir test-reports/
        junit2html $GITHUB_WORKSPACE/Release-Specs/test-report.xml \
          test-reports/test-report-iotlab-$VER-$DATE.html
        cp $GITHUB_WORKSPACE/Release-Specs/test-report.xml \
          test-reports/test-report-iotlab-$VER-$DATE.xml
    - uses: actions/upload-artifact@v2
      if: always()
      with:
        name: Test Reports
        path: test-reports/*