[ci] Introduce a test orchestrator written in bazel

ci/fpga-job.yml

@@ -0,0 +1,93 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure template for an FPGA test job.
+# This script runs an FPGA test job given the specified parameters. The job will
+# depend on the require bitstream and publish the results of the test as an artefact.
+
+parameters:
+# Name to display for the jobs.
+- name: display_name
+  type: string
+# Name of the job that other jobs can rely depend on.
+- name: job_name
+  type: string
+# Timeout for the job in minutes.
+- name: timeout
+  type: number
+# Bazel tag filters for the tests.
+- name: tag_filters
+  type: string
+# Bitstream to use.
+- name: bitstream
+  type: string
+# Azure pool board to use.
+- name: board
+  type: string
+# Opentitantool interface to use.
+- name: interface
+  type: string
+# Name of the file that holds the target patterns.
+- name: target_pattern_file
+  type: string
+  default: $(Pipeline.Workspace)/target_pattern_file.txt
+
+jobs:
+- job: ${{ parameters.job_name }}
+  displayName: ${{ parameters.display_name }}
+  pool:
+    name: $(fpga_pool)
+    demands: BOARD -equals ${{ parameters.board }}
+  timeoutInMinutes: ${{ parameters.timeout }}
+  dependsOn:
+    - ${{ parameters.bitstream }}
+    - sw_build
+  #condition: succeeded( ${{ parameters.bitstream }}, 'sw_build' )
+  condition: and(in(dependencies.${{ parameters.bitstream }}.result, 'Succeeded', 'SucceededWithIssues'), succeeded('sw_build'))
+  steps:
+  - template: ./checkout-template.yml
+  - template: ./install-package-dependencies.yml
+  - template: ./download-artifacts-template.yml
+    parameters:
+      downloadPartialBuildBinFrom:
+        - ${{ parameters.bitstream }}
+        - sw_build
+  - template: ./load-bazel-cache-write-creds.yml
+  # We run the update command twice to workaround an issue with udev on the container,
+  # where rusb cannot dynamically update its device list in CI (udev is not completely
+  # functional). If the device is in normal mode, the first thing that opentitantool
+  # does is to switch it to DFU mode and wait until it reconnects. This reconnection is
+  # never detected. But if we run the tool another time, the device list is queried again
+  # and opentitantool can finish the update. The device will now reboot in normal mode
+  # and work for the hyperdebug job.
+  - ${{ if eq(parameters.interface, 'hyper310') }}:
+    - bash: |
+        ci/bazelisk.sh run \
+            //sw/host/opentitantool:opentitantool -- \
+            --interface=hyperdebug_dfu transport update-firmware \
+        || ci/bazelisk.sh run \
+            //sw/host/opentitantool:opentitantool -- \
+            --interface=hyperdebug_dfu transport update-firmware || true
+      displayName: "Update the hyperdebug firmware"
+  - bash: |
+      set -e
+      . util/build_consts.sh
+      module load "xilinx/vivado/$(VIVADO_VERSION)"
+      # Execute a query to find all targets that match the test tags and store them in a file.
+      ci/scripts/run-bazel-test-query.sh \
+        "${{ parameters.target_pattern_file }}" \
+        "${{ parameters.tag_filters }}",-manual,-broken,-skip_in_ci \
+        //... @manufacturer_test_hooks//...
+      # Run FPGA tests.
+      if [ -s "${{ parameters.target_pattern_file }}" ]; then
+        ci/scripts/run-fpga-tests.sh "${{ parameters.interface }}" "${{ parameters.target_pattern_file }}" || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; }
+      else
+        echo "No tests to run after filtering"
+      fi
+    displayName: Execute tests
+  - template: ./publish-bazel-test-results.yml
+  - publish: "${{ parameters.target_pattern_file }}"
+    artifact: ${{ parameters.job_name }}
+    displayName: "Upload target pattern file"
+    condition: succeededOrFailed()

ci/scripts/run-bazel-test-query.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Usage:
+# run-bazel-test-query.sh <out_file> <test_tag_filters> <targets...>
+#
+# This script will perform a bazel query to include all tests specified
+# by the <targets...> but filtered according to the <test_tag_filters>
+# using the same filter logic as bazel's --test_tag_filters. The resulting
+# list of targets in placed in <out_file>
+
+set -x
+set -e
+
+if [ $# -lt 3 ]; then
+    echo >&2 "Usage: ./run-bazel-test-query.sh <out_file> <test_tag_filters> <targets...>"
+    echo >&2 "E.g. ./run-bazel-test-query.sh all_tests.txt cw310_rom_tests,-manuf //..."
+    exit 1
+fi
+out_file="$1"
+test_tags="$2"
+shift
+shift
+
+# Parse the tag filters and separate the positive from the negative ones
+declare -a positive_tags
+declare -a negative_tags
+IFS=',' read -ra tag_array <<< "$test_tags"
+for tag in "${tag_array[@]}"; do
+    if [ "${tag:0:1}" == "-" ]; then
+        negative_tags+=( "${tag:1}" )
+    else
+        positive_tags+=( "$tag" )
+    fi
+done
+# Now build a regular expression to match all tests that contain at least one positive tag.
+# Per the bazel query reference, when matching for attributes, the tags are converted to a
+# string of the form "[tag_1, tag_2, ...]", so for example "[rom, manuf, broken]" (note the
+# space after each comma). To make sure to match on entire tags, we look for the tag,
+# preceded either by "[" or a space, and followed by "]" or a comma.
+positive_tags_or=$(IFS="|"; echo "${positive_tags[*]}")
+negative_tags_or=$(IFS="|"; echo "${negative_tags[*]}")
+positive_regex="[\[ ](${positive_tags_or})[,\]]"
+negative_regex="[\[ ](${negative_tags_or})[,\]]"
+# List of targets
+targets=$(IFS="|"; echo "$*")
+targets="${targets/|/ union }"
+# Finally build the bazel query
+./ci/bazelisk.sh query \
+    --noimplicit_deps \
+    --noinclude_aspects \
+    --output=label \
+    "attr(\"tags\", \"${positive_regex}\", tests($targets)) except attr(\"tags\", \"${negative_regex}\", tests($targets))" \
+    >"${out_file}"

ci/scripts/run-fpga-tests.sh

@@ -9,13 +9,12 @@ set -e
 . util/build_consts.sh
 
 if [ $# == 0 ]; then
-    echo >&2 "Usage: run-fpga-tests.sh <fpga> <tags or test set name>"
-    echo >&2 "E.g. ./run-fpga-tests.sh cw310 manuf"
-    echo >&2 "E.g. ./run-fpga-tests.sh cw310 cw310_rom_tests"
+    echo >&2 "Usage: run-fpga-tests.sh <fpga> <target_pattern_file>"
+    echo >&2 "E.g. ./run-fpga-tests.sh cw310 list_of_test.txt"
     exit 1
 fi
 fpga="$1"
-fpga_tags="$2"
+target_pattern_file="$2"
 
 # Copy bitstreams and related files into the cache directory so Bazel will have
 # the corresponding targets in the @bitstreams workspace.
@@ -47,86 +46,6 @@ ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface="$fpga" fpga
 # Print the SAM3X firmware version. HyperDebug transports don't currently support this, so we ignore errors.
 ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface="$fpga" fpga get-sam3x-fw-version || true
 
-pattern_file=$(mktemp)
-# Recognize special test set names, otherwise we interpret it as a list of tags.
-test_args=""
-echo "tags: ${fpga_tags}"
-if [ "${fpga_tags}" == "cw310_sival_but_not_rom_ext_tests" ]
-then
-    # Only consider tests that are tagged `cw310_sival` but that do not have corresponding
-    # test tagged `cw310_sival_rom_ext`.
-
-    # The difficulty is that, technically, they are different tests since `opentitan_test` creates
-    # one target for each execution environment. The following query relies on the existence
-    # of the test suite created by `opentitan_test` that depends on all per-exec-env tests.
-    # This query only removes all tests that have sibling tagged `cw310_sival_rom_ext`. We
-    # then rely on test tag filters to only consider `cw310_sival`.
-    ci/bazelisk.sh query \
-    "
-        `# Find all tests that are dependencies of the test suite identified`
-        deps(
-            `# Find all test suites`
-            kind(
-                \"test_suite\",
-                //...
-            )
-            except
-            `# Remove all test suites depending on a test tagged cw310_sival_rom_ext`
-            `# but ignore those marked as broken or manual`
-            rdeps(
-                //...
-                except
-                attr(\"tags\",\"broken|manual\", //...),
-                `# Find all tests tagged cw310_sival_rom_ext`
-                attr(\"tags\",\"cw310_sival_rom_ext\", //...),
-                1
-            ),
-            1
-        )
-    " \
-    > "${pattern_file}"
-    # We need to remove tests tagged as manual since we are not using a wildcard target.
-    test_args="${test_args} --test_tag_filters=cw310_sival,-broken,-skip_in_ci,-manual"
-elif [ "${fpga_tags}" == "cw310_rom_but_not_manuf_and_sival_tests" ]
-then
-    # Only consider tests that are tagged `cw310_rom_with_fake_keys` or `cw310_rom_with_real_keys`
-    # but that do not have corresponding test tagged `cw310_sival` or `cw310_sival_rom_ext`. Also
-    # ignore tests tagged as `manuf`.
-
-    # This query only removes all tests that have sibling tagged  `cw310_sival` or `cw310_sival_rom_ext`.
-    # We then rely on test tag filters to only consider `cw310_sival`.
-    ci/bazelisk.sh query \
-    "
-        `# Find all tests that are dependencies of the test suite identified`
-        deps(
-            `# Find all test suites`
-            kind(
-                \"test_suite\",
-                //...
-            )
-            except
-            `# Remove all test suites depending on a test tagged cw310_sival[_rom_ext]`
-            `# but ignore those marked as broken or manual`
-            rdeps(
-                //...
-                except
-                attr(\"tags\",\"broken|manual\", //...),
-                `# Find all tests tagged cw310_sival_rom_ext`
-                attr(\"tags\",\"cw310_sival\", //...),
-                1
-            ),
-            1
-        )
-    " \
-    > "${pattern_file}"
-    # We need to remove tests tagged as manual since we are not using a wildcard target.
-    test_args="${test_args} --test_tag_filters=cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf,-broken,-skip_in_ci,-manual"
-else
-    test_args="${test_args} --test_tag_filters=${fpga_tags},-broken,-skip_in_ci"
-    echo "//..." > "${pattern_file}"
-    echo "@manufacturer_test_hooks//..." >> "${pattern_file}"
-fi
-
 ci/bazelisk.sh test \
     --define DISABLE_VERILATOR_BUILD=true \
     --nokeep_going \
@@ -135,5 +54,4 @@ ci/bazelisk.sh test \
     --build_tests_only \
     --define "$fpga"=lowrisc \
     --flaky_test_attempts=2 \
-    --target_pattern_file="${pattern_file}" \
-    ${test_args}
+    --target_pattern_file="${target_pattern_file}"

ci/verify-fpga-jobs.yml

@@ -0,0 +1,63 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+# Azure template for an FPGA test job verification.
+# This script downloads the target pattern file from all FPGA jobs that are
+# dependencies and performs some checks on them.
+
+parameters:
+# List of all FPGA jobs
+- name: 'fpga_jobs'
+  type: object
+# Test tag filter to find all FPGA tests
+- name: 'fpga_tags'
+  type: string
+
+jobs:
+- job: verify_fpga_jobs
+  displayName: Verify FPGA jobs
+  pool:
+    vmImage: ubuntu-20.04
+  dependsOn: ${{ parameters.fpga_jobs }}
+  # Run even if dependencies failed: some flaky tests might cause the job to fail
+  # but we still want to verify the FPGA jobs.
+  condition: succeededOrFailed()
+  steps:
+  - ${{ each job in parameters.fpga_jobs }}:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        buildType: current
+        targetPath: '$(Pipeline.Workspace)/verify_fpga_jobs/${{ job }}'
+        artifact: "${{ job }}"
+        patterns: target_pattern_file.txt
+      displayName: Download target pattern files from job ${{ job }}
+  - bash: |
+      ls -R $(Pipeline.Workspace)/verify_fpga_jobs
+  - bash: |
+      # Find and display all duplicates:
+      # - for each target file and each line, print '<job_name> <target>'
+      # - then sort by the target name
+      # - then keep all duplicated lines
+      pattern_files=$(find $(Pipeline.Workspace)/verify_fpga_jobs -name target_pattern_file.txt)
+      awk '{ print(gensub(/.*\/(.+)\/target_pattern_file.txt/, "\\1", "g", FILENAME) " " $0) }' $pattern_files | sort -k2 | uniq -D -f1 > duplicates.txt
+      if [ -s duplicates.txt ]; then
+        echo "The following tests ran in two or more jobs:"
+        cat duplicates.txt
+        false
+      fi
+    displayName: Checking for duplicate test runs
+  - bash: |
+      # Find and display tests that did not run:
+      ./ci/scripts/run-bazel-test-query.sh all_fpga.txt "${{ parameters.fpga_tags }}",-manual,-broken,-skip_in_ci //... @manufacturer_test_hooks//...
+      sort -o all_fpga.txt all_fpga.txt
+      pattern_files=$(find $(Pipeline.Workspace)/verify_fpga_jobs -name target_pattern_file.txt)
+      sort $pattern_files > all_run.txt
+      comm -23 all_fpga.txt all_run.txt > missing.txt
+      if [ -s missing.txt ]; then
+        echo "The following tests did not run in any job:"
+        cat missing.txt
+        false
+      fi
+    displayName: Checking for missing test runs
+    condition: succeededOrFailed()

rules/opentitan/ci.bzl

@@ -0,0 +1,40 @@
+# Copyright lowRISC contributors (OpenTitan project).
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+
+load("@bazel_skylib//lib:sets.bzl", "sets")
+
+# List of execution environments among which only one should run. The list
+# is sorted by priority: the first available will be picked.
+#
+# Important note: since only one of the those will be chosen, it is important
+# that every one of the execution environments be run in some CI job, otherwise
+# some tests will not run at all in CI.
+_ONLY_RUN_ONE_IN_CI_SORTED = [
+    "//hw/top_earlgrey:fpga_cw310_sival_rom_ext",
+    "//hw/top_earlgrey:fpga_cw310_rom_ext",
+    "//hw/top_earlgrey:fpga_cw310_sival",
+    "//hw/top_earlgrey:fpga_cw310_rom_with_fake_keys",
+    "//hw/top_earlgrey:fpga_cw310_test_rom",
+    "//hw/top_earlgrey:fpga_cw340_sival_rom_ext",
+    "//hw/top_earlgrey:fpga_cw340_sival",
+    "//hw/top_earlgrey:fpga_cw340_rom_ext",
+    "//hw/top_earlgrey:fpga_cw340_rom_with_fake_keys",
+    "//hw/top_earlgrey:fpga_cw340_test_rom",
+]
+
+def ci_orchestrator(test_name, exec_envs):
+    """
+    Given a list of execution environments, return the subset of this list
+    that should be skipped in CI.
+    """
+    exec_env_sets = sets.make(exec_envs)
+    found_one = False
+    skip_in_ci = []
+    for env in _ONLY_RUN_ONE_IN_CI_SORTED:
+        if sets.contains(exec_env_sets, env):
+            if found_one:
+                skip_in_ci.append(env)
+            found_one = True
+
+    return skip_in_ci