From 9d846d6542fcc79b9621bb3e7b7f35d85e91c6f0 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Wed, 5 Jun 2024 14:32:04 +0100 Subject: [PATCH 1/8] [ci] Introduce a test orchestrator written in bazel The commit fundamentally changes how tests are considered for running in CI. Previously, any test not marked as "manual", "broken" and "skip_in_ci" would be run in CI in all execution environment for which it is defined. We then came up with some clever bazel queries to try and avoid running them in multiple execution environment when not necessary but this is quite fragile. The approach is very different: when creating the tests using opentitan_test, we call a new function, called "ci_orchestrator" with the list of all exec_env for which it is defined. This function then returns a subset of that list that should be marked with skip_in_ci. The current implementation is that there is a list of exec_env that are mutually-exclusive (_ONLY_RUN_ONE_IN_CI_SORTED) and all but one will be skipped: the only one to run will the highest one in the list. With this approach, we are guaranteed to never run duplicate tests. If more complicated logic is required, it can always be added to this function. As a byproduct of this, we can significantly simplify the filtering logic when running FPGS jobs. Signed-off-by: Amaury Pouly --- azure-pipelines.yml | 4 +- ci/scripts/run-fpga-tests.sh | 84 +----------------------------------- rules/opentitan/ci.bzl | 36 ++++++++++++++++ rules/opentitan/defs.bzl | 34 ++++++++++++++- sw/device/tests/BUILD | 7 +++ 5 files changed, 79 insertions(+), 86 deletions(-) create mode 100644 rules/opentitan/ci.bzl diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e55daa9f6b25a..d3bf7d3ea8477 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -568,7 +568,7 @@ jobs: set -e . util/build_consts.sh module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh cw310 cw310_rom_but_not_manuf_and_sival_tests || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } + ci/scripts/run-fpga-tests.sh cw310 cw310_rom_with_fake_keys,cw310_rom_with_real_keys || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } displayName: Execute tests - template: ci/publish-bazel-test-results.yml @@ -610,7 +610,7 @@ jobs: set -e . util/build_consts.sh module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh hyper310 cw310_sival_but_not_rom_ext_tests || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } + ci/scripts/run-fpga-tests.sh hyper310 cw310_sival || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } displayName: Execute tests - template: ci/publish-bazel-test-results.yml diff --git a/ci/scripts/run-fpga-tests.sh b/ci/scripts/run-fpga-tests.sh index 7d7996b3e28c4..f11c0785fa47a 100755 --- a/ci/scripts/run-fpga-tests.sh +++ b/ci/scripts/run-fpga-tests.sh @@ -47,86 +47,6 @@ ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface="$fpga" fpga # Print the SAM3X firmware version. HyperDebug transports don't currently support this, so we ignore errors. ci/bazelisk.sh run //sw/host/opentitantool -- --rcfile= --interface="$fpga" fpga get-sam3x-fw-version || true -pattern_file=$(mktemp) -# Recognize special test set names, otherwise we interpret it as a list of tags. -test_args="" -echo "tags: ${fpga_tags}" -if [ "${fpga_tags}" == "cw310_sival_but_not_rom_ext_tests" ] -then - # Only consider tests that are tagged `cw310_sival` but that do not have corresponding - # test tagged `cw310_sival_rom_ext`. - - # The difficulty is that, technically, they are different tests since `opentitan_test` creates - # one target for each execution environment. The following query relies on the existence - # of the test suite created by `opentitan_test` that depends on all per-exec-env tests. - # This query only removes all tests that have sibling tagged `cw310_sival_rom_ext`. We - # then rely on test tag filters to only consider `cw310_sival`. - ci/bazelisk.sh query \ - " - `# Find all tests that are dependencies of the test suite identified` - deps( - `# Find all test suites` - kind( - \"test_suite\", - //... - ) - except - `# Remove all test suites depending on a test tagged cw310_sival_rom_ext` - `# but ignore those marked as broken or manual` - rdeps( - //... - except - attr(\"tags\",\"broken|manual\", //...), - `# Find all tests tagged cw310_sival_rom_ext` - attr(\"tags\",\"cw310_sival_rom_ext\", //...), - 1 - ), - 1 - ) - " \ - > "${pattern_file}" - # We need to remove tests tagged as manual since we are not using a wildcard target. - test_args="${test_args} --test_tag_filters=cw310_sival,-broken,-skip_in_ci,-manual" -elif [ "${fpga_tags}" == "cw310_rom_but_not_manuf_and_sival_tests" ] -then - # Only consider tests that are tagged `cw310_rom_with_fake_keys` or `cw310_rom_with_real_keys` - # but that do not have corresponding test tagged `cw310_sival` or `cw310_sival_rom_ext`. Also - # ignore tests tagged as `manuf`. - - # This query only removes all tests that have sibling tagged `cw310_sival` or `cw310_sival_rom_ext`. - # We then rely on test tag filters to only consider `cw310_sival`. - ci/bazelisk.sh query \ - " - `# Find all tests that are dependencies of the test suite identified` - deps( - `# Find all test suites` - kind( - \"test_suite\", - //... - ) - except - `# Remove all test suites depending on a test tagged cw310_sival[_rom_ext]` - `# but ignore those marked as broken or manual` - rdeps( - //... - except - attr(\"tags\",\"broken|manual\", //...), - `# Find all tests tagged cw310_sival_rom_ext` - attr(\"tags\",\"cw310_sival\", //...), - 1 - ), - 1 - ) - " \ - > "${pattern_file}" - # We need to remove tests tagged as manual since we are not using a wildcard target. - test_args="${test_args} --test_tag_filters=cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf,-broken,-skip_in_ci,-manual" -else - test_args="${test_args} --test_tag_filters=${fpga_tags},-broken,-skip_in_ci" - echo "//..." > "${pattern_file}" - echo "@manufacturer_test_hooks//..." >> "${pattern_file}" -fi - ci/bazelisk.sh test \ --define DISABLE_VERILATOR_BUILD=true \ --nokeep_going \ @@ -135,5 +55,5 @@ ci/bazelisk.sh test \ --build_tests_only \ --define "$fpga"=lowrisc \ --flaky_test_attempts=2 \ - --target_pattern_file="${pattern_file}" \ - ${test_args} + --test_tag_filters=${fpga_tags},-broken,-skip_in_ci \ + //... @manufacturer_test_hooks//... diff --git a/rules/opentitan/ci.bzl b/rules/opentitan/ci.bzl new file mode 100644 index 0000000000000..5fe7f1b0717e8 --- /dev/null +++ b/rules/opentitan/ci.bzl @@ -0,0 +1,36 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +load("@bazel_skylib//lib:sets.bzl", "sets") + +# List of execution environments among which only one should run. The list +# is sorted by priority: the first available will be picked. +_ONLY_RUN_ONE_IN_CI_SORTED = [ + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext", + "//hw/top_earlgrey:fpga_cw310_rom_ext", + "//hw/top_earlgrey:fpga_cw310_sival", + "//hw/top_earlgrey:fpga_cw310_rom_with_fake_keys", + "//hw/top_earlgrey:fpga_cw310_test_rom", + "//hw/top_earlgrey:fpga_cw340_sival_rom_ext", + "//hw/top_earlgrey:fpga_cw340_sival", + "//hw/top_earlgrey:fpga_cw340_rom_ext", + "//hw/top_earlgrey:fpga_cw340_rom_with_fake_keys", + "//hw/top_earlgrey:fpga_cw340_test_rom", +] + +def ci_orchestrator(test_name, exec_envs): + """ + Given a list of execution environments, return the subset of this list + that should be skipped in CI. + """ + exec_env_sets = sets.make(exec_envs) + found_one = False + skip_in_ci = [] + for env in _ONLY_RUN_ONE_IN_CI_SORTED: + if sets.contains(exec_env_sets, env): + if found_one: + skip_in_ci.append(env) + found_one = True + + return skip_in_ci diff --git a/rules/opentitan/defs.bzl b/rules/opentitan/defs.bzl index 8a238cf34e413..9f68978fd3543 100644 --- a/rules/opentitan/defs.bzl +++ b/rules/opentitan/defs.bzl @@ -46,6 +46,11 @@ load( _spx_key_by_name = "spx_key_by_name", _spx_key_for_lc_state = "spx_key_for_lc_state", ) +load( + "@lowrisc_opentitan//rules/opentitan:ci.bzl", + "ci_orchestrator", +) +load("@bazel_skylib//lib:sets.bzl", "sets") # The following definition is used to clear the key set in the signing # configuration for execution environments (exec_env) and opentitan_test @@ -185,6 +190,7 @@ def opentitan_test( silicon = _silicon_params(), verilator = _verilator_params(), data = [], + run_in_ci = None, **kwargs): """Instantiate a test per execution environment. @@ -212,6 +218,7 @@ def opentitan_test( dv: Execution overrides for a DV-based test. silicon: Execution overrides for a silicon-based test. verilator: Execution overrides for a verilator-based test. + run_in_ci: Override the automatic selection of execution environments to run in CI and run exactly those environments. kwargs: Additional execution overrides identified by the `exec_env` dict. """ test_parameters = { @@ -224,7 +231,8 @@ def opentitan_test( test_parameters.update(kwargs) kwargs_unused = kwargs.keys() - all_tests = [] + # Precompute parameters. + all_test_params = [] for (env, pname) in exec_env.items(): pname = _parameter_name(env, pname) @@ -237,6 +245,28 @@ def opentitan_test( tparam = test_parameters[pname] if pname in kwargs_unused: kwargs_unused.remove(pname) + all_test_params.append((env, pname, tparam, extra_tags)) + + # Find all exec_envs which are not marked as broken. + non_broken_exec_env = [] + for (env, pname, tparam, extra_tags) in all_test_params: + if not ('broken' in tparam.tags + extra_tags): + non_broken_exec_env.append(env) + + if run_in_ci == None: + skip_in_ci = sets.make(ci_orchestrator(name, non_broken_exec_env)) + all_envs = sets.make([env for (env, _, _, _) in all_test_params]) + run_in_ci = sets.difference(all_envs, skip_in_ci) + else: + run_in_ci = sets.make(run_in_ci) + + all_tests = [] + for (env, pname, tparam, extra_tags) in all_test_params: + # Tag test if it must not run in CI. + skip_in_ci = [] + if not sets.contains(run_in_ci, env): + skip_in_ci.append("skip_in_ci") + (_, suffix) = env.split(":") test_name = "{}_{}".format(name, suffix) all_tests.append(":" + test_name) @@ -254,7 +284,7 @@ def opentitan_test( exec_env = env, naming_convention = "{name}", # Tagging and timeout info always comes from a param block. - tags = tparam.tags + extra_tags, + tags = tparam.tags + extra_tags + skip_in_ci, timeout = tparam.timeout, local = tparam.local, # Override parameters in the test rule. diff --git a/sw/device/tests/BUILD b/sw/device/tests/BUILD index ebc665d826940..fa14c1afcd8ec 100644 --- a/sw/device/tests/BUILD +++ b/sw/device/tests/BUILD @@ -135,10 +135,17 @@ opentitan_test( EARLGREY_TEST_ENVS, EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { + "//hw/top_earlgrey:fpga_cw310_rom_ext": None, "//hw/top_earlgrey:fpga_cw310_sival": None, "//hw/top_earlgrey:silicon_creator": None, }, ), + # Use this test to ensure that all execution environments are tested in CI. + run_in_ci = EARLGREY_TEST_ENVS.keys() + [ + "//hw/top_earlgrey:fpga_cw310_sival", + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext", + "//hw/top_earlgrey:fpga_cw310_rom_ext", + ], deps = [ "//hw/ip/aes:model", "//hw/top_earlgrey/sw/autogen:top_earlgrey", From e906405357b5de596b0f1ac572164d6f7751f096 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Tue, 20 Aug 2024 16:36:27 +0200 Subject: [PATCH 2/8] [ci,bazel] Add a script to do a test-tag-filter-like query The --test_tag_filters option of `bazel test` is very convenient but is only available for testing and not for querying. This script takes as input a list of tags in the same fort as test_tag_filters and runs an equivalent bazel query to list all tests that match. Signed-off-by: Amaury Pouly --- ci/scripts/run-bazel-test-query.sh | 56 ++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 ci/scripts/run-bazel-test-query.sh diff --git a/ci/scripts/run-bazel-test-query.sh b/ci/scripts/run-bazel-test-query.sh new file mode 100755 index 0000000000000..0d26ac07fa020 --- /dev/null +++ b/ci/scripts/run-bazel-test-query.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Usage: +# run-bazel-test-query.sh +# +# This script will perform a bazel query to include all tests specified +# by the but filtered according to the +# using the same filter logic as bazel's --test_tag_filters. The resulting +# list of targets in placed in + +set -x +set -e + +if [ $# -lt 3 ]; then + echo >&2 "Usage: ./run-bazel-test-query.sh " + echo >&2 "E.g. ./run-bazel-test-query.sh all_tests.txt cw310_rom_tests,-manuf //..." + exit 1 +fi +out_file="$1" +test_tags="$2" +shift +shift + +# Parse the tag filters and separate the positive from the negative ones +declare -a positive_tags +declare -a negative_tags +IFS=',' read -ra tag_array <<< "$test_tags" +for tag in "${tag_array[@]}"; do + if [ "${tag:0:1}" == "-" ]; then + negative_tags+=( "${tag:1}" ) + else + positive_tags+=( "$tag" ) + fi +done +# Now build a regular expression to match all tests that contain at least one positive tag. +# Per the bazel query reference, when matching for attributes, the tags are converted to a +# string of the form "[tag_1, tag_2, ...]", so for example "[rom, manuf, broken]" (note the +# space after each comma). To make sure to match on entire tags, we look for the tag, +# preceded either by "[" or a space, and followed by "]" or a comma. +positive_tags_or=$(IFS="|"; echo "${positive_tags[*]}") +negative_tags_or=$(IFS="|"; echo "${negative_tags[*]}") +positive_regex="[\[ ](${positive_tags_or})[,\]]" +negative_regex="[\[ ](${negative_tags_or})[,\]]" +# List of targets +targets=$(IFS="|"; echo "$*") +targets="${targets/|/ union }" +# Finally build the bazel query +./ci/bazelisk.sh query \ + --noimplicit_deps \ + --noinclude_aspects \ + --output=label \ + "attr(\"tags\", \"${positive_regex}\", tests($targets)) except attr(\"tags\", \"${negative_regex}\", tests($targets))" \ + >"${out_file}" From 072fd998eb604bf3d42c47eeb26895a897688153 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Wed, 5 Jun 2024 16:14:13 +0100 Subject: [PATCH 3/8] [ci] Reorganize FPGA tests and put common fpga job code in a template Signed-off-by: Amaury Pouly --- azure-pipelines.yml | 376 +++++++++++------------------------ ci/fpga-job.yml | 93 +++++++++ ci/scripts/run-fpga-tests.sh | 10 +- rules/opentitan/ci.bzl | 4 + rules/opentitan/defs.bzl | 2 +- 5 files changed, 216 insertions(+), 269 deletions(-) create mode 100644 ci/fpga-job.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d3bf7d3ea8477..185e2b2722478 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,275 +518,127 @@ jobs: gcpKeyFile: "gcpkey.json" bucketURI: "gs://opentitan-bitstreams/master" -- job: execute_test_rom_fpga_tests_cw310 - displayName: CW310 Test ROM Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw310 - timeoutInMinutes: 60 - dependsOn: - - chip_earlgrey_cw310 - - sw_build - condition: succeeded( 'chip_earlgrey_cw310', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw310 - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh cw310 cw310_test_rom,-manuf || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +# CW310 FPGA jobs. +- template: ci/fpga-job.yml + parameters: + job_name: execute_test_rom_fpga_tests_cw310 + display_name: CW310 Test ROM Tests + bitstream: chip_earlgrey_cw310 + interface: cw310 + board: cw310 + tag_filters: cw310_test_rom + timeout: 60 -- job: execute_rom_fpga_tests_cw310 - displayName: CW310 ROM Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw310 - timeoutInMinutes: 90 - dependsOn: - - chip_earlgrey_cw310 - - sw_build - condition: succeeded( 'chip_earlgrey_cw310', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw310 - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh cw310 cw310_rom_with_fake_keys,cw310_rom_with_real_keys || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +- template: ci/fpga-job.yml + parameters: + job_name: execute_rom_fpga_tests_cw310 + display_name: CW310 ROM Tests + bitstream: chip_earlgrey_cw310 + interface: cw310 + board: cw310 + tag_filters: "cw310_rom_with_fake_keys,cw310_rom_with_real_keys,-manuf" + timeout: 60 -- job: execute_sival_fpga_tests_cw310 - displayName: CW310 SiVal non-ROM_EXT Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw310 - timeoutInMinutes: 75 - dependsOn: - - chip_earlgrey_cw310_hyperdebug - - sw_build - condition: succeeded( 'chip_earlgrey_cw310_hyperdebug', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw310_hyperdebug - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - # We run the update command twice to workaround an issue with udev on the container, - # where rusb cannot dynamically update its device list in CI (udev is not completely - # functional). If the device is in normal mode, the first thing that opentitantool - # does is to switch it to DFU mode and wait until it reconnects. This reconnection is - # never detected. But if we run the tool another time, the device list is queried again - # and opentitantool can finish the update. The device will now reboot in normal mode - # and work for the hyperdebug job. - - bash: | - ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware \ - || ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware || true - displayName: "Update the hyperdebug firmware" - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh hyper310 cw310_sival || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +- template: ci/fpga-job.yml + parameters: + job_name: execute_rom_ext_fpga_tests_cw310 + display_name: CW310 ROM_EXT Tests + bitstream: chip_earlgrey_cw310 + interface: cw310 + board: cw310 + tag_filters: cw310_rom_ext + timeout: 60 -- job: execute_sival_rom_ext_fpga_tests_cw310 - displayName: CW310 SiVal ROM_EXT Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw310 - timeoutInMinutes: 45 - dependsOn: - - chip_earlgrey_cw310_hyperdebug - - sw_build - condition: succeeded( 'chip_earlgrey_cw310_hyperdebug', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw310_hyperdebug - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - # We run the update command twice to workaround an issue with udev on the container, - # where rusb cannot dynamically update its device list in CI (udev is not completely - # functional). If the device is in normal mode, the first thing that opentitantool - # does is to switch it to DFU mode and wait until it reconnects. This reconnection is - # never detected. But if we run the tool another time, the device list is queried again - # and opentitantool can finish the update. The device will now reboot in normal mode - # and work for the hyperdebug job. - - bash: | - ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware \ - || ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware || true - displayName: "Update the hyperdebug firmware" - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh hyper310 cw310_sival_rom_ext || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +- template: ci/fpga-job.yml + parameters: + job_name: execute_sival_fpga_tests_cw310 + display_name: CW310 SiVal Tests + bitstream: chip_earlgrey_cw310_hyperdebug + interface: hyper310 + board: cw310 + tag_filters: "cw310_sival,-manuf" + timeout: 60 -- job: execute_rom_fpga_tests_cw340 - displayName: CW340 ROM Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw340 - timeoutInMinutes: 60 - dependsOn: - - chip_earlgrey_cw340 - - sw_build - condition: succeeded( 'chip_earlgrey_cw340', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw340 - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh cw340 cw340_rom_with_fake_keys,cw340_rom_with_real_keys,-manuf || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +- template: ci/fpga-job.yml + parameters: + job_name: execute_sival_rom_ext_fpga_tests_cw310 + display_name: CW310 SiVal ROM_EXT Tests + bitstream: chip_earlgrey_cw310_hyperdebug + interface: hyper310 + board: cw310 + tag_filters: cw310_sival_rom_ext + timeout: 60 -- job: execute_sival_rom_ext_fpga_tests_cw340 - displayName: CW340 SiVal ROM_EXT Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw340 - timeoutInMinutes: 45 - dependsOn: - - chip_earlgrey_cw340 - - sw_build - condition: succeeded( 'chip_earlgrey_cw340', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw340 - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - # We run the update command twice to workaround an issue with udev on the container, - # where rusb cannot dynamically update its device list in CI (udev is not completely - # functional). If the device is in normal mode, the first thing that opentitantool - # does is to switch it to DFU mode and wait until it reconnects. This reconnection is - # never detected. But if we run the tool another time, the device list is queried again - # and opentitantool can finish the update. The device will now reboot in normal mode - # and work for the hyperdebug job. - - bash: | - ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware \ - || ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware || true - displayName: "Update the hyperdebug firmware" - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh cw340 cw340_sival_rom_ext || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +- template: ci/fpga-job.yml + parameters: + job_name: execute_manuf_fpga_tests_cw310 + display_name: CW310 Manufacturing Tests + bitstream: chip_earlgrey_cw310_hyperdebug + interface: hyper310 + board: cw310 + tag_filters: "manuf,-cw340" + timeout: 60 -- job: execute_fpga_manuf_tests_cw310 - displayName: CW310 Manufacturing Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw310 - timeoutInMinutes: 60 - dependsOn: - - chip_earlgrey_cw310_hyperdebug - - sw_build - condition: succeeded( 'chip_earlgrey_cw310_hyperdebug', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw310_hyperdebug - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - # We run the update command for the same reasons stated above. - - bash: | - ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware \ - || ci/bazelisk.sh run \ - //sw/host/opentitantool:opentitantool -- \ - --interface=hyperdebug_dfu transport update-firmware || true - displayName: "Update the hyperdebug firmware" - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh hyper310 manuf,-cw310_sival,-broken,-cw340 || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +# CW340 FPGA jobs. +- template: ci/fpga-job.yml + parameters: + job_name: execute_test_rom_fpga_tests_cw340 + display_name: CW340 Test ROM Tests + bitstream: chip_earlgrey_cw340 + interface: cw340 + board: cw340 + tag_filters: cw340_test_rom + timeout: 60 -- job: execute_fpga_manuf_tests_cw340 - displayName: CW340 Manufacturing Tests - pool: - name: $(fpga_pool) - demands: BOARD -equals cw340 - timeoutInMinutes: 60 - dependsOn: - - chip_earlgrey_cw340 - - sw_build - condition: succeeded( 'chip_earlgrey_cw340', 'sw_build' ) - steps: - - template: ci/checkout-template.yml - - template: ci/install-package-dependencies.yml - - template: ci/download-artifacts-template.yml - parameters: - downloadPartialBuildBinFrom: - - chip_earlgrey_cw340 - - sw_build - - template: ci/load-bazel-cache-write-creds.yml - - bash: | - set -e - . util/build_consts.sh - module load "xilinx/vivado/$(VIVADO_VERSION)" - ci/scripts/run-fpga-tests.sh cw340 manuf,-cw340_sival,-broken,-hyper310 || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } - displayName: Execute tests - - template: ci/publish-bazel-test-results.yml +- template: ci/fpga-job.yml + parameters: + job_name: execute_rom_fpga_tests_cw340 + display_name: CW340 ROM Tests + bitstream: chip_earlgrey_cw340 + interface: cw340 + board: cw340 + tag_filters: "cw340_rom_with_fake_keys,cw340_rom_with_real_keys,-manuf" + timeout: 60 + +- template: ci/fpga-job.yml + parameters: + job_name: execute_rom_ext_fpga_tests_cw340 + display_name: CW340 ROM_EXT Tests + bitstream: chip_earlgrey_cw340 + interface: cw340 + board: cw340 + tag_filters: cw340_rom_ext + timeout: 60 + +- template: ci/fpga-job.yml + parameters: + job_name: execute_sival_fpga_tests_cw340 + display_name: CW340 SiVal Tests + bitstream: chip_earlgrey_cw340 + interface: cw340 + board: cw340 + tag_filters: "cw340_sival,-manuf" + timeout: 60 + +- template: ci/fpga-job.yml + parameters: + job_name: execute_sival_rom_ext_fpga_tests_cw340 + display_name: CW340 SiVal ROM_EXT Tests + bitstream: chip_earlgrey_cw340 + interface: cw340 + board: cw340 + tag_filters: cw340_sival_rom_ext + timeout: 60 + +- template: ci/fpga-job.yml + parameters: + job_name: execute_manuf_fpga_tests_cw340 + display_name: CW340 Manufacturing Tests + bitstream: chip_earlgrey_cw340 + interface: cw340 + board: cw340 + tag_filters: "manuf,-hyper310" + timeout: 60 - job: deploy_release_artifacts displayName: Package & deploy release diff --git a/ci/fpga-job.yml b/ci/fpga-job.yml new file mode 100644 index 0000000000000..734783553eed2 --- /dev/null +++ b/ci/fpga-job.yml @@ -0,0 +1,93 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Azure template for an FPGA test job. +# This script runs an FPGA test job given the specified parameters. The job will +# depend on the require bitstream and publish the results of the test as an artefact. + +parameters: +# Name to display for the jobs. +- name: display_name + type: string +# Name of the job that other jobs can rely depend on. +- name: job_name + type: string +# Timeout for the job in minutes. +- name: timeout + type: number +# Bazel tag filters for the tests. +- name: tag_filters + type: string +# Bitstream to use. +- name: bitstream + type: string +# Azure pool board to use. +- name: board + type: string +# Opentitantool interface to use. +- name: interface + type: string +# Name of the file that holds the target patterns. +- name: target_pattern_file + type: string + default: $(Pipeline.Workspace)/target_pattern_file.txt + +jobs: +- job: ${{ parameters.job_name }} + displayName: ${{ parameters.display_name }} + pool: + name: $(fpga_pool) + demands: BOARD -equals ${{ parameters.board }} + timeoutInMinutes: ${{ parameters.timeout }} + dependsOn: + - ${{ parameters.bitstream }} + - sw_build + #condition: succeeded( ${{ parameters.bitstream }}, 'sw_build' ) + condition: and(in(dependencies.${{ parameters.bitstream }}.result, 'Succeeded', 'SucceededWithIssues'), succeeded('sw_build')) + steps: + - template: ./checkout-template.yml + - template: ./install-package-dependencies.yml + - template: ./download-artifacts-template.yml + parameters: + downloadPartialBuildBinFrom: + - ${{ parameters.bitstream }} + - sw_build + - template: ./load-bazel-cache-write-creds.yml + # We run the update command twice to workaround an issue with udev on the container, + # where rusb cannot dynamically update its device list in CI (udev is not completely + # functional). If the device is in normal mode, the first thing that opentitantool + # does is to switch it to DFU mode and wait until it reconnects. This reconnection is + # never detected. But if we run the tool another time, the device list is queried again + # and opentitantool can finish the update. The device will now reboot in normal mode + # and work for the hyperdebug job. + - ${{ if eq(parameters.interface, 'hyper310') }}: + - bash: | + ci/bazelisk.sh run \ + //sw/host/opentitantool:opentitantool -- \ + --interface=hyperdebug_dfu transport update-firmware \ + || ci/bazelisk.sh run \ + //sw/host/opentitantool:opentitantool -- \ + --interface=hyperdebug_dfu transport update-firmware || true + displayName: "Update the hyperdebug firmware" + - bash: | + set -e + . util/build_consts.sh + module load "xilinx/vivado/$(VIVADO_VERSION)" + # Execute a query to find all targets that match the test tags and store them in a file. + ci/scripts/run-bazel-test-query.sh \ + "${{ parameters.target_pattern_file }}" \ + "${{ parameters.tag_filters }}",-manual,-broken,-skip_in_ci \ + //... @manufacturer_test_hooks//... + # Run FPGA tests. + if [ -s "${{ parameters.target_pattern_file }}" ]; then + ci/scripts/run-fpga-tests.sh "${{ parameters.interface }}" "${{ parameters.target_pattern_file }}" || { res=$?; echo "To reproduce failures locally, follow the instructions at https://opentitan.org/book/doc/getting_started/setup_fpga.html#reproducing-fpga-ci-failures-locally"; exit "${res}"; } + else + echo "No tests to run after filtering" + fi + displayName: Execute tests + - template: ./publish-bazel-test-results.yml + - publish: "${{ parameters.target_pattern_file }}" + artifact: ${{ parameters.job_name }} + displayName: "Upload target pattern file" + condition: succeededOrFailed() diff --git a/ci/scripts/run-fpga-tests.sh b/ci/scripts/run-fpga-tests.sh index f11c0785fa47a..26ca1d779713e 100755 --- a/ci/scripts/run-fpga-tests.sh +++ b/ci/scripts/run-fpga-tests.sh @@ -9,13 +9,12 @@ set -e . util/build_consts.sh if [ $# == 0 ]; then - echo >&2 "Usage: run-fpga-tests.sh " - echo >&2 "E.g. ./run-fpga-tests.sh cw310 manuf" - echo >&2 "E.g. ./run-fpga-tests.sh cw310 cw310_rom_tests" + echo >&2 "Usage: run-fpga-tests.sh " + echo >&2 "E.g. ./run-fpga-tests.sh cw310 list_of_test.txt" exit 1 fi fpga="$1" -fpga_tags="$2" +target_pattern_file="$2" # Copy bitstreams and related files into the cache directory so Bazel will have # the corresponding targets in the @bitstreams workspace. @@ -55,5 +54,4 @@ ci/bazelisk.sh test \ --build_tests_only \ --define "$fpga"=lowrisc \ --flaky_test_attempts=2 \ - --test_tag_filters=${fpga_tags},-broken,-skip_in_ci \ - //... @manufacturer_test_hooks//... + --target_pattern_file="${target_pattern_file}" diff --git a/rules/opentitan/ci.bzl b/rules/opentitan/ci.bzl index 5fe7f1b0717e8..4dc2f3ce00a23 100644 --- a/rules/opentitan/ci.bzl +++ b/rules/opentitan/ci.bzl @@ -6,6 +6,10 @@ load("@bazel_skylib//lib:sets.bzl", "sets") # List of execution environments among which only one should run. The list # is sorted by priority: the first available will be picked. +# +# Important note: since only one of the those will be chosen, it is important +# that every one of the execution environments be run in some CI job, otherwise +# some tests will not run at all in CI. _ONLY_RUN_ONE_IN_CI_SORTED = [ "//hw/top_earlgrey:fpga_cw310_sival_rom_ext", "//hw/top_earlgrey:fpga_cw310_rom_ext", diff --git a/rules/opentitan/defs.bzl b/rules/opentitan/defs.bzl index 9f68978fd3543..abd2570bc75f4 100644 --- a/rules/opentitan/defs.bzl +++ b/rules/opentitan/defs.bzl @@ -250,7 +250,7 @@ def opentitan_test( # Find all exec_envs which are not marked as broken. non_broken_exec_env = [] for (env, pname, tparam, extra_tags) in all_test_params: - if not ('broken' in tparam.tags + extra_tags): + if not ("broken" in tparam.tags + extra_tags): non_broken_exec_env.append(env) if run_in_ci == None: From 8e828bf2174e9b53621c13176e127924188d5ca2 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Thu, 6 Jun 2024 16:18:50 +0100 Subject: [PATCH 4/8] [sysrst_ctrl,sival] Move tests to FPGA SiVal ROM_EXT Those tests can run in the ROM_EXT stage which should be preferred. Signed-off-by: Amaury Pouly --- sw/device/tests/BUILD | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sw/device/tests/BUILD b/sw/device/tests/BUILD index fa14c1afcd8ec..8ba517e00cf8a 100644 --- a/sw/device/tests/BUILD +++ b/sw/device/tests/BUILD @@ -5315,7 +5315,7 @@ opentitan_test( exec_env = dicts.add( EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { - "//hw/top_earlgrey:fpga_cw310_sival": None, + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext": None, "//hw/top_earlgrey:sim_dv": None, }, ), @@ -5352,7 +5352,7 @@ opentitan_test( exec_env = dicts.add( EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { - "//hw/top_earlgrey:fpga_cw310_sival": None, + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext": None, "//hw/top_earlgrey:sim_dv": None, }, ), @@ -5390,7 +5390,7 @@ opentitan_test( exec_env = dicts.add( EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { - "//hw/top_earlgrey:fpga_cw310_sival": None, + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext": None, "//hw/top_earlgrey:sim_dv": None, }, ), @@ -5429,7 +5429,7 @@ opentitan_test( exec_env = dicts.add( EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { - "//hw/top_earlgrey:fpga_cw310_sival": None, + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext": None, "//hw/top_earlgrey:sim_dv": None, }, ), @@ -5467,7 +5467,7 @@ opentitan_test( exec_env = dicts.add( EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { - "//hw/top_earlgrey:fpga_cw310_sival": None, + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext": None, "//hw/top_earlgrey:sim_dv": None, }, ), @@ -5509,7 +5509,7 @@ opentitan_test( exec_env = dicts.add( EARLGREY_SILICON_OWNER_ROM_EXT_ENVS, { - "//hw/top_earlgrey:fpga_cw310_sival": None, + "//hw/top_earlgrey:fpga_cw310_sival_rom_ext": None, "//hw/top_earlgrey:sim_dv": None, }, ), From 046356b85ad77f1dd0ea9e3a0cdc6e4821689406 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Fri, 30 Aug 2024 15:45:38 +0200 Subject: [PATCH 5/8] [bazel] Automatically tag all FPGA tests with 'fpga' Previously we only tagged then with the exec_env name but for the CI it is convenient to have a global way of querying all FPGA tests. Signed-off-by: Amaury Pouly --- rules/opentitan/defs.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rules/opentitan/defs.bzl b/rules/opentitan/defs.bzl index abd2570bc75f4..4cafae04322dd 100644 --- a/rules/opentitan/defs.bzl +++ b/rules/opentitan/defs.bzl @@ -148,6 +148,8 @@ def _hacky_tags(env): (_, suffix) = env.split(":") tags = [] if suffix.startswith("fpga"): + tags.append("fpga") + # We have tags like "cw310_rom_with_real_keys" or "cw310_test_rom" # applied to our tests. Since there is no way to adjust tags in a # rule's implementation, we have to infer these tag names from the From fb58259d4bad77d471cd99ca7d8295cee56ec9d8 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Wed, 21 Aug 2024 14:14:43 +0200 Subject: [PATCH 6/8] [ci] Check that the FPGA jobs do not run duplicates and are exhaustive With the filter tags filtering logic, it is very easily to create a situation where some tests are run in several jobs, or to forget running tests because of a non-exhaustive set of filters. This commit adds a final CI job that checks the above condition. This is achieved by downaloding the list of tests run by every job and then making sure that there are no duplicates. We also check for missing tests by running a query on all non-broken, non-manual, non-skip_in_ci FPGA tests. Signed-off-by: Amaury Pouly --- azure-pipelines.yml | 18 ++++++++++++ ci/verify-fpga-jobs.yml | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 ci/verify-fpga-jobs.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 185e2b2722478..91cc4a57b6140 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -640,6 +640,24 @@ jobs: tag_filters: "manuf,-hyper310" timeout: 60 +# Verify that we have indeed run all jobs without duplication +- template: ci/verify-fpga-jobs.yml + parameters: + fpga_tags: fpga + fpga_jobs: + - execute_test_rom_fpga_tests_cw310 + - execute_rom_fpga_tests_cw310 + - execute_rom_ext_fpga_tests_cw310 + - execute_sival_fpga_tests_cw310 + - execute_sival_rom_ext_fpga_tests_cw310 + - execute_manuf_fpga_tests_cw310 + - execute_test_rom_fpga_tests_cw340 + - execute_rom_fpga_tests_cw340 + - execute_rom_ext_fpga_tests_cw340 + - execute_sival_fpga_tests_cw340 + - execute_sival_rom_ext_fpga_tests_cw340 + - execute_manuf_fpga_tests_cw340 + - job: deploy_release_artifacts displayName: Package & deploy release pool: diff --git a/ci/verify-fpga-jobs.yml b/ci/verify-fpga-jobs.yml new file mode 100644 index 0000000000000..5ae9d37c34211 --- /dev/null +++ b/ci/verify-fpga-jobs.yml @@ -0,0 +1,63 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Azure template for an FPGA test job verification. +# This script downloads the target pattern file from all FPGA jobs that are +# dependencies and performs some checks on them. + +parameters: +# List of all FPGA jobs +- name: 'fpga_jobs' + type: object +# Test tag filter to find all FPGA tests +- name: 'fpga_tags' + type: string + +jobs: +- job: verify_fpga_jobs + displayName: Verify FPGA jobs + pool: + vmImage: ubuntu-20.04 + dependsOn: ${{ parameters.fpga_jobs }} + # Run even if dependencies failed: some flaky tests might cause the job to fail + # but we still want to verify the FPGA jobs. + condition: succeededOrFailed() + steps: + - ${{ each job in parameters.fpga_jobs }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + targetPath: '$(Pipeline.Workspace)/verify_fpga_jobs/${{ job }}' + artifact: "${{ job }}" + patterns: target_pattern_file.txt + displayName: Download target pattern files from job ${{ job }} + - bash: | + ls -R $(Pipeline.Workspace)/verify_fpga_jobs + - bash: | + # Find and display all duplicates: + # - for each target file and each line, print ' ' + # - then sort by the target name + # - then keep all duplicated lines + pattern_files=$(find $(Pipeline.Workspace)/verify_fpga_jobs -name target_pattern_file.txt) + awk '{ print(gensub(/.*\/(.+)\/target_pattern_file.txt/, "\\1", "g", FILENAME) " " $0) }' $pattern_files | sort -k2 | uniq -D -f1 > duplicates.txt + if [ -s duplicates.txt ]; then + echo "The following tests ran in two or more jobs:" + cat duplicates.txt + false + fi + displayName: Checking for duplicate test runs + - bash: | + # Find and display tests that did not run: + ./ci/scripts/run-bazel-test-query.sh all_fpga.txt "${{ parameters.fpga_tags }}",-manual,-broken,-skip_in_ci //... @manufacturer_test_hooks//... + sort -o all_fpga.txt all_fpga.txt + pattern_files=$(find $(Pipeline.Workspace)/verify_fpga_jobs -name target_pattern_file.txt) + sort $pattern_files > all_run.txt + comm -23 all_fpga.txt all_run.txt > missing.txt + if [ -s missing.txt ]; then + echo "The following tests did not run in any job:" + cat missing.txt + false + fi + displayName: Checking for missing test runs + condition: succeededOrFailed() From 8a211ea04884c67f9152d4b51f2b9dc948af25ac Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Thu, 29 Aug 2024 12:40:08 +0200 Subject: [PATCH 7/8] [rom_ext,e2e] Mark some tests are broken Those tests require the host interface to support serial break but the CW310 used in the CI do not support it so they always fail. These tests will need to be changed to the hyperdebug interface. Signed-off-by: Amaury Pouly --- sw/device/silicon_creator/rom_ext/e2e/lockdown/BUILD | 8 ++++++++ sw/device/silicon_creator/rom_ext/e2e/rescue/BUILD | 3 +++ 2 files changed, 11 insertions(+) diff --git a/sw/device/silicon_creator/rom_ext/e2e/lockdown/BUILD b/sw/device/silicon_creator/rom_ext/e2e/lockdown/BUILD index f579644b31ac4..26292f9f98a4c 100644 --- a/sw/device/silicon_creator/rom_ext/e2e/lockdown/BUILD +++ b/sw/device/silicon_creator/rom_ext/e2e/lockdown/BUILD @@ -23,6 +23,7 @@ opentitan_test( # corresponds to the OTP controller at # OTP_CTRL_SW_CFG_WINDOW_REG_OFFSET + OTP_CTRL_PARAM_CREATOR_SW_CFG_OFFSET. exit_success = "FAULT: Load Access Fault.*MTVAL=40131040\r\n", + tags = ["broken"], ), deps = [ "//hw/top_earlgrey/sw/autogen:top_earlgrey", @@ -47,6 +48,7 @@ opentitan_test( # The address below # corresponds to the OTP controller at # OTP_CTRL_CHECK_REGWEN_REG_OFFSET. exit_success = "FAULT: Load Access Fault.*MTVAL=4013003c\r\n", + tags = ["broken"], ), deps = [ "//hw/top_earlgrey/sw/autogen:top_earlgrey", @@ -62,6 +64,9 @@ opentitan_test( exec_env = { "//hw/top_earlgrey:fpga_cw310_rom_ext": None, }, + fpga = fpga_params( + tags = ["broken"], + ), deps = [ "//hw/ip/sram_ctrl/data:sram_ctrl_c_regs", "//hw/top_earlgrey/sw/autogen:top_earlgrey", @@ -76,6 +81,9 @@ opentitan_test( exec_env = { "//hw/top_earlgrey:fpga_cw310_rom_ext": None, }, + fpga = fpga_params( + tags = ["broken"], + ), deps = [ "//hw/top_earlgrey/sw/autogen:top_earlgrey", "//sw/device/lib/base:status", diff --git a/sw/device/silicon_creator/rom_ext/e2e/rescue/BUILD b/sw/device/silicon_creator/rom_ext/e2e/rescue/BUILD index 1d82851d449ce..fb6e242cd7327 100644 --- a/sw/device/silicon_creator/rom_ext/e2e/rescue/BUILD +++ b/sw/device/silicon_creator/rom_ext/e2e/rescue/BUILD @@ -60,6 +60,7 @@ _POSITIONS = { ":boot_test_{}".format(name): "payload", }, slot = position["slot"], + tags = ["broken"], test_cmd = """ --exec="transport init" --exec="fpga load-bitstream {bitstream}" @@ -88,6 +89,7 @@ opentitan_test( ":boot_test_slot_a": "slot_a", ":boot_test_slot_b": "slot_b", }, + tags = ["broken"], test_cmd = """ --exec="transport init" --exec="fpga load-bitstream {bitstream}" @@ -116,6 +118,7 @@ opentitan_test( ":boot_test_slot_a": "slot_a", ":boot_test_slot_b": "slot_b", }, + tags = ["broken"], test_cmd = """ --exec="transport init" --exec="fpga load-bitstream {bitstream}" From 1c44deb899aac802eea8a3fa225757b91833b55b Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Thu, 29 Aug 2024 14:38:58 +0200 Subject: [PATCH 8/8] [tests] Mark rv_dm_mem_access_rv_dm_delayed_enable as broken The test rv_dm_mem_access_dev_rv_dm_delayed_enabled is broken. Signed-off-by: Amaury Pouly --- sw/device/tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sw/device/tests/BUILD b/sw/device/tests/BUILD index 8ba517e00cf8a..0a3efb22ffe48 100644 --- a/sw/device/tests/BUILD +++ b/sw/device/tests/BUILD @@ -4615,7 +4615,7 @@ test_suite( otp = test_cfg["otp"], tags = [ "lc_{}".format(test_cfg["lc_state"]), - ], + ] + (["broken"] if "delayed_enabled" in test_cfg["name"] else []), test_cmd = " --rom={rom:binary}", test_harness = "//sw/host/tests/chip/rv_dm:mem_access", ),