[manuf] Make orchestrator resolve paths using runfiles library

This simplifies use of the packaged orchestrator script. Instead of
having to extract the packaged runfiles manually, the rules_python
machinery extracts it into a temp dir and the library handles path
resolution.

Some hackery was required to preserve the scheme used previously for
referring to the main repo vs external repos.

Signed-off-by: Noah Moroze <[email protected]>

sw/host/provisioning/orchestrator/README.md

@@ -61,34 +61,14 @@ cp ${REPO_TOP}/bazel-bin/sw/host/provisioning/orchestrator/src/orchestrator.zip
 
 export ORCHESTRATOR_ZIP="${ORCHESTRATOR_RUN_DIR}/orchestrator.zip"
 
-# Extract runfile folders from orchestrator package.
-unzip ${ORCHESTRATOR_ZIP} \
-  "runfiles/lowrisc_opentitan/*" \
-  "runfiles/openocd/*" \
-  "runfiles/provisioning_exts/*"
-
-# All external dependencies are mapped under
-# runfiles/lowrisc_opentitan/external.
-mkdir -p  runfiles/lowrisc_opentitan/external
-
-ln -fs $(pwd)/runfiles/openocd runfiles/lowrisc_opentitan/external
-
-# The following is needed if you are using the provisioning extensions
-# infrastructure.
-PROVISIONING_EXT_RUNFILES=$(pwd)/runfiles/provisioning_exts
-[ -d "${PROVISION_EXT_RUNFILES}" ] &&   \
-  ln -fs "${PROVISIONING_EXT_RUNFILES}" \
-    runfiles/lowrisc_opentitan/external/provisioning_exts
-
 # Run tool. The path to the --sku-config parameter is relative to the
-# runfiles-dir.
+# workspace root.
 export FPGA_TARGET=hyper310
 python3 ${ORCHESTRATOR_ZIP} \
   --sku-config=sw/host/provisioning/orchestrator/configs/skus/emulation.hjson \
   --test-unlock-token="0x11111111_11111111_11111111_11111111" \
   --test-exit-token="0x22222222_22222222_22222222_22222222" \
   --fpga=${FPGA_TARGET} \
   --non-interactive \
-  --runfiles-dir=$(pwd)/runfiles/lowrisc_opentitan \
-  --db-path=$(pwd)/provisioning.sqlite
+  --db-path=provisioning.sqlite
 ```

sw/host/provisioning/orchestrator/src/BUILD

@@ -54,7 +54,10 @@ py_library(
     name = "util",
     srcs = ["util.py"],
     imports = ["."],
-    deps = [requirement("hjson")],
+    deps = [
+        requirement("hjson"),
+        "@rules_python//python/runfiles",
+    ],
 )
 
 py_library(

sw/host/provisioning/orchestrator/src/ca_config.py

@@ -6,6 +6,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 
+from util import must_resolve_runfile
+
 
 @dataclass
 class CaConfig:
@@ -18,9 +20,9 @@ class CaConfig:
 
     def __post_init__(self):
         # Update certificate and key members to Path objs if necessary.
-        self.certificate = Path(self.certificate)
+        self.certificate = Path(must_resolve_runfile(self.certificate))
         if self.key_type == "Raw":
-            self.key = Path(self.key)
+            self.key = Path(must_resolve_runfile(self.key))
         self.validate()
 
     def validate(self) -> None:

sw/host/provisioning/orchestrator/src/orchestrator.py

@@ -5,7 +5,6 @@
 
 import argparse
 import logging
-import os
 import shlex
 import subprocess
 import sys
@@ -17,7 +16,7 @@
 from device_id import DeviceId, DeviceIdentificationNumber
 from ot_dut import OtDut
 from sku_config import SkuConfig
-from util import confirm, parse_hexstring_to_int
+from util import confirm, parse_hexstring_to_int, must_resolve_runfile
 
 
 def get_user_confirmation(
@@ -104,11 +103,6 @@ def main(args_in):
         default=False,
         help="Skip all non-required user confirmations.",
     )
-    parser.add_argument(
-        "--runfiles-dir",
-        type=str,
-        help="Runfiles directory to use for provisioning.",
-    )
     parser.add_argument(
         "--log-dir",
         default="logs",
@@ -130,13 +124,10 @@ def main(args_in):
     if not args.cp_only and args.db_path is None:
         parser.error("--db-path is required when --cp-only is not provided")
 
-    # All relative paths are relative to the runfiles directory.
-    if args.runfiles_dir:
-        os.chdir(args.runfiles_dir)
-
     # Load and validate a SKU configuration file.
+    sku_config_path = must_resolve_runfile(args.sku_config)
     sku_config_args = {}
-    with open(args.sku_config, "r") as fp:
+    with open(sku_config_path, "r") as fp:
         sku_config_args = hjson.load(fp)
     sku_config = SkuConfig(**sku_config_args)
 

sw/host/provisioning/orchestrator/src/ot_dut.py

@@ -14,14 +14,17 @@
 
 from device_id import DeviceId
 from sku_config import SkuConfig
-from util import confirm, format_hex, run
+from util import confirm, format_hex, run, must_resolve_runfile
 
 # FPGA bitstream.
-_FPGA_UNIVERSAL_SPLICE_BITSTREAM = "hw/bitstream/universal/splice.bit"
+_FPGA_UNIVERSAL_SPLICE_BITSTREAM = must_resolve_runfile(
+    "hw/bitstream/universal/splice.bit")
 
 # CP and FT shared flags.
-_OPENOCD_BIN = "third_party/openocd/build_openocd/bin/openocd"
-_OPENOCD_ADAPTER_CONFIG = "external/openocd/tcl/interface/cmsis-dap.cfg"
+_OPENOCD_BIN = must_resolve_runfile(
+    "third_party/openocd/build_openocd/bin/openocd")
+_OPENOCD_ADAPTER_CONFIG = must_resolve_runfile(
+    "external/openocd/tcl/interface/cmsis-dap.cfg")
 _BASE_PROVISIONING_FLAGS = """
     --interface={target} \
     --openocd={openocd_bin} \
@@ -37,7 +40,7 @@
 _FT_PERSO_DEVICE_BIN   = "{base_dir}/ft_personalize_{sku}_{target}.prod_key_0.prod_key_0.signed.bin"  # noqa: E221, E501
 _FT_FW_BUNDLE_BIN      = "{base_dir}/ft_fw_bundle_{sku}_{target}.img"  # noqa: E221
 # CP & FT Host Binaries
-_CP_HOST_BIN = "sw/host/provisioning/cp/cp"
+_CP_HOST_BIN = must_resolve_runfile("sw/host/provisioning/cp/cp")
 _FT_HOST_BIN = "sw/host/provisioning/ft/ft_{sku}"
 # yapf: enable
 
@@ -117,6 +120,7 @@ def run_cp(self) -> None:
             host_flags += " --disable-dft-on-reset"
             device_elf = device_elf.format(base_dir=self._base_dev_dir(),
                                            target="silicon_creator")
+        device_elf = must_resolve_runfile(device_elf)
 
         # Assemble CP command.
         cmd = f"""{_CP_HOST_BIN} \
@@ -170,6 +174,7 @@ def run_ft(self) -> None:
 
         # Set cmd args and device ELF.
         host_bin = _FT_HOST_BIN.format(sku=self.sku_config.name)
+        host_bin = must_resolve_runfile(host_bin)
         host_flags = _BASE_PROVISIONING_FLAGS
         individ_elf = _FT_INDIVID_DEVICE_ELF
         perso_bin = _FT_PERSO_DEVICE_BIN
@@ -209,6 +214,10 @@ def run_ft(self) -> None:
                                                  sku=self.sku_config.name,
                                                  target="silicon_creator")
 
+        individ_elf = must_resolve_runfile(individ_elf)
+        perso_bin = must_resolve_runfile(perso_bin)
+        fw_bundle_bin = must_resolve_runfile(fw_bundle_bin)
+
         # Write CA configs to a JSON tmpfile.
         ca_config_dict = {
             "dice": self.sku_config.dice_ca.to_dict_entry(),

sw/host/provisioning/orchestrator/src/sku_config.py

@@ -10,9 +10,12 @@
 import hjson
 
 from ca_config import CaConfig
+from util import must_resolve_runfile
 
-_PRODUCT_IDS_HJSON = "sw/host/provisioning/orchestrator/data/products.hjson"
-_PACKAGE_IDS_HJSON = "sw/host/provisioning/orchestrator/data/packages/earlgrey_a1.hjson"
+_PRODUCT_IDS_HJSON = must_resolve_runfile(
+    "sw/host/provisioning/orchestrator/data/products.hjson")
+_PACKAGE_IDS_HJSON = must_resolve_runfile(
+    "sw/host/provisioning/orchestrator/data/packages/earlgrey_a1.hjson")
 
 
 @dataclass
@@ -56,6 +59,10 @@ def __post_init__(self):
         if self.package in self._package_ids:
             self.package_id = int(self._package_ids[self.package], 16)
 
+        if self.token_encrypt_key:
+            self.token_encrypt_key = must_resolve_runfile(
+                self.token_encrypt_key)
+
     @staticmethod
     def from_ids(product_id: int, si_creator_id: int,
                  package_id: int) -> "SkuConfig":

sw/host/provisioning/orchestrator/src/util.py

@@ -3,9 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 import shlex
 import subprocess
 
+from python.runfiles import Runfiles
+
 
 def parse_hexstring_to_int(x):
     """Accepts hexstrings with and without the 0x."""
@@ -56,3 +59,36 @@ def run(cmd, stdout_logfile, stderr_logfile):
     out_tee.stdin.close()
     err_tee.stdin.close()
     return res
+
+
+_runfiles = Runfiles.Create()
+
+
+def must_resolve_runfile(path):
+    """Resolves path to runfile.
+
+    Relative paths specified as external/<repo>/... will be resolved relative to
+    the external repository @<repo>. Otherwise, relative paths will be resolved
+    relative to the main workspace. Absolute paths are returned as-is.
+
+    Raises a ValueError if the path does not exist on the filesystem.
+    """
+
+    # orchestrator.py assumes the "old" style of runfiles tree, where paths to
+    # files within the main workspace do not include the repo name and external
+    # deps prepend external/.
+    #
+    # The old scheme does not work within a zipped py_binary, so this logic is a
+    # hack to fix up the supplied path.
+    #
+    # See https://docs.google.com/document/d/1skNx5o-8k5-YXUAyEETvr39eKoh9fecJbGUquPh5iy8/edit.
+    REPO = "lowrisc_opentitan"
+    if path.startswith("external/"):
+        path = path[len("external/"):]
+    else:
+        path = os.path.join(REPO, path)
+
+    resolved = _runfiles.Rlocation(path)
+    if resolved is None or not os.path.exists(resolved):
+        raise ValueError(f"Could not find runfile: {path}")
+    return resolved

sw/host/provisioning/orchestrator/tests/BUILD

@@ -31,3 +31,15 @@ py_test(
         "//sw/host/provisioning/orchestrator/src:util",
     ],
 )
+
+py_test(
+    name = "util_test",
+    srcs = ["util_test.py"],
+    data = [
+        "//sw/device/silicon_creator/manuf/keys/fake:dice_ca.pem",
+        "//third_party/openocd:jtag_cmsis_dap_adapter_cfg",
+    ],
+    deps = [
+        "//sw/host/provisioning/orchestrator/src:util",
+    ],
+)

sw/host/provisioning/orchestrator/tests/util_test.py

@@ -0,0 +1,30 @@
+import os
+import unittest
+
+from util import must_resolve_runfile
+
+
+class TestRunfileMustResolve(unittest.TestCase):
+
+    def test_main_workspace_path(self):
+        resolved = must_resolve_runfile(
+            "sw/device/silicon_creator/manuf/keys/fake/dice_ca.pem")
+        self.assertTrue(os.path.exists(resolved))
+
+    def test_external_path(self):
+        resolved = must_resolve_runfile(
+            "external/openocd/tcl/interface/cmsis-dap.cfg")
+        self.assertTrue(os.path.exists(resolved))
+
+    def test_abs_path(self):
+        path = os.path.abspath(__file__)
+        resolved = must_resolve_runfile(path)
+        self.assertEqual(path, resolved)
+
+    def test_non_existent(self):
+        with self.assertRaises(ValueError):
+            must_resolve_runfile("file-does-not-exist")
+
+
+if __name__ == '__main__':
+    unittest.main()