From 6d7888cad149f22889e7b3411434b5b66759fa84 Mon Sep 17 00:00:00 2001 From: Ryan Butler Date: Tue, 21 May 2024 14:22:30 -0400 Subject: [PATCH] ci: use subcommands in helper and stop using jq --- .github/workflows/rust-ci.yaml | 23 +++-- CONTRIBUTING.md | 17 +++- mcu-util/Cargo.toml | 2 + orb-backend-state/Cargo.toml | 2 + orb-slot-ctrl/Cargo.toml | 2 + orb-thermal-cam-ctrl/Cargo.toml | 2 + scripts/build_rust_artifacts.py | 94 ------------------ scripts/get_excludes.py | 45 --------- scripts/rust_ci_helper.py | 167 ++++++++++++++++++++++++++++++++ 9 files changed, 200 insertions(+), 154 deletions(-) delete mode 100755 scripts/build_rust_artifacts.py delete mode 100755 scripts/get_excludes.py create mode 100755 scripts/rust_ci_helper.py diff --git a/.github/workflows/rust-ci.yaml b/.github/workflows/rust-ci.yaml index b99ed217..b89b015d 100644 --- a/.github/workflows/rust-ci.yaml +++ b/.github/workflows/rust-ci.yaml @@ -114,7 +114,7 @@ jobs: - name: Configure cargo to exclude platform-specific crates run: | - tmp=($(scripts/get_excludes.py)) + tmp=($(scripts/rust_ci_helper.py excludes)) EXCLUDES="" for e in ${tmp[@]}; do EXCLUDES+="--exclude ${e} " @@ -157,33 +157,36 @@ jobs: CI_PROFILE="artifact" fi echo CI_PROFILE=${CI_PROFILE} >>${GITHUB_ENV} - - name: Compile ${{ matrix.target }} Binaries + + - name: Build linux artifacts run: | set -Eeuxo pipefail uname -a nix develop -c env - nix develop -c scripts/build_rust_artifacts.py \ - --out_dir binaries --cargo_profile ${CI_PROFILE} - ls -aRsh binaries + nix develop -c scripts/rust_ci_helper.py \ + build_linux_artifacts \ + --out_dir linux_artifacts \ + --cargo_profile ${CI_PROFILE} + ls -aRsh linux_artifacts - name: Bundle artifacts run: | set -Eeuxo pipefail - mkdir artifacts - for b in binaries/*; do + mkdir final_artifacts + for b in linux_artifacts/*; do b="$(basename ${b})" # We make sure that the tarball is idempotent: # https://stackoverflow.com/a/54908072 tar --sort=name --owner=root:0 --group=root:0 --mtime='@0' \ - -vahcf artifacts/${b}.tar.zst -C binaries/${b} . + -vahcf final_artifacts/${b}.tar.zst -C linux_artifacts/${b} . done - ls -aRsh artifacts + ls -aRsh final_artifacts - name: Upload artifacts uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # pin@v3 with: name: artifacts - path: artifacts + path: final_artifacts if-no-files-found: error cargo-deny: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c48e007..8ed11ca2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,8 +15,11 @@ change things in a backwards-incompatible way at any time! - Prefer using cargo [workspace inheritance] when possible. - Prefer cross-platform code. Please consult [deps tests](deps-tests) for more info. -- Any binaries that do not run on all platforms must be documented as such in - their README.md file and added to the tests in `deps-tests`. +- All crates and binaries must support at least + `{x86_64-aarch64}-unknown-linux-gnu` as a compilation target. Any other targets + which are not supported must be specified in + `package.metadata.orb.unsupported_targets`. Windows is implicitly not + supported as a compilation target. - Use `#![forbid(unsafe_code)]` whenever possible. This narrows the surface area for debugging memory safety issues. - Prefer the [nix crate][nix crate] for safe unix APIs instead of raw unsafe @@ -28,12 +31,16 @@ change things in a backwards-incompatible way at any time! - All first-party crates should start with the `orb-` prefix for the crate name, and the crates' directories should omit this prefix. For example, the `attest` dir contains the `orb-attest` crate. +- All binaries intended for deployment to orbs, should have a .deb produced by + CI. CI will produce any such .deb for crates with a `package.metadata.deb` + section in the Cargo.toml. ## First time Setup -1. [Install nix][install nix]. This works for both mac and linux, windows is -only supported via [WSL2][WSL2]. -2. Ensure that you have these lines in your `~/.config/nix/nix.conf`: +1. [Install nix][install nix]. This works for both mac and linux, if you are + using a windows machine, you must first set up [WSL2][WSL2]. +2. Ensure that you have these lines in your `~/.config/nix/nix.conf`. This is + done automatically by the above installer: ``` experimental-features = nix-command flakes max-jobs = auto diff --git a/mcu-util/Cargo.toml b/mcu-util/Cargo.toml index 01070402..c1d117a3 100644 --- a/mcu-util/Cargo.toml +++ b/mcu-util/Cargo.toml @@ -30,3 +30,5 @@ unsupported_targets = [ "aarch64-apple-darwin", "x86_64-apple-darwin", ] + +[package.metadata.deb] diff --git a/orb-backend-state/Cargo.toml b/orb-backend-state/Cargo.toml index ad330e97..4c13eadb 100644 --- a/orb-backend-state/Cargo.toml +++ b/orb-backend-state/Cargo.toml @@ -30,3 +30,5 @@ zbus.workspace = true [build-dependencies] orb-build-info = { path = "../build-info", features = ["build-script"] } + +[package.metadata.deb] diff --git a/orb-slot-ctrl/Cargo.toml b/orb-slot-ctrl/Cargo.toml index 8a1b1e2a..1d30b7ea 100644 --- a/orb-slot-ctrl/Cargo.toml +++ b/orb-slot-ctrl/Cargo.toml @@ -21,3 +21,5 @@ thiserror.workspace = true [build-dependencies] orb-build-info = { path = "../build-info", features = ["build-script"] } + +[package.metadata.deb] diff --git a/orb-thermal-cam-ctrl/Cargo.toml b/orb-thermal-cam-ctrl/Cargo.toml index 8f48adf4..90d35de7 100644 --- a/orb-thermal-cam-ctrl/Cargo.toml +++ b/orb-thermal-cam-ctrl/Cargo.toml @@ -30,3 +30,5 @@ unsupported_targets = [ "aarch64-apple-darwin", "x86_64-apple-darwin", ] + +[package.metadata.deb] diff --git a/scripts/build_rust_artifacts.py b/scripts/build_rust_artifacts.py deleted file mode 100755 index 2285b26f..00000000 --- a/scripts/build_rust_artifacts.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import subprocess -import sys -import os -import shlex - - -def run(command): - assert isinstance(command, str) - print(f"Running: {command}") - exit_code = subprocess.check_call(command, shell=True, text=True) - assert exit_code == 0 - - -def run_with_stdout(command): - assert isinstance(command, str) - print(f"Running: {command}") - cmd_output = subprocess.check_output(command, shell=True, text=True) - return cmd_output - - -def find_cargo_deb_crates(): - jq_query = ( - ".workspace_members[] as $wm " - "| .packages[ ] " - "| .id as $id " - "| select( $wm | contains($id)) " - '| select( .metadata | has("deb")) ' - "| .name" - ) - command = f"cargo metadata --format-version=1 | jq '{jq_query}'" - cmd_output = subprocess.check_output(command, shell=True, text=True) - crates = [c.strip('"') for c in cmd_output.strip().split("\n")] - return crates - - -def build_all_crates(*, cargo_profile, targets): - targets_option = " ".join([f"--target {t}-unknown-linux-gnu" for t in targets]) - run( - f"cargo zigbuild --all " - f"--profile {cargo_profile} " - f"{targets_option} " - f"--no-default-features" - ) - - -def run_cargo_deb(*, out_dir, cargo_profile, targets, crates): - for c in crates: - os.makedirs(os.path.join(out_dir, c), exist_ok=True) - print(f"Creating .deb package for {c}:") - for t in targets: - run( - f"cargo deb --no-build --no-strip " - f"--profile {cargo_profile} " - f"-p {c} " - f"--target {t}-unknown-linux-gnu " - f"-o {out_dir}/{c}/{c}_{t}.deb" - ) - run( - f"cp -L " - f"target/{t}-unknown-linux-gnu/{cargo_profile}/{c} " - f"{out_dir}/{c}/{c}_{t}" - ) - - -def main(): - parser = argparse.ArgumentParser(description="Builds rust artifacts for CI") - parser.add_argument( - "--out_dir", required=True, help="Output directory for artifacts" - ) - parser.add_argument( - "--cargo_profile", - required=True, - help="Cargo profile to use for compiling the crates", - ) - args = parser.parse_args() - - targets = ["aarch64", "x86_64"] - print("building all crates") - build_all_crates(cargo_profile=args.cargo_profile, targets=targets) - deb_crates = find_cargo_deb_crates() - print(f"Running cargo deb for: {deb_crates}") - run_cargo_deb( - out_dir=args.out_dir, - cargo_profile=args.cargo_profile, - targets=targets, - crates=deb_crates, - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/get_excludes.py b/scripts/get_excludes.py deleted file mode 100755 index 7e438f2a..00000000 --- a/scripts/get_excludes.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 - - -import argparse -import subprocess -import sys -import os -import shlex - - -def run_with_stdout(command): - assert isinstance(command, str) - print(f"Running: {command}", file=sys.stderr) - cmd_output = subprocess.check_output(command, shell=True, text=True) - return cmd_output - - -def get_target_triple(): - cmd_output = run_with_stdout("rustc -vV").strip().split("\n") - for s in cmd_output: - if s.startswith("host:"): - return s.split(" ")[1] - raise Exception("no target triple detected") - - -def main(): - target = get_target_triple() - print(f"identified target triple as: {target}", file=sys.stderr) - jq_query = ( - ".workspace_members[] as $wm" - "| .packages[ ] " - "| .id as $id " - "| select( $wm | contains($id)) " - "| .name as $n " - "| select (.metadata.orb.unsupported_targets " - f'| index("{target}") != null) | .name' - ) - - command = f"cargo metadata --format-version=1 | jq -r '{jq_query}'" - cmd_output = run_with_stdout(command).strip() - print(cmd_output) - - -if __name__ == "__main__": - main() diff --git a/scripts/rust_ci_helper.py b/scripts/rust_ci_helper.py new file mode 100755 index 00000000..bccc7eac --- /dev/null +++ b/scripts/rust_ci_helper.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +import json +import argparse +import subprocess +import os +from collections import defaultdict + + +def run(command): + assert isinstance(command, str) + print(f"Running: {command}") + exit_code = subprocess.check_call(command, shell=True, text=True) + assert exit_code == 0 + + +def run_with_stdout(command, print_cmd=True): + assert isinstance(command, str) + if print_cmd: + print(f"Running: {command}") + cmd_output = subprocess.check_output(command, shell=True, text=True) + return cmd_output + + +def find_cargo_deb_crates(*, workspace_crates): + def predicate(package): + m = package.get("metadata") + return m is not None and "deb" in m + + return [p for p in workspace_crates if predicate(p)] + +def find_unsupported_platform_crates(*, host_platform, workspace_crates): + def predicate(package): + tmp = package.get("metadata") or {} + tmp = tmp.get("orb") or {} + tmp = tmp.get("unsupported_targets") or {} + if tmp == {}: + return False + return host_platform in tmp + + return set([c["name"] for c in workspace_crates if predicate(c)]) + + +def workspace_crates(): + command = "cargo metadata --format-version=1" + cmd_output = run_with_stdout(command, print_cmd=False) + metadata = json.loads(cmd_output) + workspace_members = set(metadata["workspace_members"]) + + return [p for p in metadata["packages"] if p["id"] in workspace_members] + + +def get_target_triple(): + cmd_output = run_with_stdout("rustc -vV", print_cmd=False).strip().split("\n") + for s in cmd_output: + if s.startswith("host:"): + return s.split(" ")[1] + raise Exception("no target triple detected") + + +def build_all_crates(*, cargo_profile, targets): + targets_option = " ".join([f"--target {t}-unknown-linux-gnu" for t in targets]) + run( + f"cargo zigbuild --all " + f"--profile {cargo_profile} " + f"{targets_option} " + f"--no-default-features" + ) + + +def run_cargo_deb(*, out_dir, cargo_profile, targets, crate): + crate_name = crate['name'] + out = os.path.join(out_dir, crate_name) + os.makedirs(out, exist_ok=True) + print(f"Creating .deb packages for {crate_name} and copying to {out}:") + for t in targets: + run( + f"cargo deb --no-build --no-strip " + f"--profile {cargo_profile} " + f"-p {crate_name} " + f"--target {t}-unknown-linux-gnu " + f"-o {out}/{crate_name}_{t}.deb" + ) + +def get_binaries(*, workspace_crates): + """returns map of crate name to set of binaries for that crate""" + binaries = defaultdict(lambda: []) + for c in workspace_crates: + for t in c['targets']: + if t['kind'] != ['bin']: + continue + binaries[c['name']].append(t['name']) + return {k: set(v) for k, v in binaries.items() } + +def copy_cargo_binaries(*, out_dir, cargo_profile, targets, workspace_crates): + wksp_binaries = get_binaries(workspace_crates=workspace_crates) + for crate_name, binaries in wksp_binaries.items(): + out = os.path.join(out_dir, crate_name) + os.makedirs(out, exist_ok=True) + print(f"Copying binaries for {crate_name} to {out}:") + for t in targets: + target_dir = f"target/{t}-unknown-linux-gnu/{cargo_profile}" + for b in binaries: + run( + f"cp -L " + f"target/{t}-unknown-linux-gnu/{cargo_profile}/{b} " + f"{out}/{b}_{t}" + ) + +def main(): + parser = argparse.ArgumentParser(description="Scripts for Rust in CI") + subparsers = parser.add_subparsers() + + build = subparsers.add_parser( + "build_linux_artifacts", + description="Builds rust binaries and .deb packages for x86 and aarch64 linux", + ) + build.add_argument( + "--out_dir", required=True, help="Output directory for artifacts" + ) + build.add_argument( + "--cargo_profile", + required=True, + help="Cargo profile to use for compiling the crates", + ) + build.set_defaults(entry_point=subcmd_build_linux_artifacts) + + excludes = subparsers.add_parser( + "excludes", description="Detects crates that should be excluded when building" + ) + excludes.set_defaults(entry_point=subcmd_excludes) + + args = parser.parse_args() + args.entry_point(args) + + +def subcmd_build_linux_artifacts(args): + """entry point for `build_linux_artifacts` subcommand""" + targets = ["aarch64", "x86_64"] + print("building all crates") + build_all_crates(cargo_profile=args.cargo_profile, targets=targets) + + wksp_crates = workspace_crates() + deb_crates = find_cargo_deb_crates(workspace_crates=wksp_crates) + print(f"Running cargo deb for: {[c['name'] for c in deb_crates]}") + for crate in deb_crates: + run_cargo_deb( + out_dir=args.out_dir, + cargo_profile=args.cargo_profile, + targets=targets, + crate=crate, + ) + copy_cargo_binaries(workspace_crates=wksp_crates, targets=targets, out_dir=args.out_dir, cargo_profile=args.cargo_profile) + + +def subcmd_excludes(args): + """entry point for `excludes` subcommand""" + wksp_crates = workspace_crates() + host = get_target_triple() + excludes = find_unsupported_platform_crates( + host_platform=host, workspace_crates=wksp_crates + ) + print(" ".join(sorted([c for c in excludes]))) + + +if __name__ == "__main__": + main()