ci: integrated-benchmark (#159)

## Limitations

* **Statistical outliers:** Caused by other jobs still running in parallel. The current workaround is to re-run the job. To completely fix this however, this benchmark must be set to be after all other jobs have been completed, which requires touching other workflows.
* **No windows:** Executing this on Windows CI is just too hard: cannot use powershell because of execution policy, and cannot use bash because it points to WSL which causes error.
* **No macOS:** The benchmark values are too chaotic and unreliable.
* **No clean install yet:** It's too slow.

.github/actions/binstall/action.yml

@@ -0,0 +1,19 @@
+name: Cargo B(inary)Install
+
+description: Install binaries with cargo-binstall
+
+inputs:
+  packages:
+    description: List of arguments to pass to `cargo binstall`
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+      - name: Install cargo-binstall
+        uses: taiki-e/install-action@cargo-binstall
+
+      - name: Running cargo-binstall with provided arguments
+        shell: bash
+        run: cargo binstall --no-confirm ${{ inputs.packages }}

.github/workflows/integrated-benchmark.yml

@@ -0,0 +1,152 @@
+name: Integrated-Benchmark
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  benchmark:
+    strategy:
+      max-parallel: 1
+      matrix:
+        os: [ubuntu-latest] # windows is skipped because of complexity, macos is skipped because of inconsistency
+    name: Run benchmark on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Make main branch visible
+        shell: bash
+        run: |
+          subsection() {
+            echo >&2
+            echo -e "\033[0;33m$*\033[0m" >&2
+          }
+
+          subsection 'Configuring git...'
+          git config --global advice.detachedHead false
+
+          subsection 'Checking out main branch...'
+          git checkout main
+          git log --oneline -n 3
+
+          subsection 'Switching back...'
+          git checkout -
+          git log --oneline -n 3
+
+          subsection 'Inspecting branches...'
+          git branch
+
+      - name: Cache verdaccio
+        uses: actions/cache@v3
+        with:
+          key: integrated-benchmark-verdaccio
+          path: |
+            ~/.local/share/verdaccio/storage
+        timeout-minutes: 1
+        continue-on-error: true
+
+      - name: Cache Rust builds
+        uses: actions/cache@v3
+        with:
+          key: integrated-benchmark-builds-${{ matrix.os }}
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+            bench-work-env/*/pacquet/target
+        timeout-minutes: 1
+        continue-on-error: true
+
+      - name: Install Rust Toolchain
+        uses: ./.github/actions/rustup
+        with:
+          shared-key: integrated-benchmark
+          restore-cache: false # it's insufficient
+          save-cache: false # it's insufficient
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+          run_install: false
+          standalone: true
+
+      - name: Cache pnpm
+        uses: actions/cache@v3
+        with:
+          key: integrated-benchmark-pnpm
+          path: |
+            ${{ env.PNPM_HOME }}/store/v3
+        timeout-minutes: 1
+        continue-on-error: true
+
+      - name: Install verdaccio
+        run: pnpm install --global verdaccio
+
+      - name: Install hyperfine
+        uses: ./.github/actions/binstall
+        with:
+          packages: [email protected]
+
+      - name: Build the benchmark executor
+        run: cargo build --bin=integrated-benchmark
+
+      - name: 'Benchmark: Frozen Lockfile'
+        shell: bash
+        run: |
+          cargo run --bin=integrated-benchmark -- --scenario=frozen-lockfile --verdaccio HEAD main
+          cp bench-work-env/BENCHMARK_REPORT.md bench-work-env/BENCHMARK_REPORT_FROZEN_LOCKFILE.md
+
+      # - name: 'Benchmark: Clean Install'
+      #   shell: bash
+      #   run: |
+      #     cargo run --bin=integrated-benchmark -- --scenario=clean-install --verdaccio HEAD main
+      #     cp bench-work-env/BENCHMARK_REPORT.md bench-work-env/BENCHMARK_REPORT_CLEAN_INSTALL.md
+
+      - name: Generate summary
+        shell: bash
+        run: |
+          (
+            echo '## Integrated-Benchmark Report (${{ runner.os }})'
+            echo
+            echo '### Scenario: Frozen Lockfile'
+            echo
+            cat bench-work-env/BENCHMARK_REPORT_FROZEN_LOCKFILE.md
+            # echo
+            # echo '### Scenario: Clean Install'
+            # echo
+            # cat bench-work-env/BENCHMARK_REPORT_CLEAN_INSTALL.md
+          ) > bench-work-env/SUMMARY.md
+
+      - name: Find Comment
+        # Check if the event is not triggered by a fork
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Integrated-Benchmark Report (${{ runner.os }})
+
+      - name: Create or update comment
+        # Check if the event is not triggered by a fork
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          edit-mode: replace
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body-file: bench-work-env/SUMMARY.md

.gitignore

@@ -3,3 +3,4 @@
 node_modules
 .npmrc
 /bench-work-env
+*.tmp

tasks/integrated-benchmark/src/main.rs

@@ -23,7 +23,6 @@ async fn main() {
     }
     let work_env = std::fs::canonicalize(work_env).expect("get absolute path to work env");
     let verdaccio = if verdaccio {
-        verify::ensure_program("verdaccio");
         verdaccio::VerdaccioOptions {
             client: &Default::default(),
             listen: &registry,

tasks/integrated-benchmark/src/verdaccio.rs

@@ -85,9 +85,8 @@ impl<'a> VerdaccioOptions<'a> {
     async fn spawn(self) -> Verdaccio {
         let VerdaccioOptions { listen, stdout, stderr, .. } = self;
 
-        ensure_program("verdaccio");
-
-        let process = Command::new("verdaccio")
+        let process = "verdaccio"
+            .pipe(ensure_program)
             .arg("--listen")
             .arg(listen)
             .stdin(Stdio::null())

tasks/integrated-benchmark/src/verify.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{path::Path, process::Command};
 use which::which;
 
 pub async fn ensure_virtual_registry(registry: &str) {
@@ -16,9 +16,9 @@ pub fn ensure_git_repo(path: &Path) {
     assert!(path.join("Cargo.lock").is_file());
 }
 
-pub fn ensure_program(program: &str) {
+pub fn ensure_program(program: &str) -> Command {
     match which(program) {
-        Ok(_) => (),
+        Ok(real_program) => Command::new(real_program),
         Err(which::Error::CannotFindBinaryPath) => panic!("Cannot find {program} in $PATH"),
         Err(error) => panic!("{error}"),
     }

tasks/integrated-benchmark/src/work_env.rs

@@ -58,6 +58,12 @@ impl WorkEnv {
         self.bench_dir(id).join("install.bash")
     }
 
+    fn bash_command(&self, id: BenchId) -> String {
+        let script_path = self.script_path(id);
+        let script_path = script_path.to_str().expect("convert script path to UTF-8");
+        format!("bash {script_path}")
+    }
+
     fn revision_repo(&self, revision: &str) -> PathBuf {
         self.bench_dir(BenchId::PacquetRevision(revision)).join("pacquet")
     }
@@ -99,8 +105,8 @@ impl WorkEnv {
         }
 
         eprintln!("Populating proxy registry cache...");
-        self.script_path(WorkEnv::INIT_PROXY_CACHE)
-            .pipe(Command::new)
+        Command::new("bash")
+            .arg(self.script_path(WorkEnv::INIT_PROXY_CACHE))
             .pipe_mut(executor("install.bash"))
     }
 
@@ -112,10 +118,21 @@ impl WorkEnv {
             let repository = self.repository();
             let revision_repo = self.revision_repo(revision);
             if revision_repo.exists() {
-                eprintln!("Updating {revision_repo:?} to upstream...");
+                if !revision_repo.join(".git").exists() {
+                    eprintln!("Initializing a git repository at {revision_repo:?}...");
+                    Command::new("git")
+                        .current_dir(&revision_repo)
+                        .arg("init")
+                        .arg(&revision_repo)
+                        .arg("--initial-branch=__blank__")
+                        .pipe(executor("git init"));
+                }
+
+                eprintln!("Fetching from {repository:?}...");
                 Command::new("git")
                     .current_dir(&revision_repo)
                     .arg("fetch")
+                    .arg(repository)
                     .pipe(executor("git fetch"));
             } else {
                 eprintln!("Cloning {repository:?} to {revision_repo:?}...");
@@ -166,7 +183,7 @@ impl WorkEnv {
         self.hyperfine_options.append_to(&mut command);
 
         for id in self.revision_ids().chain(self.with_pnpm.then_some(WorkEnv::PNPM)) {
-            command.arg("--command-name").arg(id.to_string()).arg(self.script_path(id));
+            command.arg("--command-name").arg(id.to_string()).arg(self.bash_command(id));
         }
 
         command