feat(tests): automated migration testing

Building on the smoke-test rewrite to use process-compose,
let's script the migration process, so that we can test
current HEAD of the monorepo against a prior tagged version,
and validate that necessary migrations are in place.

One possible approach is to fetch prebuilt binaries from uploaded
artifacts on Github. That's fine for `pd`, but doesn't work for
running the smoke tests, due to client/server incompatibility.
Therefore we'll clone the entire repo in a git-ignored subdir,
and build the old binaries there. Heavy, but reliable.

Right now, the process works as far as re-running the smokes
against the post-migration network. Confoundingly, I'm getting
a proto incompatible error, about missing AuctionParams.
AFAICT this shouldn't be happening, and I must have screwed up
somewhere. Might fall back and do all this manually to sanity
check whether it's a scripting problem or actually indicative
of a migration problem.

.gitignore

@@ -50,8 +50,11 @@ deployments/relayer/configs/penumbra-local.json
 
 # Logs, and other files from smoke tests
 deployments/logs/
+deployments/worktrees/
+deployments/bin/
 crates/bin/pcli/proposal.toml
 
+
 # Memory profiler, via bytehound or otherwise
 *.dat
 

deployments/compose/process-compose-migration-test-1.yml

@@ -0,0 +1,57 @@
+---
+# A process-compose configuration for running penumbra migration-tests.
+
+# Interleave logs from all services in single file, so it's greppable.
+log_location: deployments/logs/migration-test-1-combined.log
+
+processes:
+  build-code:
+    working_dir: deployments/worktrees/v0.73.1
+
+  # Create network configuration, for running a pd validator locally.
+  network-generate:
+    working_dir: deployments/worktrees/v0.73.1
+    command: >
+      cargo run --quiet --release --bin pd -- 
+      testnet generate --unbonding-delay 50
+      --epoch-duration 50 --timeout-commit 500ms
+
+  # Run pd validator based on generated network.
+  pd:
+    working_dir: deployments/worktrees/v0.73.1
+
+  # Run `pclientd` integration tests.
+  test-pclientd:
+    working_dir: deployments/worktrees/v0.73.1
+    log_location: deployments/logs/migration-test-1-pclientd.log
+
+  # Run `pcli` integration tests.
+  test-pcli:
+    working_dir: deployments/worktrees/v0.73.1
+    log_location: deployments/logs/migration-test-1-pcli.log
+    # We add `--skip delegate_and_undelegate` because the old smoke-test has an unreliable
+    # regex, that's since been fixed.
+    command: >-
+      cargo test --release --features sct-divergence-check,download-proving-keys --package pcli --
+      --ignored --test-threads 1 --nocapture
+      --skip delegate_and_undelegate
+    log_location: deployments/logs/smoke-test-pcli.log
+    depends_on:
+      pd:
+        condition: process_healthy
+      cometbft:
+        condition: process_started
+      test-pclientd:
+        condition: process_completed
+    availability:
+      restart: exit_on_failure
+
+  test-pd:
+    working_dir: deployments/worktrees/v0.73.1
+
+  # Finalizer task, which will wait until all test suites have finished.
+  # This allows us to ensure that.
+  summary:
+    # The `command` only runs if all tests were succesful,
+    # otherwise the process exits due to dep failure.
+    command: echo "migration tests phase 1 finished"

deployments/compose/process-compose-migration-test-2.yml

@@ -0,0 +1,19 @@
+---
+# A process-compose configuration for running penumbra migration-tests.
+# This series of commands represents performing the actual migration,
+# then starting up the network again.
+
+# Interleave logs from all services in single file, so it's greppable.
+log_location: deployments/logs/migration-test-2-combined.log
+
+processes:
+  # Don't generate, since we already did that on the old tag.
+  network-generate:
+    command: echo "skipping network generation, deferring to migration..."
+
+  # Finalizer task, which will wait until all test suites have finished.
+  # This allows us to ensure that.
+  summary:
+    # The `command` only runs if all tests were succesful,
+    # otherwise the process exits due to dep failure.
+    command: echo "migration tests phase 2 finished"

deployments/compose/process-compose.yml

@@ -0,0 +1,45 @@
+---
+# A process-compose configuration for running a local Penumbra devnet.
+# This isn't used in scripts anywhere (yet?) but serves as a reference point.
+# Potentially could be integrated with e.g. https://www.jetify.com/devbox later on.
+#
+version: "0.5"
+
+# Env vars set here will be accessible to all processes.
+environment:
+  - "RUST_LOG=info,network_integration=debug,pclientd=debug,pcli=info,pd=info,penumbra=info"
+
+log_level: info
+is_strict: true
+# Interleave logs from all services in single file, so it's greppable.
+log_location: deployments/logs/dev-env-combined.log
+
+processes:
+  # Build latest version of local code. We do this once, up front,
+  # so that each test suite runs immediately when ready, without iterative building.
+  build-code:
+    command: |-
+      printf "Building source code before running tests..."
+      cargo --quiet build --release --all-targets
+      printf " OK"
+
+  # Run pd validator based on generated network.
+  pd:
+    command: "cargo run --release --bin pd -- start"
+    readiness_probe:
+      http_get:
+        host: 127.0.0.1
+        scheme: http
+        path: "/"
+        port: 8080
+      period_seconds: 5
+    depends_on:
+      build-code:
+        condition: process_completed_successfully
+
+  # Run CometBFT for pd p2p.
+  cometbft:
+    command: "cometbft --home ~/.penumbra/testnet_data/node0/cometbft start"
+    depends_on:
+      pd:
+        condition: process_healthy

deployments/scripts/migration-test

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+# CI script to test migration compatibility. Orchestrates the following:
+#
+#   * [x] checks out prior tag and builds its binaries and tests
+#   * [x] creates local devnet via that target version
+#   * [ ] make sure devnet has very fast voting
+#   * [x] runs smoke tests against devnet, to generate txs
+#   * [ ] submit governance proposal for chain halt
+#   * [ ] vote for proposal to pass
+#   * [ ] wait for halt
+#   * [x] stops devnet ~ ends phase 1
+#   * [x] migrates devnet via latest (i.e. HEAD) pd version ~ begins phase 2
+#   * [x] restarts devnet
+#   * [ ] re-runs smoke tests, from latest version, to validate
+#
+# After all that's done, we only know that the migration didn't badly
+# break things. We should also have dedicated integration tests
+# for pre- and post-migration checks.
+
+set -euo pipefail
+
+
+# By default, look up the most recently released version of `pd` and use that to test against.
+# Otherwise, accept an override version and pull that.
+get_latest_penumbra_version() {
+    curl -sSfL https://api.github.com/repos/penumbra-zone/penumbra/releases/latest \
+        | jq -r .tag_name
+}
+
+target_version="${1:-}"
+if [[ -z "$target_version" ]] ; then
+    target_version="$(get_latest_penumbra_version)"
+    >&2 echo "No target version specified, using latest release: ${target_version}"
+fi
+
+>&2 echo "Beginning migration test from '$target_version' -> '$(git rev-parse HEAD) (HEAD)'"
+
+repo_root="$(git rev-parse --show-toplevel)"
+
+# Download prebuilt artifact from a prior release. 
+# Unlikely to use this, since we need compatible tests, too.
+download_historical_binary() {
+    target_version_bin_dir="deployments/bin/${target_version}"
+    pd_pre_migration_bin="${target_version_bin_dir}/pd"
+    if [[ ! -e "$pd_pre_migration_bin" ]] ; then
+        >&2 echo "Installing pd ${target_version}..."
+        # Download jawn.
+        # N.B. the one-liner script only exists >=0.73.1.
+        curl --proto '=https' --tlsv1.2 -LsSf "https://github.com/penumbra-zone/penumbra/releases/download/${target_version}/pd-installer.sh" | sh
+        mkdir -p "$target_version_bin_dir"
+        cp -v ~/.cargo/bin/pd "$pd_pre_migration_bin"
+    else
+        >&2 echo "pd ${target_version} already present locally, skipping download..."
+    fi
+}
+
+worktree_dir="${repo_root}/deployments/worktrees/${target_version}"
+# Create a local git-worktree so that we can check out a prior
+# tag and build its version of tests and suchwhat.
+prepare_local_worktree() {
+    if [[ ! -d "$worktree_dir" ]] ; then
+        >&2 echo "Creating new git worktree: $worktree_dir"
+        git worktree add "$worktree_dir" "$target_version"
+    fi
+}
+
+prepare_local_worktree
+
+function run_migration_test_phase_1() {
+    >&2 echo "Running smoke-tests against pre-migration devnet, phase 1..."
+    # Override the pc API port 8080 -> 9191, to avoid conflict with pd.
+    if ! process-compose \
+        --config deployments/compose/process-compose-smoke-test.yml \
+        --config deployments/compose/process-compose-migration-test-1.yml \
+        --port 9191  \
+        -t=true \
+        ; then
+        >&2 echo "ERROR: migration tests phase 1 failed"
+        >&2 echo "Review logs in: deployments/logs/migration-*.log"
+        find "${repo_root}/deployments/logs/migration-"*".log" | sort >&2
+        exit 1
+    else
+        echo "SUCCESS! Migration test phase 1 complete."
+    fi
+}
+
+
+# Post-migration, restart the network, and rerun the smoke tests against it.
+function run_migration_test_phase_2() {
+    >&2 echo "Running smoke-tests against post-migration devnet, phase 2..."
+    if ! process-compose \
+        --config deployments/compose/process-compose-smoke-test.yml \
+        --config deployments/compose/process-compose-migration-test-2.yml \
+        --port 9191  \
+        -t=true \
+        --keep-tui \
+        ; then
+        >&2 echo "ERROR: migration tests phase 2 failed"
+        >&2 echo "Review logs in: deployments/logs/migration-*.log"
+        find "${repo_root}/deployments/logs/migration-"*".log" | sort >&2
+        exit 1
+    else
+        echo "SUCCESS! Migration test phase 2 complete."
+    fi
+}
+
+# Perform most recent migration against local state, according to steps in
+# https://guide.penumbra.zone/main/node/pd/chain-upgrade.html
+function perform_migration() {
+    node0_dir="${HOME}/.penumbra/testnet_data/node0"
+    export_dir="${node0_dir}/pd-exported-state"
+    pd_home="${node0_dir}/pd"
+    cometbft_home="${node0_dir}/cometbft"
+
+    # TEMPORARY: backup entire state during development so it's easy to recover
+    tarball_backup="$HOME/tmp-state-backup.tar"
+    if [[ ! -e "$tarball_backup" ]] ; then
+        tar -cf "$tarball_backup" "$node0_dir"
+    fi
+
+    rm -rf "$export_dir"
+    cargo run --quiet --release --bin pd -- export \
+        --home "$pd_home" \
+        --export-directory "$export_dir"
+
+    cargo run --quiet --release --bin pd -- migrate \
+        --target-directory "$export_dir"
+
+    rm -rf "${pd_home}/rocksdb"
+    mv -v "${export_dir}/rocksdb" "${pd_home}/"
+    cp -v "${export_dir}/genesis.json" "${cometbft_home}/config/genesis.json"
+    cp -v "${export_dir}/priv_validator_state.json" "${cometbft_home}/data/priv_validator_state.json"
+    find "${cometbft_home}/data" \
+        -mindepth 1 -maxdepth 1 -type d -and -not -name tx_index.db -exec rm -r {} +
+}
+
+function main() {
+    run_migration_test_phase_1
+    >&2 echo PERFORMING MIGRATION
+    perform_migration
+    run_migration_test_phase_2
+}
+
+main

justfile

@@ -1,4 +1,12 @@
+migration-test:
+    # resetting network state
+    cargo run --release --bin pd -- testnet unsafe-reset-all || true
+    ./deployments/scripts/migration-test v0.73.1
+
 smoke:
     # resetting network state
     cargo run --release --bin pd -- testnet unsafe-reset-all || true
     ./deployments/scripts/smoke-test.sh
+
+dev:
+    process-compose up --port 9191 --config ./deployments/compose/process-compose.yml