ci: fix smoke test

There were a few problems with the changes in #3656. First, the YAML
workflow file was poorly indented, which caused the job not to be
evaluated, so we got a false green.

Beyond that, the wrapper script needed handling for an undefined var,
and a tweak to the backgrounding/logging changes. All are updated, and
this change passes locally.

.github/workflows/smoke.yml

@@ -22,30 +22,24 @@ jobs:
           toolchain: stable
       - name: Load rust cache
         uses: astriaorg/[email protected]
-
       - name: Install cometbft binary
         run: ./deployments/scripts/install-cometbft
-
       - name: Run the smoke test suite
         run: |
           export PATH="$HOME/bin:$PATH"
           ./deployments/scripts/smoke-test.sh
+      - name: Display comet logs
+        if: always()
+        run: cat deployments/logs/comet.log
+      - name: Display pd logs
+        if: always()
+        run: cat deployments/logs/pd.log
+      - name: Display pclientd logs
+        if: always()
+        run: cat deployments/logs/pclientd.log
+      - name: Display pcli logs
+        if: always()
+        run: cat deployments/logs/pcli.log
+    env:
+      TESTNET_RUNTIME: 2m
 
-       - name: Display comet logs
-         if: always()
-         run: cat comet.log
-
-       - name: Display pd logs
-         if: always()
-         run: cat pd.log
-
-       - name: Display pclientd logs
-         if: always()
-         run: cat pclientd.log
-
-       - name: Display pcli logs
-         if: always()
-         run: cat pcli.log
-
-        env:
-          TESTNET_RUNTIME: 2m

.gitignore

@@ -40,6 +40,8 @@ tools/parameter-setup/Cargo.lock
 # Relayer config referencing local devnet, will always be unique
 # to current host env.
 deployments/relayer/configs/penumbra-local.json
+# Log files from smoke tests
+deployments/logs/
 
 # Memory profiler, via bytehound or otherwise
 *.dat

deployments/scripts/smoke-test.sh

@@ -28,7 +28,7 @@ if ! hash cometbft > /dev/null 2>&1 ; then
 fi
 
 # If the action is running in debugging mode, then show me *everything*
-if [ -n "$RUNNER_DEBUG" ]; then
+if [ -n "${RUNNER_DEBUG:-}" ]; then
     export RUST_LOG=debug
 else
     export RUST_LOG="info,network_integration=debug,pclientd=debug,pcli=info,pd=info,penumbra=info"
@@ -39,6 +39,9 @@ TESTNET_RUNTIME="${TESTNET_RUNTIME:-120}"
 # Duration that the network will run before integration tests are run.
 TESTNET_BOOTTIME="${TESTNET_BOOTTIME:-20}"
 
+# Directory to store log output, useful for debugging; is git-ignored.
+SMOKE_LOG_DIR="deployments/logs"
+
 echo "Building latest version of pd from source..."
 cargo build --quiet --release --bin pd
 
@@ -47,11 +50,11 @@ EPOCH_DURATION="${EPOCH_DURATION:-100}"
 cargo run --quiet --release --bin pd -- testnet generate --epoch-duration "$EPOCH_DURATION" --timeout-commit 500ms
 
 echo "Starting CometBFT..."
-cometbft start --log_level=error --home "${HOME}/.penumbra/testnet_data/node0/cometbft" &> comet.log
+cometbft start --log_level=error --home "${HOME}/.penumbra/testnet_data/node0/cometbft" > "${SMOKE_LOG_DIR}/comet.log" &
 cometbft_pid="$!"
 
 echo "Starting pd..."
-cargo run --release --bin pd -- start --home "${HOME}/.penumbra/testnet_data/node0/pd" &> pd.log
+cargo run --release --bin pd -- start --home "${HOME}/.penumbra/testnet_data/node0/pd" > "${SMOKE_LOG_DIR}/pd.log" &
 pd_pid="$!"
 
 # Ensure processes are cleaned up after script exits, regardless of status.
@@ -63,12 +66,12 @@ sleep "$TESTNET_BOOTTIME"
 echo "Running pclientd integration tests against network"
 PENUMBRA_NODE_PD_URL="http://127.0.0.1:8080" \
     PCLI_UNLEASH_DANGER="yes" \
-    cargo test --release --features sct-divergence-check --package pclientd -- --ignored --test-threads 1 --nocapture &> pclientd.log
+    cargo test --release --features sct-divergence-check --package pclientd -- --ignored --test-threads 1 --nocapture | tee "${SMOKE_LOG_DIR}/pclientd.log"
 
 echo "Running pcli integration tests against network"
 PENUMBRA_NODE_PD_URL="http://127.0.0.1:8080" \
     PCLI_UNLEASH_DANGER="yes" \
-    cargo test --release --features sct-divergence-check,download-proving-keys --package pcli -- --ignored --test-threads 1 --nocapture &> pcli.log
+    cargo test --release --features sct-divergence-check,download-proving-keys --package pcli -- --ignored --test-threads 1 --nocapture | tee "${SMOKE_LOG_DIR}/pcli.log"
 
 echo "Waiting another $TESTNET_RUNTIME seconds while network runs..."
 sleep "$TESTNET_RUNTIME"
@@ -77,6 +80,7 @@ sleep "$TESTNET_RUNTIME"
 # is if it failed, so it's good enough for our needs.
 if ! kill -0 "$cometbft_pid" || ! kill -0 "$pd_pid" ; then
     >&2 echo "ERROR: smoke test process exited early"
+    >&2 echo "Review logs in: ${SMOKE_LOG_DIR}/"
     exit 1
 else
     echo "SUCCESS! Smoke test complete. Ran for $TESTNET_RUNTIME, found no errors."