feat(cov): simulator test coverage (#15)

* feat(cov): simulator tests coverage

* ci: add simulator tests

* ci: run ci when `main` changes

* chore(make): add simulator support

* chore: fix ci comment

---------

Co-authored-by: Reece Williams <[email protected]>

.coverageignore

@@ -1,4 +1,3 @@
 *.pb.go
 *.pb.gw.go
 *.pulsar.go
-*_simulation.go

.github/workflows/build.yml

@@ -1,6 +1,9 @@
 name: Build appd
 
 on:
+  push:
+    branches:
+      - main
   pull_request:
 
 concurrency:

.github/workflows/e2e.yml

@@ -1,6 +1,9 @@
 name: E2E
 
 on:
+  push:
+    branches:
+      - main
   pull_request:
 
 permissions:

.github/workflows/simulator.yml

@@ -0,0 +1,35 @@
+name: Simulator tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  GO_VERSION: 1.21.0
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source
+        uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          check-latest: true
+      - name: Full application simulation (fixed seed)
+        run: make sim-full-app
+      - name: Simulation after state import (fixed seed)
+        run: make sim-after-import
+#      #  Requires wiring v2
+#      # https://github.com/strangelove-ventures/tokenfactory/issues/13
+#      - name: Simulation import/export (fixed seed)
+#        run: make sim-import-export
+      - name: Simulate application state determinism (fixed seed)
+        run: make sim-app-determinism

.github/workflows/test.yml

@@ -1,6 +1,9 @@
 name: Unit Tests
 
 on:
+  push:
+    branches:
+      - main
   pull_request:
 
 env:

.gitignore

@@ -27,3 +27,4 @@ cache
 state
 
 configs/logs.json
+go.work.sum

Makefile

@@ -130,18 +130,44 @@ test-integration:
 	@echo "--> Running integration tests"
 	cd integration; go test -v ./...
 
+COV_ROOT="/tmp/tokenfactory-coverage"
+COV_UNIT_E2E="${COV_ROOT}/unit-e2e"
+COV_SIMULATION="${COV_ROOT}/simulation"
+COV_PKG="github.com/strangelove-ventures/tokenfactory/..."
+COV_SIM_CMD=${COV_SIMULATION}/simulation.test
+COV_SIM_COMMON=-Enabled=True -NumBlocks=100 -Commit=true -Period=5 -Verbose=false -test.v -test.gocoverdir=${COV_SIMULATION}
+
 coverage: ## Run coverage report
-	@echo "--> Running coverage"
-	@go test -race -cpu=$$(nproc) -covermode=atomic -coverprofile=coverage.out $$(go list ./...) ./interchaintest/... -coverpkg=github.com/strangelove-ventures/tokenfactory/... > /dev/null 2>&1
-	@echo "--> Running coverage filter"
-	@./scripts/filter-coverage.sh
-	@echo "--> Running coverage report"
-	@go tool cover -func=coverage-filtered.out
-	@echo "--> Running coverage html"
-	@go tool cover -html=coverage-filtered.out -o coverage.html
+	@echo "--> Creating GOCOVERDIR"
+	@mkdir -p ${COV_UNIT_E2E} ${COV_SIMULATION}
+	@echo "--> Cleaning up coverage files, if any"
+	@rm -rf ${COV_UNIT_E2E}/* ${COV_SIMULATION}/*
+	@echo "--> Building instrumented simulation test binary"
+	@go test -c ./app -mod=readonly -covermode=atomic -coverpkg=${COV_PKG} -cover -o ${COV_SIM_CMD}
+	@echo "  --> Running Full App Simulation"
+	@${COV_SIM_CMD} -test.run TestFullAppSimulation ${COV_SIM_COMMON} > /dev/null 2>&1
+#   Enable after wiring v2 is implemented
+#	@echo "  --> Running App Import/Export Simulation"
+#	@${COV_SIM_CMD} -test.run TestAppImportExport ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "  --> Running App Simulation After Import"
+	@${COV_SIM_CMD} -test.run TestAppSimulationAfterImport ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "  --> Running App State Determinism Simulation"
+	@${COV_SIM_CMD} -test.run TestAppStateDeterminism ${COV_SIM_COMMON} > /dev/null 2>&1
+	@echo "--> Running unit & e2e tests coverage"
+	@go test -timeout 30m -race -covermode=atomic -v -cpu=$$(nproc) -cover $$(go list ./...) ./interchaintest/... -coverpkg=${COV_PKG} -args -test.gocoverdir="${COV_UNIT_E2E}" > /dev/null 2>&1
+	@echo "--> Merging coverage reports"
+	@go tool covdata merge -i=${COV_UNIT_E2E},${COV_SIMULATION} -o ${COV_ROOT}
+	@echo "--> Converting binary coverage report to text format"
+	@go tool covdata textfmt -i=${COV_ROOT} -o ${COV_ROOT}/coverage-merged.out
+	@echo "--> Filtering coverage reports"
+	@./scripts/filter-coverage.sh ${COV_ROOT}/coverage-merged.out ${COV_ROOT}/coverage-merged-filtered.out
+	@echo "--> Generating coverage report"
+	@go tool cover -func=${COV_ROOT}/coverage-merged-filtered.out
+	@echo "--> Generating HTML coverage report"
+	@go tool cover -html=${COV_ROOT}/coverage-merged-filtered.out -o coverage.html
 	@echo "--> Coverage report available at coverage.html"
 	@echo "--> Cleaning up coverage files"
-	@rm coverage.out
+	@rm -rf ${COV_UNIT_E2E}/* ${COV_SIMULATION}/*
 	@echo "--> Running coverage complete"
 
 .PHONY: test test-integration coverage
@@ -189,6 +215,49 @@ lint-fix:
 .PHONY: lint lint-fix
 
 ##################
+### Simulation ###
+##################
+
+SIM_NUM_BLOCKS ?= 100
+SIM_PERIOD ?= 5
+SIM_COMMIT ?= true
+SIM_ENABLED ?= true
+SIM_VERBOSE ?= false
+SIM_TIMEOUT ?= 24h
+SIM_SEED ?= 42
+SIM_COMMON_ARGS = -NumBlocks=${SIM_NUM_BLOCKS} -Enabled=${SIM_ENABLED} -Commit=${SIM_COMMIT} -Period=${SIM_PERIOD} -Verbose=${SIM_VERBOSE} -Seed=${SIM_SEED} -v -timeout ${SIM_TIMEOUT}
+
+sim-full-app:
+	@echo "--> Running full app simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}"
+	@go test ./app -run TestFullAppSimulation ${SIM_COMMON_ARGS}
+
+sim-full-app-random:
+	$(MAKE) sim-full-app SIM_SEED=$$RANDOM
+
+# Note: known to fail when using app wiring v1
+sim-import-export:
+	@echo "--> Running app import/export simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}"
+	@go test ./app -run TestAppImportExport ${SIM_COMMON_ARGS}
+
+# Note: known to fail when using app wiring v1
+sim-import-export-random:
+	$(MAKE) sim-import-export SIM_SEED=$$RANDOM
+
+sim-after-import:
+	@echo "--> Running app after import simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}"
+	@go test ./app -run TestAppSimulationAfterImport ${SIM_COMMON_ARGS}
+
+sim-after-import-random:
+	$(MAKE) sim-after-import SIM_SEED=$$RANDOM
+
+sim-app-determinism:
+	@echo "--> Running app determinism simulation (blocks: ${SIM_NUM_BLOCKS}, commit: ${SIM_COMMIT}, period: ${SIM_PERIOD}, seed: ${SIM_SEED}"
+	@go test ./app -run TestAppStateDeterminism ${SIM_COMMON_ARGS}
+
+sim-app-determinism-random:
+	$(MAKE) sim-app-determinism SIM_SEED=$$RANDOM
+
+.PHONY: sim-full-app sim-full-app-random sim-import-export sim-after-import sim-app-determinism sim-import-export-random sim-after-import-random sim-app-determinism-random
 ###  Security  ###
 ##################
 govulncheck_version=latest