diff --git a/.github/workflows/automated-tests.yml b/.github/workflows/automated-tests.yml index ad53314e76..f7d9988bad 100644 --- a/.github/workflows/automated-tests.yml +++ b/.github/workflows/automated-tests.yml @@ -56,3 +56,17 @@ jobs: go-version: "1.20" - name: E2E tests run: make test-e2e-short-cometmock + Trace-Tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Checkout LFS objects + run: git lfs checkout + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.20" + - name: E2E tests + run: make test-trace diff --git a/.github/workflows/manual-e2e.yml b/.github/workflows/manual-e2e.yml index 3893d009df..ff3780bcde 100644 --- a/.github/workflows/manual-e2e.yml +++ b/.github/workflows/manual-e2e.yml @@ -20,7 +20,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E happy-path test - run: go run ./tests/e2e/... --tc happy-path + run: go run ./tests/e2e/... --tc happy-path::default changeover-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -36,7 +36,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E changeover test - run: go run ./tests/e2e/... --tc changeover + run: go run ./tests/e2e/... --tc changeover::changeover democracy-reward-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -52,7 +52,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E democracy-reward tests - run: go run ./tests/e2e/... --tc democracy-reward + run: go run ./tests/e2e/... --tc democracy-reward::democracy-reward democracy-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -68,7 +68,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E democracy tests - run: go run ./tests/e2e/... --tc democracy + run: go run ./tests/e2e/... --tc democracy::democracy slash-throttle-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -84,7 +84,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E slash-throttle tests - run: go run ./tests/e2e/... --tc slash-throttle + run: go run ./tests/e2e/... --tc slash-throttle::slash-throttle multiconsumer-test: runs-on: ubuntu-latest timeout-minutes: 40 @@ -100,4 +100,4 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E multi-consumer tests - run: go run ./tests/e2e/... --tc multiconsumer + run: go run ./tests/e2e/... --tc multiconsumer::multiconsumer diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 913f7e0bbe..39bb0ec37f 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -33,7 +33,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E happy-path test - run: go run ./tests/e2e/... --tc happy-path + run: go run ./tests/e2e/... --tc happy-path::default changeover-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -49,7 +49,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E changeover test - run: go run ./tests/e2e/... --tc changeover + run: go run ./tests/e2e/... --tc changeover::changeover democracy-reward-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -65,7 +65,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E democracy-reward tests - run: go run ./tests/e2e/... --tc democracy-reward + run: go run ./tests/e2e/... --tc democracy-reward::democracy-reward democracy-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -81,7 +81,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E democracy tests - run: go run ./tests/e2e/... --tc democracy + run: go run ./tests/e2e/... --tc democracy::democracy slash-throttle-test: runs-on: ubuntu-latest timeout-minutes: 20 @@ -97,7 +97,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E slash-throttle tests - run: go run ./tests/e2e/... --tc slash-throttle + run: go run ./tests/e2e/... --tc slash-throttle::slash-throttle multiconsumer-test: runs-on: ubuntu-latest timeout-minutes: 40 @@ -113,7 +113,7 @@ jobs: with: go-version: "1.20" # The Go version to download (if necessary) and use. - name: E2E multi-consumer tests - run: go run ./tests/e2e/... --tc multiconsumer + run: go run ./tests/e2e/... --tc multiconsumer::multiconsumer nightly-test-fail: needs: diff --git a/Makefile b/Makefile index a65db076ab..3f053cc9b8 100644 --- a/Makefile +++ b/Makefile @@ -31,12 +31,12 @@ test-diff: # run only happy path E2E tests test-e2e-short: - go run ./tests/e2e/... --tc happy-path + go run ./tests/e2e/... --tc happy-path::default # run only happy path E2E tests with cometmock # this set of traces does not test equivocation but it does check downtime test-e2e-short-cometmock: - go run ./tests/e2e/... --tc happy-path-short --use-cometmock --use-gorelayer + go run ./tests/e2e/... --tc happy-path-short::default --use-cometmock --use-gorelayer # run full E2E tests in sequence (including multiconsumer) test-e2e-multi-consumer: @@ -52,7 +52,7 @@ test-gaia-e2e: # run only happy path E2E tests using latest tagged gaia test-gaia-e2e-short: - go run ./tests/e2e/... --tc happy-path --use-gaia + go run ./tests/e2e/... --tc happy-path::default --use-gaia # run full E2E tests in parallel (including multiconsumer) using latest tagged gaia test-gaia-e2e-parallel: @@ -66,7 +66,7 @@ test-gaia-e2e-tagged: # run only happy path E2E tests using latest tagged gaia # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-short-tagged test-gaia-e2e-short-tagged: - go run ./tests/e2e/... --tc happy-path --use-gaia --gaia-tag $(GAIA_TAG) + go run ./tests/e2e/... --tc happy-path::default --use-gaia --gaia-tag $(GAIA_TAG) # run full E2E tests in parallel (including multiconsumer) using specific tagged version of gaia # usage: GAIA_TAG=v9.0.0 make test-gaia-e2e-parallel-tagged @@ -77,6 +77,10 @@ test-gaia-e2e-parallel-tagged: test-no-cache: go test ./... -count=1 && go run ./tests/e2e/... +# test reading a trace from a file +test-trace: + go run ./tests/e2e/... --test-file tests/e2e/tracehandler_testdata/happyPath.json::default + ############################################################################### ### Linting ### ############################################################################### @@ -97,8 +101,6 @@ mockgen_cmd=go run github.com/golang/mock/mockgen mocks: $(mockgen_cmd) -package=keeper -destination=testutil/keeper/mocks.go -source=x/ccv/types/expected_keepers.go - -BUILDDIR ?= $(CURDIR)/build BUILD_TARGETS := build build: BUILD_ARGS=-o $(BUILDDIR)/ diff --git a/tests/e2e/main.go b/tests/e2e/main.go index 0a376664a0..b2d3ef8baf 100644 --- a/tests/e2e/main.go +++ b/tests/e2e/main.go @@ -13,6 +13,7 @@ import ( "time" "github.com/kylelemons/godebug/pretty" + "golang.org/x/exp/slices" ) // The list of test cases to be executed @@ -49,29 +50,71 @@ var ( ) var ( - testSelection TestSet - testMap map[string]*testRunWithSteps = map[string]*testRunWithSteps{ - "happy-path-short": { - testRun: DefaultTestRun(), steps: shortHappyPathSteps, - description: `This is like the happy path, but skips steps -that involve starting or stopping nodes for the same chain outside of the chain setup or teardown. -This is suited for CometMock+Gorelayer testing`, - }, - "light-client-attack": { - testRun: DefaultTestRun(), steps: lightClientAttackSteps, - description: `This is like the short happy path, but will slash validators for LightClientAttackEvidence instead of DuplicateVoteEvidence. -This is suited for CometMock+Gorelayer testing, but currently does not work with CometBFT, -since causing light client attacks is not implemented.`, - }, - "happy-path": {testRun: DefaultTestRun(), steps: happyPathSteps, description: "happy path tests"}, - "changeover": {testRun: ChangeoverTestRun(), steps: changeoverSteps, description: "changeover tests"}, - "democracy-reward": {testRun: DemocracyTestRun(true), steps: democracyRewardsSteps, description: "democracy tests allowing rewards"}, - "democracy": {testRun: DemocracyTestRun(false), steps: democracySteps, description: "democracy tests"}, - "slash-throttle": {testRun: SlashThrottleTestRun(), steps: slashThrottleSteps, description: "slash throttle tests"}, - "multiconsumer": {testRun: MultiConsumerTestRun(), steps: multipleConsumers, description: "multi consumer tests"}, + selectedTests TestSet + testRuns = map[string]TestRunChoice{ + "default": {name: "default", testRun: DefaultTestRun(), description: "default test run"}, + "changeover": {name: "changeover", testRun: ChangeoverTestRun(), description: "changeover test run"}, + "democracy": {name: "democracy", testRun: DemocracyTestRun(false), description: "democracy test run"}, + "democracy-reward": {name: "democracy-reward", testRun: DemocracyTestRun(true), description: "democracy test run with rewards"}, + "slash-throttle": {name: "slash-throttle", testRun: SlashThrottleTestRun(), description: "slash throttle test run"}, + "multiconsumer": {name: "multiconsumer", testRun: MultiConsumerTestRun(), description: "multi consumer test run"}, } + // helper function to get the test run choices by matching test runs ) +var selectedTestfiles TestSet + +var stepChoices = map[string]StepChoice{ + "happy-path-short": { + name: "happy-path-short", + steps: shortHappyPathSteps, + description: `This is like the happy path, but skips steps that involve starting or stopping nodes for the same chain outside of the chain setup or teardown. This is suited for CometMock+Gorelayer testing`, + testRuns: []string{"default"}, + }, + "light-client-attack": { + name: "light-client-attack", + steps: lightClientAttackSteps, + description: `This is like the short happy path, but will slash validators for LightClientAttackEvidence instead of DuplicateVoteEvidence. This is suited for CometMock+Gorelayer testing, but currently does not work with CometBFT, since causing light client attacks is not implemented`, + testRuns: []string{"default"}, + }, + "happy-path": { + name: "happy-path", + steps: happyPathSteps, + description: "happy path tests", + testRuns: []string{"default"}, + }, + "changeover": { + name: "changeover", + steps: changeoverSteps, + description: "changeover tests", + testRuns: []string{"changeover"}, + }, + "democracy-reward": { + name: "democracy-reward", + steps: democracyRewardsSteps, + description: "democracy tests allowing rewards", + testRuns: []string{"democracy-reward"}, + }, + "democracy": { + name: "democracy", + steps: democracySteps, + description: "democracy tests", + testRuns: []string{"democracy"}, + }, + "slash-throttle": { + name: "slash-throttle", + steps: slashThrottleSteps, + description: "slash throttle tests", + testRuns: []string{"slash-throttle"}, + }, + "multiconsumer": { + name: "multiconsumer", + steps: multipleConsumers, + description: "multi consumer tests", + testRuns: []string{"multiconsumer"}, + }, +} + func executeTests(tests []testRunWithSteps) (err error) { if parallel != nil && *parallel { fmt.Println("=============== running all tests in parallel ===============") @@ -97,57 +140,147 @@ func executeTests(tests []testRunWithSteps) (err error) { return } +func getTestCaseUsageString() string { + var builder strings.Builder + + // Test case selection + builder.WriteString("This flag is used to reference existing, defined test cases to be run.") + builder.WriteString("Test case selection:\nSelection of test steps to be executed:\n") + for _, stepChoice := range stepChoices { + builder.WriteString(fmt.Sprintf("- %s : %s. Compatible with test runners: %s\n", stepChoice.name, stepChoice.description, strings.Join(stepChoice.testRuns, ","))) + } + builder.WriteString("\n") + + // Test runner selection + builder.WriteString("Test runner selection:\nSelection of test runners to be executed:\n") + for _, testRunChoice := range testRuns { + builder.WriteString(fmt.Sprintf("- %s : %s\n", testRunChoice.name, testRunChoice.description)) + } + builder.WriteString("\n") + + // Example + builder.WriteString("Example: -tc multiconsumer::multiconsumer -tc happy-path::default") + + return builder.String() +} + +func getTestFileUsageString() string { + var builder strings.Builder + + builder.WriteString("This flag is used to reference files containing step traces to be run.\n") + builder.WriteString("Each filename should be separated by '::' from the test runner name.\n") + + // Test runner selection + builder.WriteString("Test runner selection:\nSelection of test runners to be executed:\n") + for _, testRunChoice := range testRuns { + builder.WriteString(fmt.Sprintf("- %s : %s\n", testRunChoice.name, testRunChoice.description)) + } + builder.WriteString("\n") + + // Example + builder.WriteString("Example: -test-file awesome-trace.json::default -test-file other-trace.json::default") + + return builder.String() +} + func parseArguments() (err error) { - flag.Var(&testSelection, "tc", - fmt.Sprintf("Selection of test cases to be executed:\n%s,\n%s", - func() string { - var keys []string - for k, v := range testMap { - keys = append(keys, fmt.Sprintf("- %s : %s", k, v.description)) - } - return strings.Join(keys, "\n") - }(), - "Example: -tc multiconsumer -tc happy-path ")) + flag.Var(&selectedTests, "tc", + getTestCaseUsageString()) + + flag.Var(&selectedTestfiles, "test-file", + getTestFileUsageString()) flag.Parse() // Enforce go-relayer in case of cometmock as hermes is not yet supported if useCometmock != nil && *useCometmock && (useGorelayer == nil || !*useGorelayer) { fmt.Println("Enforcing go-relayer as cometmock is requested") if err = flag.Set("use-gorelayer", "true"); err != nil { - return - } - } - // check if specified test case exists - for _, tc := range testSelection { - if _, hasKey := testMap[tc]; !hasKey { - err := fmt.Errorf("unknown test case '%s'", tc) return err } } - return + return nil } -func getTestCases(selection TestSet) (tests []testRunWithSteps) { +type testRunWithSteps struct { + testRun TestRun + steps []Step +} + +func getTestCases(selectedPredefinedTests, selectedTestFiles TestSet) (tests []testRunWithSteps) { // Run default tests if no test cases were selected - if len(selection) == 0 { - selection = TestSet{ - "changeover", "happy-path", - "democracy-reward", "democracy", "slash-throttle", + if len(selectedPredefinedTests) == 0 && len(selectedTestFiles) == 0 { + selectedPredefinedTests = TestSet{ + "changeover::changeover", "happy-path::default", + "democracy-reward::democracy-reward", "democracy::democracy", "slash-throttle::slash-throttle", } if includeMultiConsumer != nil && *includeMultiConsumer { - selection = append(selection, "multiconsumer") + selectedPredefinedTests = append(selectedPredefinedTests, "multiconsumer::multiconsumer") } } - // Get tests from selection tests = []testRunWithSteps{} - for _, tc := range selection { - if _, exists := testMap[tc]; !exists { - log.Fatalf("Test case '%s' not found", tc) + // Get predefined from selection + for _, tc := range selectedPredefinedTests { + // first part of tc is the steps, second part is the test runner + splitTcString := strings.Split(tc, "::") + if len(splitTcString) != 2 { + log.Fatalf("Test case '%s' is invalid.\nsee usage info:\n%s", tc, getTestCaseUsageString()) } - tests = append(tests, *testMap[tc]) + stepsName := splitTcString[0] + testRunnerName := splitTcString[1] + + if _, exists := stepChoices[stepsName]; !exists { + log.Fatalf("Step choice '%s' not found.\nsee usage info:\n%s", tc, getTestCaseUsageString()) + } + + stepChoice := stepChoices[stepsName] + + if _, exists := testRuns[testRunnerName]; !exists { + log.Fatalf("Test runner '%s' not found.\nsee usage info:\n%s", testRunnerName, getTestCaseUsageString()) + } + + testRunChoice := testRuns[testRunnerName] + + if !slices.Contains(stepChoice.testRuns, testRunChoice.name) { + log.Fatalf("Step choice '%s' is not compatible with test runner '%s'. compatible test runs: %s", stepsName, testRunnerName, strings.Join(stepChoice.testRuns, ",")) + } + + tests = append(tests, testRunWithSteps{ + testRun: testRunChoice.testRun, + steps: stepChoice.steps, + }, + ) } - return + + // get test cases from files + for _, testFile := range selectedTestFiles { + // first part is the file, second part is the test runner + splitTcString := strings.Split(testFile, "::") + if len(splitTcString) != 2 { + log.Fatalf("Test file '%s' is invalid.\nsee usage info:\n%s", testFile, getTestFileUsageString()) + } + + testFileName := splitTcString[0] + testRunnerName := splitTcString[1] + + if _, exists := testRuns[testRunnerName]; !exists { + log.Fatalf("Test runner '%s' not found.\nsee usage info:\n%s", testRunnerName, getTestFileUsageString()) + } + + testRunChoice := testRuns[testRunnerName] + + testCase, err := GlobalJSONParser.ReadTraceFromFile(testFileName) + if err != nil { + log.Fatalf("Error reading test file '%s': %s", testFileName, err) + } + + tests = append(tests, testRunWithSteps{ + testRun: testRunChoice.testRun, + steps: testCase, + }) + } + + return tests } // runs E2E tests @@ -159,7 +292,7 @@ func main() { log.Fatalf("Error parsing command arguments %s\n", err) } - testCases := getTestCases(testSelection) + testCases := getTestCases(selectedTests, selectedTestfiles) start := time.Now() err := executeTests(testCases) @@ -182,10 +315,18 @@ func (tr *TestRun) Run(steps []Step, localSdkPath string, useGaia bool, gaiaTag tr.teardownDocker() } -type testRunWithSteps struct { - testRun TestRun +type StepChoice struct { + name string steps []Step description string + // contains the names of the test runs that are compatible with this step choice + testRuns []string +} + +type TestRunChoice struct { + name string + testRun TestRun + description string } func (tr *TestRun) runStep(step Step, verbose bool) {