From 7c34200d74832a7de31503a8f883ae20c9740f83 Mon Sep 17 00:00:00 2001 From: lightclient Date: Thu, 21 Nov 2024 14:50:03 +0800 Subject: [PATCH 1/3] cmd/evm: unify staterunner and blockrunner more, add human-readable output, stateless cross-checking option --- cmd/evm/blockrunner.go | 78 ++++---- cmd/evm/compiler.go | 55 ------ cmd/evm/disasm.go | 55 ------ cmd/evm/eest.go | 49 +++++ cmd/evm/eofparse.go | 32 ++- cmd/evm/internal/compiler/compiler.go | 39 ---- cmd/evm/main.go | 272 +++++++++++++------------- cmd/evm/reporter.go | 87 ++++++++ cmd/evm/runner.go | 141 +++++++++---- cmd/evm/staterunner.go | 186 +++++++----------- core/blockchain.go | 2 +- core/stateless.go | 4 +- eth/catalyst/api.go | 3 +- 13 files changed, 519 insertions(+), 484 deletions(-) delete mode 100644 cmd/evm/compiler.go delete mode 100644 cmd/evm/disasm.go create mode 100644 cmd/evm/eest.go delete mode 100644 cmd/evm/internal/compiler/compiler.go create mode 100644 cmd/evm/reporter.go diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index d5cd8d8e3de2..b6b5f822989e 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -22,79 +22,87 @@ import ( "fmt" "os" "regexp" + "slices" "sort" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" ) -var RunFlag = &cli.StringFlag{ - Name: "run", - Value: ".*", - Usage: "Run only those tests matching the regular expression.", -} - var blockTestCommand = &cli.Command{ Action: blockTestCmd, Name: "blocktest", Usage: "Executes the given blockchain tests", - ArgsUsage: "", - Flags: []cli.Flag{RunFlag}, + ArgsUsage: "", + Flags: slices.Concat([]cli.Flag{ + DumpFlag, + HumanReadableFlag, + RunFlag, + WitnessCrossCheckFlag, + }, traceFlags), } func blockTestCmd(ctx *cli.Context) error { - if len(ctx.Args().First()) == 0 { - return errors.New("path-to-test argument required") + path := ctx.Args().First() + if len(path) == 0 { + return errors.New("path argument required") } - - var tracer *tracing.Hooks - // Configure the EVM logger - if ctx.Bool(MachineFlag.Name) { - tracer = logger.NewJSONLogger(&logger.Config{ - EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), - DisableStack: ctx.Bool(DisableStackFlag.Name), - DisableStorage: ctx.Bool(DisableStorageFlag.Name), - EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), - }, os.Stderr) + var ( + collected = collectJSONFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runBlockTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) } - // Load the test content from the input file - src, err := os.ReadFile(ctx.Args().First()) + report(ctx, results) + return nil +} + +func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { + src, err := os.ReadFile(fname) if err != nil { - return err + return nil, err } - var tests map[string]tests.BlockTest + var tests map[string]*tests.BlockTest if err = json.Unmarshal(src, &tests); err != nil { - return err + return nil, err } re, err := regexp.Compile(ctx.String(RunFlag.Name)) if err != nil { - return fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err) + return nil, fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err) } + tracer := tracerFromFlags(ctx) - // Run them in order + // Pull out keys to sort and ensure tests are run in order. var keys []string for key := range tests { keys = append(keys, key) } sort.Strings(keys) + + // Run all the tests. + var results []testResult for _, name := range keys { if !re.MatchString(name) { continue } - test := tests[name] - if err := test.Run(false, rawdb.HashScheme, false, tracer, func(res error, chain *core.BlockChain) { + result := &testResult{Name: name, Pass: true} + if err := tests[name].Run(false, rawdb.HashScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { if ctx.Bool(DumpFlag.Name) { - if state, _ := chain.State(); state != nil { - fmt.Println(string(state.Dump(nil))) + if s, _ := chain.State(); s != nil { + result.State = dump(s) } } }); err != nil { - return fmt.Errorf("test %v: %w", name, err) + result.Pass, result.Error = false, err.Error() } + results = append(results, *result) } - return nil + return results, nil } diff --git a/cmd/evm/compiler.go b/cmd/evm/compiler.go deleted file mode 100644 index c071834b5941..000000000000 --- a/cmd/evm/compiler.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "fmt" - "os" - - "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" - - "github.com/urfave/cli/v2" -) - -var compileCommand = &cli.Command{ - Action: compileCmd, - Name: "compile", - Usage: "Compiles easm source to evm binary", - ArgsUsage: "", -} - -func compileCmd(ctx *cli.Context) error { - debug := ctx.Bool(DebugFlag.Name) - - if len(ctx.Args().First()) == 0 { - return errors.New("filename required") - } - - fn := ctx.Args().First() - src, err := os.ReadFile(fn) - if err != nil { - return err - } - - bin, err := compiler.Compile(fn, src, debug) - if err != nil { - return err - } - fmt.Println(bin) - return nil -} diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go deleted file mode 100644 index b1f35cbaf512..000000000000 --- a/cmd/evm/disasm.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "fmt" - "os" - "strings" - - "github.com/ethereum/go-ethereum/core/asm" - "github.com/urfave/cli/v2" -) - -var disasmCommand = &cli.Command{ - Action: disasmCmd, - Name: "disasm", - Usage: "Disassembles evm binary", - ArgsUsage: "", -} - -func disasmCmd(ctx *cli.Context) error { - var in string - switch { - case len(ctx.Args().First()) > 0: - fn := ctx.Args().First() - input, err := os.ReadFile(fn) - if err != nil { - return err - } - in = string(input) - case ctx.IsSet(InputFlag.Name): - in = ctx.String(InputFlag.Name) - default: - return errors.New("missing filename or --input value") - } - - code := strings.TrimSpace(in) - fmt.Printf("%v\n", code) - return asm.PrintDisassembled(code) -} diff --git a/cmd/evm/eest.go b/cmd/evm/eest.go new file mode 100644 index 000000000000..43071a3e5d31 --- /dev/null +++ b/cmd/evm/eest.go @@ -0,0 +1,49 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import "regexp" + +// testMetadata provides more granular access to the test information encoded +// within its filename by the execution spec test (EEST). +type testMetadata struct { + fork string + module string // which python module gnerated the test, e.g. eip7702 + file string // exact file the test came from, e.g. test_gas.py + function string // func that created the test, e.g. test_valid_mcopy_operations + parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs +} + +// parseTestMetadata reads a test name and parses out more specific information +// about the test. +func parseTestMetadata(s string) *testMetadata { + var ( + pattern = `tests\/([^\/]+)\/([^\/]+)\/([^:]+)::([^[]+)\[fork_([^-\]]+)-[^-]+-(.+)\]` + re = regexp.MustCompile(pattern) + ) + match := re.FindStringSubmatch(s) + if len(match) == 0 { + return nil + } + return &testMetadata{ + fork: match[5], + module: match[2], + file: match[3], + function: match[4], + parameters: match[6], + } +} diff --git a/cmd/evm/eofparse.go b/cmd/evm/eofparse.go index 2122270942c3..df8581146a43 100644 --- a/cmd/evm/eofparse.go +++ b/cmd/evm/eofparse.go @@ -31,13 +31,41 @@ import ( "github.com/urfave/cli/v2" ) +var jt vm.JumpTable + +const initcode = "INITCODE" + func init() { jt = vm.NewPragueEOFInstructionSetForTesting() } var ( - jt vm.JumpTable - initcode = "INITCODE" + hexFlag = &cli.StringFlag{ + Name: "hex", + Usage: "Single container data parse and validation", + } + refTestFlag = &cli.StringFlag{ + Name: "test", + Usage: "Path to EOF validation reference test.", + } + eofParseCommand = &cli.Command{ + Name: "eofparse", + Aliases: []string{"eof"}, + Usage: "Parses hex eof container and returns validation errors (if any)", + Action: eofParseAction, + Flags: []cli.Flag{ + hexFlag, + refTestFlag, + }, + } + eofDumpCommand = &cli.Command{ + Name: "eofdump", + Usage: "Parses hex eof container and prints out human-readable representation of the container.", + Action: eofDumpAction, + Flags: []cli.Flag{ + hexFlag, + }, + } ) func eofParseAction(ctx *cli.Context) error { diff --git a/cmd/evm/internal/compiler/compiler.go b/cmd/evm/internal/compiler/compiler.go deleted file mode 100644 index 54981b669768..000000000000 --- a/cmd/evm/internal/compiler/compiler.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package compiler - -import ( - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/core/asm" -) - -func Compile(fn string, src []byte, debug bool) (string, error) { - compiler := asm.NewCompiler(debug) - compiler.Feed(asm.Lex(src, debug)) - - bin, compileErrors := compiler.Compile() - if len(compileErrors) > 0 { - // report errors - for _, err := range compileErrors { - fmt.Printf("%s:%v\n", fn, err) - } - return "", errors.New("compiling failed") - } - return bin, nil -} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 0d4471b8d544..398187244042 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -19,11 +19,14 @@ package main import ( "fmt" - "math/big" + "io/fs" "os" - "slices" + "path/filepath" "github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/flags" "github.com/urfave/cli/v2" @@ -33,122 +36,100 @@ import ( _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) +// Some other nice-to-haves: +// * accumulate traces into an object to bundle with test +// * write tx identifier for trace before hand (blocktest only) +// * combine blocktest and statetest runner logic using unified test interface + +const traceCategory = "TRACING" + var ( - DebugFlag = &cli.BoolFlag{ - Name: "debug", - Usage: "output full trace logs", - Category: flags.VMCategory, - } - StatDumpFlag = &cli.BoolFlag{ - Name: "statdump", - Usage: "displays stack and heap memory information", - Category: flags.VMCategory, - } - CodeFlag = &cli.StringFlag{ - Name: "code", - Usage: "EVM code", - Category: flags.VMCategory, - } - CodeFileFlag = &cli.StringFlag{ - Name: "codefile", - Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", - Category: flags.VMCategory, - } - GasFlag = &cli.Uint64Flag{ - Name: "gas", - Usage: "gas limit for the evm", - Value: 10000000000, - Category: flags.VMCategory, - } - PriceFlag = &flags.BigFlag{ - Name: "price", - Usage: "price set for the evm", - Value: new(big.Int), - Category: flags.VMCategory, - } - ValueFlag = &flags.BigFlag{ - Name: "value", - Usage: "value set for the evm", - Value: new(big.Int), - Category: flags.VMCategory, - } - DumpFlag = &cli.BoolFlag{ - Name: "dump", - Usage: "dumps the state after the run", - Category: flags.VMCategory, - } - InputFlag = &cli.StringFlag{ - Name: "input", - Usage: "input for the EVM", - Category: flags.VMCategory, - } - InputFileFlag = &cli.StringFlag{ - Name: "inputfile", - Usage: "file containing input for the EVM", - Category: flags.VMCategory, + // Test running flags. + RunFlag = &cli.StringFlag{ + Name: "run", + Value: ".*", + Usage: "Run only those tests matching the regular expression.", } BenchFlag = &cli.BoolFlag{ Name: "bench", Usage: "benchmark the execution", Category: flags.VMCategory, } - CreateFlag = &cli.BoolFlag{ - Name: "create", - Usage: "indicates the action should be create rather than call", - Category: flags.VMCategory, + WitnessCrossCheckFlag = &cli.BoolFlag{ + Name: "cross-check", + Aliases: []string{"xc"}, + Usage: "Cross-check stateful execution against stateless, verifying the witness generation.", } - GenesisFlag = &cli.StringFlag{ - Name: "prestate", - Usage: "JSON file with prestate (genesis) config", - Category: flags.VMCategory, + + // Debugging flags. + DumpFlag = &cli.BoolFlag{ + Name: "dump", + Usage: "dumps the state after the run", } - MachineFlag = &cli.BoolFlag{ - Name: "json", - Usage: "output trace logs in machine readable format (json)", - Category: flags.VMCategory, + HumanReadableFlag = &cli.BoolFlag{ + Name: "human", + Usage: "\"Human-readable\" output", } - SenderFlag = &cli.StringFlag{ - Name: "sender", - Usage: "The transaction origin", - Category: flags.VMCategory, + StatDumpFlag = &cli.BoolFlag{ + Name: "statdump", + Usage: "displays stack and heap memory information", } - ReceiverFlag = &cli.StringFlag{ - Name: "receiver", - Usage: "The transaction receiver (execution context)", - Category: flags.VMCategory, + + // Tracing flags. + TraceFlag = &cli.BoolFlag{ + Name: "trace", + Usage: "Enable tracing and output trace log.", + Category: traceCategory, + } + TraceFormatFlag = &cli.StringFlag{ + Name: "trace.format", + Usage: "Trace output format to use (struct|json)", + Value: "struct", + Category: traceCategory, } - DisableMemoryFlag = &cli.BoolFlag{ - Name: "nomemory", + TraceDisableMemoryFlag = &cli.BoolFlag{ + Name: "trace.nomemory", + Aliases: []string{"nomemory"}, Value: true, Usage: "disable memory output", - Category: flags.VMCategory, + Category: traceCategory, } - DisableStackFlag = &cli.BoolFlag{ - Name: "nostack", + TraceDisableStackFlag = &cli.BoolFlag{ + Name: "trace.nostack", + Aliases: []string{"nostack"}, Usage: "disable stack output", - Category: flags.VMCategory, + Category: traceCategory, } - DisableStorageFlag = &cli.BoolFlag{ - Name: "nostorage", + TraceDisableStorageFlag = &cli.BoolFlag{ + Name: "trace.nostorage", + Aliases: []string{"nostorage"}, Usage: "disable storage output", - Category: flags.VMCategory, + Category: traceCategory, } - DisableReturnDataFlag = &cli.BoolFlag{ - Name: "noreturndata", + TraceDisableReturnDataFlag = &cli.BoolFlag{ + Name: "trace.noreturndata", + Aliases: []string{"noreturndata"}, Value: true, Usage: "enable return data output", - Category: flags.VMCategory, + Category: traceCategory, } - refTestFlag = &cli.StringFlag{ - Name: "test", - Usage: "Path to EOF validation reference test.", + + // Deprecated flags. + DebugFlag = &cli.BoolFlag{ + Name: "debug", + Usage: "output full trace logs (deprecated)", + Hidden: true, + Category: traceCategory, } - hexFlag = &cli.StringFlag{ - Name: "hex", - Usage: "single container data parse and validation", + MachineFlag = &cli.BoolFlag{ + Name: "json", + Usage: "output trace logs in machine readable format, json (deprecated)", + Hidden: true, + Category: traceCategory, } ) +// Command definitions. var ( stateTransitionCommand = &cli.Command{ Name: "transition", @@ -175,7 +156,6 @@ var ( t8ntool.RewardFlag, }, } - transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, @@ -203,62 +183,27 @@ var ( t8ntool.SealCliqueFlag, }, } - eofParseCommand = &cli.Command{ - Name: "eofparse", - Aliases: []string{"eof"}, - Usage: "Parses hex eof container and returns validation errors (if any)", - Action: eofParseAction, - Flags: []cli.Flag{ - hexFlag, - refTestFlag, - }, - } - - eofDumpCommand = &cli.Command{ - Name: "eofdump", - Usage: "Parses hex eof container and prints out human-readable representation of the container.", - Action: eofDumpAction, - Flags: []cli.Flag{ - hexFlag, - }, - } ) -// vmFlags contains flags related to running the EVM. -var vmFlags = []cli.Flag{ - CodeFlag, - CodeFileFlag, - CreateFlag, - GasFlag, - PriceFlag, - ValueFlag, - InputFlag, - InputFileFlag, - GenesisFlag, - SenderFlag, - ReceiverFlag, -} - // traceFlags contains flags that configure tracing output. var traceFlags = []cli.Flag{ - BenchFlag, + TraceFlag, + TraceFormatFlag, + TraceDisableMemoryFlag, + TraceDisableStackFlag, + TraceDisableStorageFlag, + TraceDisableReturnDataFlag, + + // deprecated DebugFlag, - DumpFlag, MachineFlag, - StatDumpFlag, - DisableMemoryFlag, - DisableStackFlag, - DisableStorageFlag, - DisableReturnDataFlag, } var app = flags.NewApp("the evm command line interface") func init() { - app.Flags = slices.Concat(vmFlags, traceFlags, debug.Flags) + app.Flags = debug.Flags app.Commands = []*cli.Command{ - compileCommand, - disasmCommand, runCommand, blockTestCommand, stateTestCommand, @@ -280,11 +225,56 @@ func init() { func main() { if err := app.Run(os.Args); err != nil { - code := 1 - if ec, ok := err.(*t8ntool.NumberedError); ok { - code = ec.ExitCode() + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// tracerFromFlags parses the cli flags and returns the specified tracer. +func tracerFromFlags(ctx *cli.Context) *tracing.Hooks { + config := &logger.Config{ + EnableMemory: !ctx.Bool(TraceDisableMemoryFlag.Name), + DisableStack: ctx.Bool(TraceDisableStackFlag.Name), + DisableStorage: ctx.Bool(TraceDisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name), + } + switch { + case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "struct": + return logger.NewStructLogger(config).Hooks() + case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "json": + return logger.NewJSONLogger(config, os.Stderr) + case ctx.Bool(MachineFlag.Name): + return logger.NewJSONLogger(config, os.Stderr) + case ctx.Bool(DebugFlag.Name): + return logger.NewStructLogger(config).Hooks() + default: + return nil + } +} + +// collectJSONFiles walks the given path and accumulates all files with json +// extension. +func collectJSONFiles(path string) []string { + var out []string + err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err } + if !info.IsDir() && filepath.Ext(info.Name()) == ".json" { + out = append(out, path) + } + return nil + }) + if err != nil { fmt.Fprintln(os.Stderr, err) - os.Exit(code) } + return out +} + +// dump returns a state dump for the most current trie. +func dump(s *state.StateDB) *state.Dump { + root := s.IntermediateRoot(false) + cpy, _ := state.New(root, s.Database()) + dump := cpy.RawDump(nil) + return &dump } diff --git a/cmd/evm/reporter.go b/cmd/evm/reporter.go new file mode 100644 index 000000000000..f6249e184369 --- /dev/null +++ b/cmd/evm/reporter.go @@ -0,0 +1,87 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/urfave/cli/v2" +) + +const ( + PASS = "\033[32mPASS\033[0m" + FAIL = "\033[31mFAIL\033[0m" +) + +// testResult contains the execution status after running a state test, any +// error that might have occurred and a dump of the final state if requested. +type testResult struct { + Name string `json:"name"` + Pass bool `json:"pass"` + Root *common.Hash `json:"stateRoot,omitempty"` + Fork string `json:"fork"` + Error string `json:"error,omitempty"` + State *state.Dump `json:"state,omitempty"` + Stats *execStats `json:"benchStats,omitempty"` +} + +func (r testResult) String() string { + var status string + if r.Pass { + status = fmt.Sprintf("[%s]", PASS) + } else { + status = fmt.Sprintf("[%s]", FAIL) + } + info := r.Name + m := parseTestMetadata(r.Name) + if m != nil { + info = fmt.Sprintf("%s %s, param=%s", m.module, m.function, m.parameters) + } + var extra string + if !r.Pass { + extra = fmt.Sprintf(", err=%v, fork=%s", r.Error, r.Fork) + } + out := fmt.Sprintf("%s %s%s", status, info, extra) + if r.State != nil { + state, _ := json.MarshalIndent(r.State, "", " ") + out += "\n" + string(state) + } + return out +} + +// report prints the after-test summary. +func report(ctx *cli.Context, results []testResult) { + if ctx.Bool(HumanReadableFlag.Name) { + pass := 0 + for _, r := range results { + if r.Pass { + pass++ + } + } + for _, r := range results { + fmt.Println(r) + } + fmt.Println("--") + fmt.Printf("%d tests passed, %d tests failed.\n", pass, len(results)-pass) + return + } + out, _ := json.MarshalIndent(results, "", " ") + fmt.Println(string(out)) +} diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 8c382e625722..8e0fd7881f80 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -25,10 +25,10 @@ import ( "os" goruntime "runtime" "slices" + "strings" "testing" "time" - "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -51,14 +51,82 @@ var runCommand = &cli.Command{ Usage: "Run arbitrary evm binary", ArgsUsage: "", Description: `The run command runs arbitrary EVM code.`, - Flags: slices.Concat(vmFlags, traceFlags), + Flags: slices.Concat([]cli.Flag{ + BenchFlag, + CodeFileFlag, + CreateFlag, + GasFlag, + GenesisFlag, + InputFlag, + InputFileFlag, + PriceFlag, + ReceiverFlag, + SenderFlag, + ValueFlag, + StatDumpFlag, + }, traceFlags), } +var ( + CodeFileFlag = &cli.StringFlag{ + Name: "codefile", + Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", + Category: flags.VMCategory, + } + CreateFlag = &cli.BoolFlag{ + Name: "create", + Usage: "Indicates the action should be create rather than call", + Category: flags.VMCategory, + } + GasFlag = &cli.Uint64Flag{ + Name: "gas", + Usage: "Gas limit for the evm", + Value: 10000000000, + Category: flags.VMCategory, + } + GenesisFlag = &cli.StringFlag{ + Name: "prestate", + Usage: "JSON file with prestate (genesis) config", + Category: flags.VMCategory, + } + InputFlag = &cli.StringFlag{ + Name: "input", + Usage: "Input for the EVM", + Category: flags.VMCategory, + } + InputFileFlag = &cli.StringFlag{ + Name: "inputfile", + Usage: "File containing input for the EVM", + Category: flags.VMCategory, + } + PriceFlag = &flags.BigFlag{ + Name: "price", + Usage: "Price set for the evm", + Value: new(big.Int), + Category: flags.VMCategory, + } + ReceiverFlag = &cli.StringFlag{ + Name: "receiver", + Usage: "The transaction receiver (execution context)", + Category: flags.VMCategory, + } + SenderFlag = &cli.StringFlag{ + Name: "sender", + Usage: "The transaction origin", + Category: flags.VMCategory, + } + ValueFlag = &flags.BigFlag{ + Name: "value", + Usage: "Value set for the evm", + Value: new(big.Int), + Category: flags.VMCategory, + } +) + // readGenesis will read the given JSON format genesis file and return // the initialized Genesis structure func readGenesis(genesisPath string) *core.Genesis { // Make sure we have a valid genesis JSON - //genesisPath := ctx.Args().First() if len(genesisPath) == 0 { utils.Fatalf("Must supply path to genesis JSON file") } @@ -127,10 +195,10 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, exe func runCmd(ctx *cli.Context) error { logconfig := &logger.Config{ - EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), - DisableStack: ctx.Bool(DisableStackFlag.Name), - DisableStorage: ctx.Bool(DisableStorageFlag.Name), - EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), + EnableMemory: !ctx.Bool(TraceDisableMemoryFlag.Name), + DisableStack: ctx.Bool(TraceDisableStackFlag.Name), + DisableStorage: ctx.Bool(TraceDisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name), Debug: ctx.Bool(DebugFlag.Name), } @@ -187,48 +255,35 @@ func runCmd(ctx *cli.Context) error { var code []byte codeFileFlag := ctx.String(CodeFileFlag.Name) - codeFlag := ctx.String(CodeFlag.Name) + hexcode := ctx.Args().First() - // The '--code' or '--codefile' flag overrides code in state - if codeFileFlag != "" || codeFlag != "" { - var hexcode []byte - if codeFileFlag != "" { - var err error - // If - is specified, it means that code comes from stdin - if codeFileFlag == "-" { - //Try reading from stdin - if hexcode, err = io.ReadAll(os.Stdin); err != nil { - fmt.Printf("Could not load code from stdin: %v\n", err) - os.Exit(1) - } - } else { - // Codefile with hex assembly - if hexcode, err = os.ReadFile(codeFileFlag); err != nil { - fmt.Printf("Could not load code from file: %v\n", err) - os.Exit(1) - } - } - } else { - hexcode = []byte(codeFlag) - } - hexcode = bytes.TrimSpace(hexcode) - if len(hexcode)%2 != 0 { - fmt.Printf("Invalid input length for hex data (%d)\n", len(hexcode)) - os.Exit(1) - } - code = common.FromHex(string(hexcode)) - } else if fn := ctx.Args().First(); len(fn) > 0 { - // EASM-file to compile - src, err := os.ReadFile(fn) + // The '--codefile' flag overrides code in state + if codeFileFlag == "-" { + // If - is specified, it means that code comes from stdin + // Try reading from stdin + input, err := io.ReadAll(os.Stdin) if err != nil { - return err + fmt.Printf("Could not load code from stdin: %v\n", err) + os.Exit(1) } - bin, err := compiler.Compile(fn, src, false) + hexcode = string(input) + } else if codeFileFlag != "" { + // Codefile with hex assembly + input, err := os.ReadFile(codeFileFlag) if err != nil { - return err + fmt.Printf("Could not load code from file: %v\n", err) + os.Exit(1) } - code = common.Hex2Bytes(bin) + hexcode = string(input) + } + + hexcode = strings.TrimSpace(hexcode) + if len(hexcode)%2 != 0 { + fmt.Printf("Invalid input length for hex data (%d)\n", len(hexcode)) + os.Exit(1) } + code = common.FromHex(hexcode) + runtimeConfig := runtime.Config{ Origin: sender, State: statedb, diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index d0a0d3287cb5..79dacfef1b4f 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -21,12 +21,12 @@ import ( "encoding/json" "fmt" "os" + "regexp" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" @@ -35,157 +35,123 @@ import ( var ( forkFlag = &cli.StringFlag{ Name: "statetest.fork", - Usage: "The hard-fork to run the test against", + Usage: "Only run tests for the specified fork.", Category: flags.VMCategory, } idxFlag = &cli.IntFlag{ Name: "statetest.index", - Usage: "The index of the subtest to run", + Usage: "The index of the subtest to run.", Category: flags.VMCategory, Value: -1, // default to select all subtest indices } - testNameFlag = &cli.StringFlag{ - Name: "statetest.name", - Usage: "The name of the state test to run", - Category: flags.VMCategory, - } ) var stateTestCommand = &cli.Command{ Action: stateTestCmd, Name: "statetest", Usage: "Executes the given state tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", - Flags: []cli.Flag{ - forkFlag, - idxFlag, - testNameFlag, - }, -} - -// StatetestResult contains the execution status after running a state test, any -// error that might have occurred and a dump of the final state if requested. -type StatetestResult struct { - Name string `json:"name"` - Pass bool `json:"pass"` - Root *common.Hash `json:"stateRoot,omitempty"` - Fork string `json:"fork"` - Error string `json:"error,omitempty"` - State *state.Dump `json:"state,omitempty"` - BenchStats *execStats `json:"benchStats,omitempty"` + Flags: slices.Concat([]cli.Flag{ + DumpFlag, + HumanReadableFlag, + RunFlag, + }, traceFlags), } func stateTestCmd(ctx *cli.Context) error { - // Configure the EVM logger - config := &logger.Config{ - EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), - DisableStack: ctx.Bool(DisableStackFlag.Name), - DisableStorage: ctx.Bool(DisableStorageFlag.Name), - EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), - } - var cfg vm.Config - switch { - case ctx.Bool(MachineFlag.Name): - cfg.Tracer = logger.NewJSONLogger(config, os.Stderr) + path := ctx.Args().First() - case ctx.Bool(DebugFlag.Name): - cfg.Tracer = logger.NewStructLogger(config).Hooks() - } - // Load the test content from the input file - if len(ctx.Args().First()) != 0 { - return runStateTest(ctx, ctx.Args().First(), cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)) + // If path is provided, run the tests at that path. + if len(path) != 0 { + var ( + collected = collectJSONFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runStateTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) + } + report(ctx, results) + return nil } - // Read filenames from stdin and execute back-to-back + // Otherwise, read filenames from stdin and execute back-to-back. scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { fname := scanner.Text() if len(fname) == 0 { return nil } - if err := runStateTest(ctx, fname, cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)); err != nil { + results, err := runStateTest(ctx, fname) + if err != nil { return err } + report(ctx, results) } return nil } -type stateTestCase struct { - name string - test tests.StateTest - st tests.StateSubtest -} - -// collectMatchedSubtests returns test cases which match against provided filtering CLI parameters -func collectMatchedSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { - var res []stateTestCase - subtestName := ctx.String(testNameFlag.Name) - if subtestName != "" { - if subtest, ok := testsByName[subtestName]; ok { - testsByName := make(map[string]tests.StateTest) - testsByName[subtestName] = subtest - } - } - idx := ctx.Int(idxFlag.Name) - fork := ctx.String(forkFlag.Name) - - for key, test := range testsByName { - for _, st := range test.Subtests() { - if idx != -1 && st.Index != idx { - continue - } - if fork != "" && st.Fork != fork { - continue - } - res = append(res, stateTestCase{name: key, st: st, test: test}) - } - } - return res -} - // runStateTest loads the state-test given by fname, and executes the test. -func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, bench bool) error { +func runStateTest(ctx *cli.Context, fname string) ([]testResult, error) { src, err := os.ReadFile(fname) if err != nil { - return err + return nil, err } var testsByName map[string]tests.StateTest if err := json.Unmarshal(src, &testsByName); err != nil { - return err + return nil, fmt.Errorf("unable to read test file %s: %w", fname, err) } - matchingTests := collectMatchedSubtests(ctx, testsByName) + cfg := vm.Config{Tracer: tracerFromFlags(ctx)} + re, err := regexp.Compile(ctx.String(RunFlag.Name)) + if err != nil { + return nil, fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err) + } // Iterate over all the tests, run them and aggregate the results - var results []StatetestResult - for _, test := range matchingTests { - // Run the test and aggregate the result - result := &StatetestResult{Name: test.name, Fork: test.st.Fork, Pass: true} - test.test.Run(test.st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) { - var root common.Hash - if tstate.StateDB != nil { - root = tstate.StateDB.IntermediateRoot(false) - result.Root = &root - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) - if dump { // Dump any state to aid debugging - cpy, _ := state.New(root, tstate.StateDB.Database()) - dump := cpy.RawDump(nil) - result.State = &dump - } + results := make([]testResult, 0, len(testsByName)) + for key, test := range testsByName { + if !re.MatchString(key) { + continue + } + for i, st := range test.Subtests() { + if idx := ctx.Int(idxFlag.Name); idx != -1 && idx != i { + // If specific index requested, skip all tests that do not match. + continue } - if err != nil { - // Test failed, mark as so - result.Pass, result.Error = false, err.Error() + if fork := ctx.String(forkFlag.Name); fork != "" && st.Fork != fork { + // If specific fork requested, skip all tests that do not match. + continue } - }) - if bench { - _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { - _, _, gasUsed, _ := test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) - return nil, gasUsed, nil + // Run the test and aggregate the result + result := &testResult{Name: key, Fork: st.Fork, Pass: true} + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, state *tests.StateTestState) { + var root common.Hash + if state.StateDB != nil { + root = state.StateDB.IntermediateRoot(false) + result.Root = &root + // Dump any state to aid debugging. + if ctx.Bool(DumpFlag.Name) { + result.State = dump(state.StateDB) + } + } + // Collect bench stats if requested. + if ctx.Bool(BenchFlag.Name) { + _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { + _, _, gasUsed, _ := test.RunNoVerify(st, cfg, false, rawdb.HashScheme) + return nil, gasUsed, nil + }) + result.Stats = &stats + } + if err != nil { + // Test failed, mark as so. + result.Pass, result.Error = false, err.Error() + return + } }) - result.BenchStats = &stats + results = append(results, *result) } - results = append(results, *result) } - out, _ := json.MarshalIndent(results, "", " ") - fmt.Println(string(out)) - return nil + return results, nil } diff --git a/core/blockchain.go b/core/blockchain.go index c3da61b28108..0fe481262684 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1926,7 +1926,7 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s task := types.NewBlockWithHeader(context).WithBody(*block.Body()) // Run the stateless self-cross-validation - crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, task, witness) + crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, bc.vmConfig, task, witness) if err != nil { return nil, fmt.Errorf("stateless self-validation failed: %v", err) } diff --git a/core/stateless.go b/core/stateless.go index 5b37d5020ec3..d21a62b4a51c 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -40,7 +40,7 @@ import ( // - It cannot be placed outside of core, because it needs to construct a dud headerchain // // TODO(karalabe): Would be nice to resolve both issues above somehow and move it. -func ExecuteStateless(config *params.ChainConfig, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { +func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { // Sanity check if the supplied block accidentally contains a set root or // receipt hash. If so, be very loud, but still continue. if block.Root() != (common.Hash{}) { @@ -66,7 +66,7 @@ func ExecuteStateless(config *params.ChainConfig, block *types.Block, witness *s validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block // Run the stateless blocks processing and self-validate certain fields - res, err := processor.Process(block, db, vm.Config{}) + res, err := processor.Process(block, db, vmconfig) if err != nil { return common.Hash{}, common.Hash{}, err } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 4779f9756b00..fa9752734e64 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/internal/version" @@ -995,7 +996,7 @@ func (api *ConsensusAPI) executeStatelessPayload(params engine.ExecutableData, v api.lastNewPayloadLock.Unlock() log.Trace("Executing block statelessly", "number", block.Number(), "hash", params.BlockHash) - stateRoot, receiptRoot, err := core.ExecuteStateless(api.eth.BlockChain().Config(), block, witness) + stateRoot, receiptRoot, err := core.ExecuteStateless(api.eth.BlockChain().Config(), vm.Config{}, block, witness) if err != nil { log.Warn("ExecuteStatelessPayload: execution failed", "err", err) errorMsg := err.Error() From 092b99f49f352a317f7a9570d3257c203f997c1e Mon Sep 17 00:00:00 2001 From: lightclient Date: Thu, 21 Nov 2024 17:23:07 +0800 Subject: [PATCH 2/3] cmd/evm: review fixes --- cmd/evm/blockrunner.go | 9 +++------ cmd/evm/main.go | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index b6b5f822989e..2cb0531e281e 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -23,12 +23,12 @@ import ( "os" "regexp" "slices" - "sort" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" + "golang.org/x/exp/maps" ) var blockTestCommand = &cli.Command{ @@ -80,11 +80,8 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { tracer := tracerFromFlags(ctx) // Pull out keys to sort and ensure tests are run in order. - var keys []string - for key := range tests { - keys = append(keys, key) - } - sort.Strings(keys) + keys := maps.Keys(tests) + slices.Sort(keys) // Run all the tests. var results []testResult diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 398187244042..750b269906d9 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -189,8 +189,8 @@ var ( var traceFlags = []cli.Flag{ TraceFlag, TraceFormatFlag, - TraceDisableMemoryFlag, TraceDisableStackFlag, + TraceDisableMemoryFlag, TraceDisableStorageFlag, TraceDisableReturnDataFlag, From 2ae47cf2c9577ce5cf85a3afcc0d761d66e98294 Mon Sep 17 00:00:00 2001 From: lightclient Date: Thu, 21 Nov 2024 22:22:47 +0800 Subject: [PATCH 3/3] cmd/evm: print state root for jsonl format --- cmd/evm/staterunner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 79dacfef1b4f..323b7d60abc8 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -131,6 +131,7 @@ func runStateTest(ctx *cli.Context, fname string) ([]testResult, error) { if state.StateDB != nil { root = state.StateDB.IntermediateRoot(false) result.Root = &root + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) // Dump any state to aid debugging. if ctx.Bool(DumpFlag.Name) { result.State = dump(state.StateDB)