diff --git a/.circleci/config.yml b/.circleci/config.yml index 91a3e1d816..e186648ea9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,10 +15,11 @@ jobs: - go/mod-download - go/save-cache - run: {command: "go build ."} + - run: {command: "go build junit/junitformatter.go"} # Store the executable. - persist_to_workspace: root: . - paths: ["hive"] + paths: ["hive", "junitformatter"] # The below job runs the optimism test simulations. This requires a virtual # machine instead of the container-based build environment because hive needs @@ -41,16 +42,26 @@ jobs: -sim=<> \ -sim.loglevel=5 \ -docker.pull=true \ - -client=go-ethereum,op-geth_optimism,op-proposer_develop,op-batcher_develop,op-node_develop |& tee /tmp/build/hive.log || echo "failed." + -client=go-ethereum,op-geth_optimism,op-proposer_develop,op-batcher_develop,op-node_develop |& tee /tmp/build/hive.log - run: command: | tar -cvf /tmp/workspace.tgz -C /home/circleci/project /home/circleci/project/workspace name: "Archive workspace" + when: always - store_artifacts: path: /tmp/workspace.tgz destination: hive-workspace.tgz + when: always - run: - command: "! grep 'pass.*=false' /tmp/build/hive.log" + command: | + /tmp/build/junitformatter /home/circleci/project/workspace/logs/*.json > /home/circleci/project/workspace/logs/junit.xml + when: always + - store_test_results: + path: /home/circleci/project/workspace/logs/junit.xml + when: always + - store_artifacts: + path: /home/circleci/project/workspace/logs/junit.xml + when: always - slack/notify: channel: C03N11M0BBN branch_pattern: optimism diff --git a/.gitignore b/.gitignore index 71fa6671ee..f456d579bb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ workspace .idea/ # build output /hive +/junitformatter diff --git a/junit/junitformatter.go b/junit/junitformatter.go new file mode 100644 index 0000000000..12f78049ed --- /dev/null +++ b/junit/junitformatter.go @@ -0,0 +1,155 @@ +package main + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "os" + "strconv" + + "github.com/ethereum/hive/internal/libhive" +) + +func main() { + if len(os.Args) <= 1 { + fail(errors.New("no input files specified")) + } + + result := TestSuites{ + Failures: 0, + Name: "Hive Results", + Tests: 0, + } + var suites []TestSuite + + for i := 1; i < len(os.Args); i++ { + suite, err := readInput(os.Args[i]) + if err != nil { + fail(err) + } + junitSuite := mapTestSuite(suite) + result.Failures = result.Failures + junitSuite.Failures + result.Tests = result.Tests + junitSuite.Tests + suites = append(suites, junitSuite) + } + result.Suites = suites + + junit, err := xml.MarshalIndent(result, "", " ") + if err != nil { + fail(err) + } + fmt.Println(string(junit)) +} + +func readInput(file string) (libhive.TestSuite, error) { + inData, err := os.ReadFile(file) + if err != nil { + return libhive.TestSuite{}, fmt.Errorf("failed to read file '%v': %w", file, err) + } + + var suite libhive.TestSuite + err = json.Unmarshal(inData, &suite) + if err != nil { + return libhive.TestSuite{}, fmt.Errorf("failed to parse file '%v': %w", file, err) + } + return suite, nil +} + +func mapTestSuite(suite libhive.TestSuite) TestSuite { + junitSuite := TestSuite{ + Name: suite.Name, + Failures: 0, + Tests: len(suite.TestCases), + Properties: Properties{}, + } + for clientName, clientVersion := range suite.ClientVersions { + junitSuite.Properties.Properties = append(junitSuite.Properties.Properties, Property{ + Name: clientName, + Value: clientVersion, + }) + } + for _, testCase := range suite.TestCases { + if !testCase.SummaryResult.Pass { + junitSuite.Failures = junitSuite.Failures + 1 + } + junitSuite.TestCases = append(junitSuite.TestCases, mapTestCase(testCase)) + } + return junitSuite +} + +func mapTestCase(source *libhive.TestCase) TestCase { + result := TestCase{ + Name: source.Name, + } + if source.SummaryResult.Pass { + result.SystemOut = source.SummaryResult.Details + } else { + result.Failure = &Failure{Message: source.SummaryResult.Details} + } + duration := source.End.Sub(source.Start) + result.Time = strconv.FormatFloat(duration.Seconds(), 'f', 6, 64) + return result +} + +func fail(reason error) { + fmt.Println(reason) + os.Exit(1) +} + +/* +Target XML format (lots of it being optional): + + + + + + + + + + + + + + + + +*/ + +type TestSuites struct { + XMLName string `xml:"testsuites,omitempty"` + Failures int `xml:"failures,attr"` + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Suites []TestSuite `xml:"testsuite"` +} + +type TestSuite struct { + Name string `xml:"name,attr"` + Failures int `xml:"failures,attr"` + Tests int `xml:"tests,attr"` + Properties Properties `xml:"properties,omitempty"` + TestCases []TestCase `xml:"testcase"` +} + +type TestCase struct { + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` + Failure *Failure `xml:"failure,omitempty"` + SystemOut string `xml:"system-out,omitempty"` +} + +type Failure struct { + Message string `xml:"message,attr"` +} + +type Properties struct { + Properties []Property `xml:"property,omitempty"` +} + +type Property struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +}