diff --git a/docs/cli/test.md b/docs/cli/test.md index 3a9ec639a5d930..8ff796f5b02d05 100644 --- a/docs/cli/test.md +++ b/docs/cli/test.md @@ -55,6 +55,49 @@ $ bun test ./test/specific-file.test.ts The test runner runs all tests in a single process. It loads all `--preload` scripts (see [Lifecycle](https://bun.sh/docs/test/lifecycle) for details), then runs all tests. If a test fails, the test runner will exit with a non-zero exit code. +## CI/CD integration + +`bun test` supports a variety of CI/CD integrations. + +### GitHub Actions + +`bun test` automatically detects if it's running inside GitHub Actions and will emit GitHub Actions annotations to the console directly. + +No configuration is needed, other than installing `bun` in the workflow and running `bun test`. + +#### How to install `bun` in a GitHub Actions workflow + +To use `bun test` in a GitHub Actions workflow, add the following step: + +```yaml +jobs: + build: + name: build-app + runs-on: ubuntu-latest + steps: + - name: Install bun + uses: oven-sh/setup-bun + - name: Install dependencies # (assuming your project has dependencies) + run: bun install # You can use npm/yarn/pnpm instead if you prefer + - name: Run tests + run: bun test +``` + +From there, you'll get GitHub Actions annotations. + +### JUnit XML reports (GitLab, etc.) + +To use `bun test` with a JUnit XML reporter, you can use the `--reporter=junit` in combination with `--reporter-outfile`. + +```sh +$ bun test --reporter=junit --reporter-outfile=./bun.xml +``` + +This will continue to output to stdout/stderr as usual, and also write a JUnit +XML report to the given path at the very end of the test run. + +JUnit XML is a popular format for reporting test results in CI/CD pipelines. + ## Timeouts Use the `--timeout` flag to specify a _per-test_ timeout in milliseconds. If a test times out, it will be marked as failed. The default value is `5000`. diff --git a/src/bunfig.zig b/src/bunfig.zig index 4f1d73d68c3162..f5e41c434deb0e 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -266,6 +266,17 @@ pub const Bunfig = struct { this.ctx.test_options.coverage.enabled = expr.data.e_boolean.value; } + if (test_.get("reporter")) |expr| { + try this.expect(expr, .e_object); + if (expr.get("junit")) |junit_expr| { + try this.expectString(junit_expr); + if (junit_expr.data.e_string.len() > 0) { + this.ctx.test_options.file_reporter = .junit; + this.ctx.test_options.reporter_outfile = try junit_expr.data.e_string.string(allocator); + } + } + } + if (test_.get("coverageReporter")) |expr| brk: { this.ctx.test_options.coverage.reporters = .{ .text = false, .lcov = false }; if (expr.data == .e_string) { diff --git a/src/cli.zig b/src/cli.zig index f76416230c74ee..a28b799b5e36ce 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -306,6 +306,8 @@ pub const Arguments = struct { clap.parseParam("--coverage-dir Directory for coverage files. Defaults to 'coverage'.") catch unreachable, clap.parseParam("--bail ? Exit the test suite after failures. If you do not specify a number, it defaults to 1.") catch unreachable, clap.parseParam("-t, --test-name-pattern Run only tests with a name that matches the given regex.") catch unreachable, + clap.parseParam("--reporter Specify the test reporter. Currently --reporter=junit is the only supported format.") catch unreachable, + clap.parseParam("--reporter-outfile The output file used for the format from --reporter.") catch unreachable, }; pub const test_params = test_only_params ++ runtime_params_ ++ transpiler_params_ ++ base_params_; @@ -524,6 +526,23 @@ pub const Arguments = struct { } } + if (args.option("--reporter-outfile")) |reporter_outfile| { + ctx.test_options.reporter_outfile = reporter_outfile; + } + + if (args.option("--reporter")) |reporter| { + if (strings.eqlComptime(reporter, "junit")) { + if (ctx.test_options.reporter_outfile == null) { + Output.errGeneric("--reporter=junit expects an output file from --reporter-outfile", .{}); + Global.crash(); + } + ctx.test_options.file_reporter = .junit; + } else { + Output.errGeneric("unrecognized reporter format: '{s}'. Currently, only 'junit' is supported", .{reporter}); + Global.crash(); + } + } + if (args.option("--coverage-dir")) |dir| { ctx.test_options.coverage.reports_directory = dir; } @@ -1354,6 +1373,9 @@ pub const Command = struct { bail: u32 = 0, coverage: TestCommand.CodeCoverageOptions = .{}, test_filter_regex: ?*RegularExpression = null, + + file_reporter: ?TestCommand.FileReporter = null, + reporter_outfile: ?[]const u8 = null, }; pub const Debugger = union(enum) { diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index 451f1e2a529269..cfe25b04792c56 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -9,6 +9,7 @@ const stringZ = bun.stringZ; const default_allocator = bun.default_allocator; const C = bun.C; const std = @import("std"); +const OOM = bun.OOM; const lex = bun.js_lexer; const logger = bun.logger; @@ -45,6 +46,44 @@ const Test = TestRunner.Test; const CodeCoverageReport = bun.sourcemap.CodeCoverageReport; const uws = bun.uws; +fn escapeXml(str: string, writer: anytype) !void { + var last: usize = 0; + var i: usize = 0; + const len = str.len; + while (i < len) : (i += 1) { + const c = str[i]; + switch (c) { + '&', + '<', + '>', + '"', + '\'', + => { + if (i > last) { + try writer.writeAll(str[last..i]); + } + const escaped = switch (c) { + '&' => "&", + '<' => "<", + '>' => ">", + '"' => """, + '\'' => "'", + else => unreachable, + }; + try writer.writeAll(escaped); + last = i + 1; + }, + 0...0x1f => { + // Escape all control characters + try writer.print("&#{d};", .{c}); + }, + else => {}, + } + } + if (len > last) { + try writer.writeAll(str[last..]); + } +} fn fmtStatusTextLine(comptime status: @Type(.EnumLiteral), comptime emoji_or_color: bool) []const u8 { comptime { // emoji and color might be split into two different options in the future @@ -76,6 +115,357 @@ fn writeTestStatusLine(comptime status: @Type(.EnumLiteral), writer: anytype) vo writer.print(fmtStatusTextLine(status, false), .{}) catch unreachable; } +// Remaining TODOs: +// - Add stdout/stderr to the JUnit report +// - Add timestamp field to the JUnit report +pub const JunitReporter = struct { + contents: std.ArrayListUnmanaged(u8) = .{}, + total_metrics: Metrics = .{}, + testcases_metrics: Metrics = .{}, + offset_of_testsuites_value: usize = 0, + offset_of_testsuite_value: usize = 0, + current_file: string = "", + properties_list_to_repeat_in_every_test_suite: ?[]const u8 = null, + + hostname_value: ?string = null, + + pub fn getHostname(this: *JunitReporter) ?string { + if (this.hostname_value == null) { + if (Environment.isWindows) { + return null; + } + + var name_buffer: [bun.HOST_NAME_MAX]u8 = undefined; + const hostname = std.posix.gethostname(&name_buffer) catch { + this.hostname_value = ""; + return null; + }; + + var arraylist_writer = std.ArrayList(u8).init(bun.default_allocator); + escapeXml(hostname, arraylist_writer.writer()) catch { + this.hostname_value = ""; + return null; + }; + this.hostname_value = arraylist_writer.items; + } + + if (this.hostname_value) |hostname| { + if (hostname.len > 0) { + return hostname; + } + } + return null; + } + + const Metrics = struct { + test_cases: u32 = 0, + assertions: u32 = 0, + failures: u32 = 0, + skipped: u32 = 0, + elapsed_time: u64 = 0, + + pub fn add(this: *Metrics, other: *const Metrics) void { + this.test_cases += other.test_cases; + this.assertions += other.assertions; + this.failures += other.failures; + this.skipped += other.skipped; + } + }; + pub fn init() *JunitReporter { + return JunitReporter.new( + .{ .contents = .{}, .total_metrics = .{} }, + ); + } + + pub usingnamespace bun.New(JunitReporter); + + fn generatePropertiesList(this: *JunitReporter) !void { + const PropertiesList = struct { + ci: string, + commit: string, + }; + var arena = std.heap.ArenaAllocator.init(bun.default_allocator); + defer arena.deinit(); + var stack = std.heap.stackFallback(1024, arena.allocator()); + const allocator = stack.get(); + + const properties: PropertiesList = .{ + .ci = brk: { + if (bun.getenvZ("GITHUB_RUN_ID")) |github_run_id| { + if (bun.getenvZ("GITHUB_SERVER_URL")) |github_server_url| { + if (bun.getenvZ("GITHUB_REPOSITORY")) |github_repository| { + if (github_run_id.len > 0 and github_server_url.len > 0 and github_repository.len > 0) { + break :brk try std.fmt.allocPrint(allocator, "{s}/{s}/actions/runs/{s}", .{ github_server_url, github_repository, github_run_id }); + } + } + } + } + + if (bun.getenvZ("CI_JOB_URL")) |ci_job_url| { + if (ci_job_url.len > 0) { + break :brk ci_job_url; + } + } + + break :brk ""; + }, + .commit = brk: { + if (bun.getenvZ("GITHUB_SHA")) |github_sha| { + if (github_sha.len > 0) { + break :brk github_sha; + } + } + + if (bun.getenvZ("CI_COMMIT_SHA")) |sha| { + if (sha.len > 0) { + break :brk sha; + } + } + + if (bun.getenvZ("GIT_SHA")) |git_sha| { + if (git_sha.len > 0) { + break :brk git_sha; + } + } + + break :brk ""; + }, + }; + + if (properties.ci.len == 0 and properties.commit.len == 0) { + this.properties_list_to_repeat_in_every_test_suite = ""; + return; + } + + var buffer = std.ArrayList(u8).init(bun.default_allocator); + var writer = buffer.writer(); + + try writer.writeAll( + \\ + \\ + ); + + if (properties.ci.len > 0) { + try writer.writeAll( + \\ \n"); + } + if (properties.commit.len > 0) { + try writer.writeAll( + \\ \n"); + } + + try writer.writeAll(" \n"); + + this.properties_list_to_repeat_in_every_test_suite = buffer.items; + } + + pub fn beginTestSuite(this: *JunitReporter, name: string) !void { + if (this.contents.items.len == 0) { + try this.contents.appendSlice(bun.default_allocator, + \\ + \\ + ); + + try this.contents.appendSlice(bun.default_allocator, + \\\n"); + } + + try this.contents.appendSlice(bun.default_allocator, + \\ \n"); + + if (this.properties_list_to_repeat_in_every_test_suite == null) { + try this.generatePropertiesList(); + } + + if (this.properties_list_to_repeat_in_every_test_suite) |properties_list| { + if (properties_list.len > 0) { + try this.contents.appendSlice(bun.default_allocator, properties_list); + } + } + + this.current_file = name; + } + + pub fn endTestSuite(this: *JunitReporter) !void { + var arena = std.heap.ArenaAllocator.init(bun.default_allocator); + defer arena.deinit(); + var stack_fallback_allocator = std.heap.stackFallback(4096, arena.allocator()); + const allocator = stack_fallback_allocator.get(); + + const metrics = &this.testcases_metrics; + this.total_metrics.add(metrics); + + const elapsed_time_ms = metrics.elapsed_time; + const elapsed_time_ms_f64: f64 = @floatFromInt(elapsed_time_ms); + const elapsed_time_seconds = elapsed_time_ms_f64 / std.time.ms_per_s; + + // Insert the summary attributes + const summary = try std.fmt.allocPrint(allocator, + \\tests="{d}" assertions="{d}" failures="{d}" skipped="{d}" time="{d}" hostname="{s}" + , .{ + metrics.test_cases, + metrics.assertions, + metrics.failures, + metrics.skipped, + elapsed_time_seconds, + this.getHostname() orelse "", + }); + this.testcases_metrics = .{}; + this.contents.insertSlice(bun.default_allocator, this.offset_of_testsuite_value, summary) catch bun.outOfMemory(); + + try this.contents.appendSlice(bun.default_allocator, " \n"); + } + + pub fn writeTestCase( + this: *JunitReporter, + status: TestRunner.Test.Status, + file: string, + name: string, + class_name: string, + assertions: u32, + elapsed_ns: u64, + ) !void { + const elapsed_ns_f64: f64 = @floatFromInt(elapsed_ns); + const elapsed_ms = elapsed_ns_f64 / std.time.ns_per_ms; + this.testcases_metrics.elapsed_time +|= @as(u64, @intFromFloat(elapsed_ms)); + this.testcases_metrics.test_cases += 1; + + try this.contents.appendSlice(bun.default_allocator, " { + try this.contents.appendSlice(bun.default_allocator, " />\n"); + }, + .fail => { + this.testcases_metrics.failures += 1; + try this.contents.appendSlice(bun.default_allocator, ">\n \n \n"); + // TODO: add the failure message + // if (failure_message) |msg| { + // try this.contents.appendSlice(bun.default_allocator, " message=\""); + // try escapeXml(msg, this.contents.writer(bun.default_allocator)); + // try this.contents.appendSlice(bun.default_allocator, "\""); + // } + }, + .fail_because_expected_assertion_count => { + this.testcases_metrics.failures += 1; + // TODO: add the failure message + try this.contents.writer(bun.default_allocator).print( + \\> + \\ + \\ + , .{assertions}); + }, + .fail_because_todo_passed => { + this.testcases_metrics.failures += 1; + // TODO: add the failure message + try this.contents.writer(bun.default_allocator).print( + \\> + \\ + \\ + , .{}); + }, + .fail_because_expected_has_assertions => { + this.testcases_metrics.failures += 1; + try this.contents.writer(bun.default_allocator).print( + \\> + \\ + \\ + , .{}); + }, + .skip => { + this.testcases_metrics.skipped += 1; + try this.contents.appendSlice(bun.default_allocator, ">\n \n \n"); + }, + .todo => { + this.testcases_metrics.skipped += 1; + try this.contents.appendSlice(bun.default_allocator, ">\n \n \n"); + }, + .pending => unreachable, + } + } + + pub fn writeToFile(this: *JunitReporter, path: string) !void { + if (this.contents.items.len == 0) return; + { + var arena = std.heap.ArenaAllocator.init(bun.default_allocator); + defer arena.deinit(); + var stack_fallback_allocator = std.heap.stackFallback(4096, arena.allocator()); + const allocator = stack_fallback_allocator.get(); + const metrics = this.total_metrics; + const elapsed_time = @as(f64, @floatFromInt(std.time.nanoTimestamp() - bun.start_time)) / std.time.ns_per_s; + const summary = try std.fmt.allocPrint(allocator, + \\tests="{d}" assertions="{d}" failures="{d}" skipped="{d}" time="{d}" + , .{ + metrics.test_cases, + metrics.assertions, + metrics.failures, + metrics.skipped, + elapsed_time, + }); + this.contents.insertSlice(bun.default_allocator, this.offset_of_testsuites_value, summary) catch bun.outOfMemory(); + this.contents.appendSlice(bun.default_allocator, "\n") catch bun.outOfMemory(); + } + + var junit_path_buf: bun.PathBuffer = undefined; + + @memcpy(junit_path_buf[0..path.len], path); + junit_path_buf[path.len] = 0; + + switch (bun.sys.File.openat(std.fs.cwd(), junit_path_buf[0..path.len :0], bun.O.WRONLY | bun.O.CREAT | bun.O.TRUNC, 0o664)) { + .err => |err| { + Output.err(error.JUnitReportFailed, "Failed to write JUnit report to {s}\n{}", .{ path, err }); + }, + .result => |fd| { + defer _ = fd.close(); + switch (bun.sys.File.writeAll(fd, this.contents.items)) { + .result => {}, + .err => |err| { + Output.err(error.JUnitReportFailed, "Failed to write JUnit report to {s}\n{}", .{ path, err }); + }, + } + }, + } + } + + pub fn deinit(this: *JunitReporter) void { + this.contents.deinit(bun.default_allocator); + } +}; + pub const CommandLineReporter = struct { jest: TestRunner, callback: TestRunner.Callback, @@ -88,6 +478,12 @@ pub const CommandLineReporter = struct { skips_to_repeat_buf: std.ArrayListUnmanaged(u8) = .{}, todos_to_repeat_buf: std.ArrayListUnmanaged(u8) = .{}, + file_reporter: ?FileReporter = null, + + pub const FileReporter = union(enum) { + junit: *JunitReporter, + }; + pub const Summary = struct { pass: u32 = 0, expectations: u32 = 0, @@ -110,7 +506,17 @@ pub const CommandLineReporter = struct { pub fn handleTestStart(_: *TestRunner.Callback, _: Test.ID) void {} - fn printTestLine(label: string, elapsed_ns: u64, parent: ?*jest.DescribeScope, comptime skip: bool, writer: anytype) void { + fn printTestLine( + status: TestRunner.Test.Status, + label: string, + elapsed_ns: u64, + parent: ?*jest.DescribeScope, + assertions: u32, + comptime skip: bool, + writer: anytype, + file: string, + file_reporter: ?FileReporter, + ) void { var scopes_stack = std.BoundedArray(*jest.DescribeScope, 64).init(0) catch unreachable; var parent_ = parent; @@ -165,9 +571,51 @@ pub const CommandLineReporter = struct { } writer.writeAll("\n") catch unreachable; + + if (file_reporter) |reporter| { + switch (reporter) { + .junit => |junit| { + const filename = brk: { + if (strings.hasPrefix(file, bun.fs.FileSystem.instance.top_level_dir)) { + break :brk strings.withoutLeadingPathSeparator(file[bun.fs.FileSystem.instance.top_level_dir.len..]); + } else { + break :brk file; + } + }; + if (!strings.eql(junit.current_file, filename)) { + if (junit.current_file.len > 0) { + junit.endTestSuite() catch bun.outOfMemory(); + } + + junit.beginTestSuite(filename) catch bun.outOfMemory(); + } + + var arena = std.heap.ArenaAllocator.init(bun.default_allocator); + defer arena.deinit(); + var stack_fallback = std.heap.stackFallback(4096, arena.allocator()); + const allocator = stack_fallback.get(); + var concatenated_describe_scopes = std.ArrayList(u8).init(allocator); + + { + const initial_length = concatenated_describe_scopes.items.len; + for (scopes) |scope| { + if (scope.label.len > 0) { + if (initial_length != concatenated_describe_scopes.items.len) { + concatenated_describe_scopes.appendSlice(" > ") catch bun.outOfMemory(); + } + + escapeXml(scope.label, concatenated_describe_scopes.writer()) catch bun.outOfMemory(); + } + } + } + + junit.writeTestCase(status, filename, display_label, concatenated_describe_scopes.items, assertions, elapsed_ns) catch bun.outOfMemory(); + }, + } + } } - pub fn handleTestPass(cb: *TestRunner.Callback, id: Test.ID, _: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { + pub fn handleTestPass(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { const writer_ = Output.errorWriter(); var buffered_writer = std.io.bufferedWriter(writer_); var writer = buffered_writer.writer(); @@ -177,14 +625,14 @@ pub const CommandLineReporter = struct { writeTestStatusLine(.pass, &writer); - printTestLine(label, elapsed_ns, parent, false, writer); + printTestLine(.pass, label, elapsed_ns, parent, expectations, false, writer, file, this.file_reporter); this.jest.tests.items(.status)[id] = TestRunner.Test.Status.pass; this.summary.pass += 1; this.summary.expectations += expectations; } - pub fn handleTestFail(cb: *TestRunner.Callback, id: Test.ID, _: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { + pub fn handleTestFail(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { var writer_ = Output.errorWriter(); var this: *CommandLineReporter = @fieldParentPtr("callback", cb); @@ -194,7 +642,7 @@ pub const CommandLineReporter = struct { var writer = this.failures_to_repeat_buf.writer(bun.default_allocator); writeTestStatusLine(.fail, &writer); - printTestLine(label, elapsed_ns, parent, false, writer); + printTestLine(.fail, label, elapsed_ns, parent, expectations, false, writer, file, this.file_reporter); // We must always reset the colors because (skip) will have set them to if (Output.enable_ansi_colors_stderr) { @@ -217,7 +665,7 @@ pub const CommandLineReporter = struct { } } - pub fn handleTestSkip(cb: *TestRunner.Callback, id: Test.ID, _: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { + pub fn handleTestSkip(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { var writer_ = Output.errorWriter(); var this: *CommandLineReporter = @fieldParentPtr("callback", cb); @@ -229,7 +677,7 @@ pub const CommandLineReporter = struct { var writer = this.skips_to_repeat_buf.writer(bun.default_allocator); writeTestStatusLine(.skip, &writer); - printTestLine(label, elapsed_ns, parent, true, writer); + printTestLine(.skip, label, elapsed_ns, parent, expectations, true, writer, file, this.file_reporter); writer_.writeAll(this.skips_to_repeat_buf.items[initial_length..]) catch unreachable; Output.flush(); @@ -241,7 +689,7 @@ pub const CommandLineReporter = struct { this.jest.tests.items(.status)[id] = TestRunner.Test.Status.skip; } - pub fn handleTestTodo(cb: *TestRunner.Callback, id: Test.ID, _: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { + pub fn handleTestTodo(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { var writer_ = Output.errorWriter(); var this: *CommandLineReporter = @fieldParentPtr("callback", cb); @@ -252,7 +700,7 @@ pub const CommandLineReporter = struct { var writer = this.todos_to_repeat_buf.writer(bun.default_allocator); writeTestStatusLine(.todo, &writer); - printTestLine(label, elapsed_ns, parent, true, writer); + printTestLine(.todo, label, elapsed_ns, parent, expectations, true, writer, file, this.file_reporter); writer_.writeAll(this.todos_to_repeat_buf.items[initial_length..]) catch unreachable; Output.flush(); @@ -723,6 +1171,10 @@ pub const TestCommand = struct { lcov: bool, }; + pub const FileReporter = enum { + junit, + }; + pub fn exec(ctx: Command.Context) !void { if (comptime is_bindgen) unreachable; @@ -782,6 +1234,13 @@ pub const TestCommand = struct { reporter.jest.callback = &reporter.callback; jest.Jest.runner = &reporter.jest; reporter.jest.test_options = &ctx.test_options; + + if (ctx.test_options.file_reporter) |file_reporter| { + reporter.file_reporter = switch (file_reporter) { + .junit => .{ .junit = JunitReporter.init() }, + }; + } + js_ast.Expr.Data.Store.create(); js_ast.Stmt.Data.Store.create(); var vm = try JSC.VirtualMachine.init( @@ -1089,6 +1548,17 @@ pub const TestCommand = struct { Output.prettyError("\n", .{}); Output.flush(); + if (reporter.file_reporter) |file_reporter| { + switch (file_reporter) { + .junit => |junit| { + if (junit.current_file.len > 0) { + junit.endTestSuite() catch {}; + } + junit.writeToFile(ctx.test_options.reporter_outfile.?) catch {}; + }, + } + } + if (vm.hot_reload == .watch) { vm.eventLoop().tickPossiblyForever(); diff --git a/test/bun.lockb b/test/bun.lockb index b025efa7b6e562..13caab649147e2 100755 Binary files a/test/bun.lockb and b/test/bun.lockb differ diff --git a/test/js/junit-reporter/junit.test.js b/test/js/junit-reporter/junit.test.js new file mode 100644 index 00000000000000..fdbc19196ba94c --- /dev/null +++ b/test/js/junit-reporter/junit.test.js @@ -0,0 +1,234 @@ +import { it, describe, expect } from "bun:test"; +import { file, spawn } from "bun"; +import { tempDirWithFiles, bunEnv, bunExe } from "harness"; + +const xml2js = require("xml2js"); + +describe("junit reporter", () => { + for (let withCIEnvironmentVariables of [false, true]) { + it(`should generate valid junit xml for passing tests ${withCIEnvironmentVariables ? "with CI environment variables" : ""}`, async () => { + const tmpDir = tempDirWithFiles("junit", { + "package.json": "{}", + "passing.test.js": ` + + it("should pass", () => { + expect(1 + 1).toBe(2); + }); + + it("second test", () => { + expect(1 + 1).toBe(2); + }); + + it("failing test", () => { + expect(1 + 1).toBe(3); + }); + + it.skip("skipped test", () => { + expect(1 + 1).toBe(2); + }); + + it.todo("todo test"); + + describe("nested describe", () => { + it("should pass inside nested describe", () => { + expect(1 + 1).toBe(2); + }); + + it("should fail inside nested describe", () => { + expect(1 + 1).toBe(3); + }); + }); + `, + + "test-2.test.js": ` + + it("should pass", () => { + expect(1 + 1).toBe(2); + }); + + it("failing test", () => { + expect(1 + 1).toBe(3); + }); + + describe("nested describe", () => { + it("should pass inside nested describe", () => { + expect(1 + 1).toBe(2); + }); + + it("should fail inside nested describe", () => { + expect(1 + 1).toBe(3); + }); + }); + `, + }); + + let env = bunEnv; + + if (withCIEnvironmentVariables) { + env = { + ...env, + CI_JOB_URL: "https://ci.example.com/123", + CI_COMMIT_SHA: "1234567890", + }; + } + + const junitPath = `${tmpDir}/junit.xml`; + const proc = spawn([bunExe(), "test", "--reporter=junit", "--reporter-outfile", junitPath], { + cwd: tmpDir, + env, + stdout: "inherit", + "stderr": "inherit", + }); + await proc.exited; + console.log(junitPath); + + expect(proc.exitCode).toBe(1); + const xmlContent = await file(junitPath).text(); + + // Parse XML to verify structure + const result = await new Promise((resolve, reject) => { + xml2js.parseString(xmlContent, (err, result) => { + if (err) reject(err); + else resolve(result); + }); + }); + + /** + * ------ Vitest ------ + * + * + * + * + * + * + * + * + * + * AssertionError: expected 2 to be 3 // Object.is equality + * + * - Expected + * + Received + * + * - 3 + * + 2 + * + * ❯ passing.test.js:12:25 + * + * + + * + * + * + * + * + * + * + * + * + * AssertionError: expected 2 to be 3 // Object.is equality + * + * - Expected + * + Received + * + * - 3 + * + 2 + * + * ❯ passing.test.js:27:27 + * + * + + * + * + * + * + * + * AssertionError: expected 2 to be 3 // Object.is equality + * + * - Expected + * + Received + * + * - 3 + * + 2 + * + * ❯ test-2.test.js:8:25 + * + * + * + * + * + * + * AssertionError: expected 2 to be 3 // Object.is equality + * + * - Expected + * + Received + * + * - 3 + * + 2 + * + * ❯ test-2.test.js:17:27 + * + * + * + * + */ + + expect(result.testsuites).toBeDefined(); + expect(result.testsuites.testsuite).toBeDefined(); + + let firstSuite = result.testsuites.testsuite[0]; + let secondSuite = result.testsuites.testsuite[1]; + + if (firstSuite.$.name === "passing.test.js") { + [firstSuite, secondSuite] = [secondSuite, firstSuite]; + } + + expect(firstSuite.testcase).toHaveLength(4); + expect(firstSuite.testcase[0].$.name).toBe("should pass inside nested describe"); + expect(firstSuite.$.name).toBe("test-2.test.js"); + expect(firstSuite.$.tests).toBe("4"); + expect(firstSuite.$.failures).toBe("2"); + expect(firstSuite.$.skipped).toBe("0"); + expect(parseFloat(firstSuite.$.time)).toBeGreaterThanOrEqual(0.0); + + expect(secondSuite.testcase).toHaveLength(7); + expect(secondSuite.testcase[0].$.name).toBe("should pass inside nested describe"); + expect(secondSuite.$.name).toBe("passing.test.js"); + expect(secondSuite.$.tests).toBe("7"); + expect(secondSuite.$.failures).toBe("2"); + expect(secondSuite.$.skipped).toBe("2"); + expect(parseFloat(secondSuite.$.time)).toBeGreaterThanOrEqual(0.0); + + expect(result.testsuites.$.tests).toBe("11"); + expect(result.testsuites.$.failures).toBe("4"); + expect(result.testsuites.$.skipped).toBe("2"); + expect(parseFloat(result.testsuites.$.time)).toBeGreaterThanOrEqual(0.0); + + if (withCIEnvironmentVariables) { + // "properties": [ + // { + // "property": [ + // { + // "$": { + // "name": "ci", + // "value": "https://ci.example.com/123" + // } + // }, + // { + // "$": { + // "name": "commit", + // "value": "1234567890" + // } + // } + // ] + // } + // ], + expect(firstSuite.properties).toHaveLength(1); + expect(firstSuite.properties[0].property).toHaveLength(2); + expect(firstSuite.properties[0].property[0].$.name).toBe("ci"); + expect(firstSuite.properties[0].property[0].$.value).toBe("https://ci.example.com/123"); + expect(firstSuite.properties[0].property[1].$.name).toBe("commit"); + expect(firstSuite.properties[0].property[1].$.value).toBe("1234567890"); + } + }); + } +}); diff --git a/test/package.json b/test/package.json index 3f75a236a42d80..b360be82bbb2e8 100644 --- a/test/package.json +++ b/test/package.json @@ -65,6 +65,7 @@ "vitest": "0.32.2", "webpack": "5.88.0", "webpack-cli": "4.7.2", + "xml2js": "0.6.2", "yargs": "17.7.2" }, "private": true,