From 6c08759c6be517aaa6209c7c3a178b371f7d30c7 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 17 Apr 2024 19:01:07 -0700 Subject: [PATCH 1/9] Fixes #10344 --- src/js_ast.zig | 5 ++-- test/bundler/bundler_compile.test.ts | 36 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/js_ast.zig b/src/js_ast.zig index 227278b62bbf89..3b704620248f95 100644 --- a/src/js_ast.zig +++ b/src/js_ast.zig @@ -601,7 +601,7 @@ pub const CharFreq = struct { // https://zig.godbolt.org/z/P5dPojWGK var freqs = out.*; defer out.* = freqs; - var deltas: [255]i32 = [_]i32{0} ** 255; + var deltas: [256]i32 = [_]i32{0} ** 256; var remain = text; bun.assert(remain.len >= scan_big_chunk_size); @@ -613,8 +613,7 @@ pub const CharFreq = struct { while (unrolled_ptr != remain_end) : (unrolled_ptr += scan_big_chunk_size) { const chunk = unrolled_ptr[0..scan_big_chunk_size].*; - comptime var i: usize = 0; - inline while (i < scan_big_chunk_size) : (i += scan_big_chunk_size) { + inline for (0..scan_big_chunk_size) |i| { deltas[@as(usize, chunk[i])] += delta; } } diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 1fdada5873270b..c120646a2d4eb9 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -2,6 +2,7 @@ import assert from "assert"; import dedent from "dedent"; import { ESBUILD, itBundled, testForFile } from "./expectBundled"; import { Database } from "bun:sqlite"; +import { fillRepeating } from "harness"; var { describe, test, expect } = testForFile(import.meta.path); describe("bundler", () => { @@ -186,6 +187,41 @@ describe("bundler", () => { }, compile: true, }); + // https://github.com/oven-sh/bun/issues/10344 + itBundled("compile/#10344", { + minifyIdentifiers: true, + minifySyntax: true, + minifyWhitespace: true, + target: "bun", + sourceMap: "external", + compile: true, + files: { + "/entry.ts": /* js */ ` + import big from './generated.big.binary' with {type: "file"}; + import small from './generated.small.binary' with {type: "file"}; + import fs from 'fs'; + fs.readFileSync(big).toString("hex"); + fs.readFileSync(small).toString("hex"); + console.log("PASS"); + `, + "/generated.big.binary": (() => { + // make sure the size is not divisible by 32 + const buffer = new Uint8ClampedArray(4096 + (32 - 2)); + for (let i = 0; i < buffer.length; i++) { + buffer[i] = i; + } + return buffer; + })(), + "/generated.small.binary": (() => { + const buffer = new Uint8ClampedArray(31); + for (let i = 0; i < buffer.length; i++) { + buffer[i] = i; + } + return buffer; + })(), + }, + run: { stdout: "PASS" }, + }); itBundled("compile/EmbeddedSqlite", { compile: true, files: { From fcb306407371d0c6bdfb29d8e3c58e5b5ddf34dc Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 17 Apr 2024 19:01:55 -0700 Subject: [PATCH 2/9] Update bundler_compile.test.ts --- test/bundler/bundler_compile.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index c120646a2d4eb9..750c8d9907278e 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -213,6 +213,7 @@ describe("bundler", () => { return buffer; })(), "/generated.small.binary": (() => { + // make sure the size is less than 32 const buffer = new Uint8ClampedArray(31); for (let i = 0; i < buffer.length; i++) { buffer[i] = i; From 0eab77854ba1e933b938ae9f05554cf6ba5be1e4 Mon Sep 17 00:00:00 2001 From: Jarred-Sumner Date: Thu, 18 Apr 2024 02:07:55 +0000 Subject: [PATCH 3/9] Apply formatting changes --- test/harness.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/harness.ts b/test/harness.ts index a9d6276509785e..2d00ec8d631f01 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -20,8 +20,8 @@ export const bunEnv: NodeJS.ProcessEnv = { TZ: "Etc/UTC", CI: "1", BUN_RUNTIME_TRANSPILER_CACHE_PATH: "0", - BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING: '1', - BUN_GARBAGE_COLLECTOR_LEVEL: process.env.BUN_GARBAGE_COLLECTOR_LEVEL || '0', + BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING: "1", + BUN_GARBAGE_COLLECTOR_LEVEL: process.env.BUN_GARBAGE_COLLECTOR_LEVEL || "0", }; if (isWindows) { From 82ba71e13b4fa051a7c5b67ac33fe8ab41ba50e5 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 17 Apr 2024 20:03:31 -0700 Subject: [PATCH 4/9] Track comments when bundling --- src/js_ast.zig | 37 +++++++++++++++++----------- src/js_parser.zig | 1 + test/bundler/bundler_compile.test.ts | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/js_ast.zig b/src/js_ast.zig index 3b704620248f95..fe05194f4c3906 100644 --- a/src/js_ast.zig +++ b/src/js_ast.zig @@ -567,21 +567,30 @@ pub const SlotCounts = struct { } }; +const char_freq_count = 64; pub const CharAndCount = struct { char: u8 = 0, count: i32 = 0, index: usize = 0, - pub const Array = [64]CharAndCount; + pub const Array = [char_freq_count]CharAndCount; pub fn lessThan(_: void, a: CharAndCount, b: CharAndCount) bool { - return a.count > b.count or (a.count == b.count and a.index < b.index); + if (a.count != b.count) { + return a.count > b.count; + } + + if (a.index != b.index) { + return a.index < b.index; + } + + return a.char < b.char; } }; pub const CharFreq = struct { - const Vector = @Vector(64, i32); - const Buffer = [64]i32; + const Vector = @Vector(char_freq_count, i32); + const Buffer = [char_freq_count]i32; freqs: Buffer align(1) = undefined, @@ -630,7 +639,7 @@ pub const CharFreq = struct { } fn scanSmall(out: *align(1) Buffer, text: string, delta: i32) void { - var freqs: [64]i32 = out.*; + var freqs: [char_freq_count]i32 = out.*; defer out.* = freqs; for (text) |c| { @@ -648,19 +657,17 @@ pub const CharFreq = struct { pub fn include(this: *CharFreq, other: CharFreq) void { // https://zig.godbolt.org/z/Mq8eK6K9s - var left: @Vector(64, i32) = this.freqs; - defer this.freqs = left; - const right: @Vector(64, i32) = other.freqs; + const left: @Vector(char_freq_count, i32) = this.freqs; + const right: @Vector(char_freq_count, i32) = other.freqs; - left += right; + this.freqs = left + right; } pub fn compile(this: *const CharFreq, allocator: std.mem.Allocator) NameMinifier { var array: CharAndCount.Array = brk: { var _array: CharAndCount.Array = undefined; - const freqs = this.freqs; - for (&_array, NameMinifier.default_tail, &freqs, 0..) |*dest, char, freq, i| { + for (&_array, NameMinifier.default_tail, this.freqs, 0..) |*dest, char, freq, i| { dest.* = CharAndCount{ .char = char, .index = i, @@ -710,9 +717,9 @@ pub const NameMinifier = struct { while (i > 0) { i -= 1; - j = @as(usize, @intCast(@mod(i, 64))); + j = @as(usize, @intCast(@mod(i, char_freq_count))); try name.appendSlice(this.tail.items[j .. j + 1]); - i = @divFloor(i, 64); + i = @divFloor(i, char_freq_count); } } @@ -725,9 +732,9 @@ pub const NameMinifier = struct { while (i > 0) { i -= 1; - j = @as(usize, @intCast(@mod(i, 64))); + j = @as(usize, @intCast(@mod(i, char_freq_count))); try name.appendSlice(default_tail[j .. j + 1]); - i = @divFloor(i, 64); + i = @divFloor(i, char_freq_count); } return name.items; diff --git a/src/js_parser.zig b/src/js_parser.zig index 3ff5cb1992e237..af154af1d32af9 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -22970,6 +22970,7 @@ fn NewParser_( // Only enable during bundling .commonjs_named_exports_deoptimized = !opts.bundle, }; + this.lexer.track_comments = opts.features.minify_identifiers; this.unwrap_all_requires = brk: { if (opts.bundle) { diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 750c8d9907278e..05b3b68ddbf410 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -188,7 +188,7 @@ describe("bundler", () => { compile: true, }); // https://github.com/oven-sh/bun/issues/10344 - itBundled("compile/#10344", { + itBundled("compile/10344", { minifyIdentifiers: true, minifySyntax: true, minifyWhitespace: true, From 048c3823cfa48146873366c1af4d87d7ad4f6639 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 18 Apr 2024 00:03:53 -0700 Subject: [PATCH 5/9] Fix embedded files and add test --- src/bundler/bundle_v2.zig | 6 +-- test/bundler/bundler_compile.test.ts | 64 ++++++++++++++------------ test/bundler/esbuild/splitting.test.ts | 3 +- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index e3fb47ef3b5180..2d297b1370a32e 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -11104,7 +11104,7 @@ pub const Chunk = struct { .chunk, .asset => { const index = piece.index.index; const file_path = switch (piece.index.kind) { - .asset => graph.additional_output_files.items[additional_files[index].last().?.output_file].src_path.text, + .asset => graph.additional_output_files.items[additional_files[index].last().?.output_file].dest_path, .chunk => chunks[index].final_rel_path, else => unreachable, }; @@ -11155,7 +11155,7 @@ pub const Chunk = struct { .asset => { shift.before.advance(unique_key_for_additional_files[index]); const file = graph.additional_output_files.items[additional_files[index].last().?.output_file]; - break :brk file.src_path.text; + break :brk file.dest_path; }, .chunk => { const piece_chunk = chunks[index]; @@ -11589,7 +11589,7 @@ fn cheapPrefixNormalizer(prefix: []const u8, suffix: []const u8) [2]string { if (strings.endsWithChar(prefix, '/')) { if (strings.startsWithChar(suffix, '/')) { return .{ - prefix[0 .. prefix.len - 1], + prefix[0..prefix.len], suffix[1..suffix.len], }; } diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 05b3b68ddbf410..ab1cc762a76742 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -187,42 +187,48 @@ describe("bundler", () => { }, compile: true, }); - // https://github.com/oven-sh/bun/issues/10344 - itBundled("compile/10344", { - minifyIdentifiers: true, - minifySyntax: true, - minifyWhitespace: true, - target: "bun", - sourceMap: "external", - compile: true, - files: { - "/entry.ts": /* js */ ` + for (const sourceMap of ["external", "inline", "none"] as const) { + for (const minify of [true, false] as const) { + // https://github.com/oven-sh/bun/issues/10344 + itBundled("compile/10344+sourcemap=" + sourceMap + (minify ? "+minify" : ""), { + minifyIdentifiers: minify, + minifySyntax: minify, + minifyWhitespace: minify, + target: "bun", + sourceMap, + compile: true, + files: { + "/entry.ts": /* js */ ` import big from './generated.big.binary' with {type: "file"}; import small from './generated.small.binary' with {type: "file"}; import fs from 'fs'; fs.readFileSync(big).toString("hex"); + await Bun.file(big).arrayBuffer(); fs.readFileSync(small).toString("hex"); + await Bun.file(small).arrayBuffer(); console.log("PASS"); `, - "/generated.big.binary": (() => { - // make sure the size is not divisible by 32 - const buffer = new Uint8ClampedArray(4096 + (32 - 2)); - for (let i = 0; i < buffer.length; i++) { - buffer[i] = i; - } - return buffer; - })(), - "/generated.small.binary": (() => { - // make sure the size is less than 32 - const buffer = new Uint8ClampedArray(31); - for (let i = 0; i < buffer.length; i++) { - buffer[i] = i; - } - return buffer; - })(), - }, - run: { stdout: "PASS" }, - }); + "/generated.big.binary": (() => { + // make sure the size is not divisible by 32 + const buffer = new Uint8ClampedArray(4096 + (32 - 2)); + for (let i = 0; i < buffer.length; i++) { + buffer[i] = i; + } + return buffer; + })(), + "/generated.small.binary": (() => { + // make sure the size is less than 32 + const buffer = new Uint8ClampedArray(31); + for (let i = 0; i < buffer.length; i++) { + buffer[i] = i; + } + return buffer; + })(), + }, + run: { stdout: "PASS" }, + }); + } + } itBundled("compile/EmbeddedSqlite", { compile: true, files: { diff --git a/test/bundler/esbuild/splitting.test.ts b/test/bundler/esbuild/splitting.test.ts index fbf5634fe78e92..69cdf712b26c23 100644 --- a/test/bundler/esbuild/splitting.test.ts +++ b/test/bundler/esbuild/splitting.test.ts @@ -541,14 +541,13 @@ describe("bundler", () => { }, }); itBundled("splitting/PublicPathEntryName", { - todo: true, files: { "/a.js": `import("./b")`, "/b.js": `console.log('b')`, }, outdir: "/out", splitting: true, - publicPath: "/www", + publicPath: "/www/", onAfterBundle(api) { const t = new Bun.Transpiler(); const imports = t.scanImports(api.readFile("/out/a.js")); From 1d2c7c23ddf2d1e06d12e0bd47e1bc57904ca694 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 18 Apr 2024 00:04:02 -0700 Subject: [PATCH 6/9] Make this const --- src/js_ast.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/js_ast.zig b/src/js_ast.zig index fe05194f4c3906..d0ad8ff66e2fa4 100644 --- a/src/js_ast.zig +++ b/src/js_ast.zig @@ -664,7 +664,7 @@ pub const CharFreq = struct { } pub fn compile(this: *const CharFreq, allocator: std.mem.Allocator) NameMinifier { - var array: CharAndCount.Array = brk: { + const array: CharAndCount.Array = brk: { var _array: CharAndCount.Array = undefined; for (&_array, NameMinifier.default_tail, this.freqs, 0..) |*dest, char, freq, i| { @@ -674,11 +674,12 @@ pub const CharFreq = struct { .count = freq, }; } + + std.sort.pdq(CharAndCount, &_array, {}, CharAndCount.lessThan); + break :brk _array; }; - std.sort.pdq(CharAndCount, &array, {}, CharAndCount.lessThan); - var minifier = NameMinifier.init(allocator); minifier.head.ensureTotalCapacityPrecise(NameMinifier.default_head.len) catch unreachable; minifier.tail.ensureTotalCapacityPrecise(NameMinifier.default_tail.len) catch unreachable; From 3007d73574cbcb4864fa37be1c7818b83e86ad38 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 18 Apr 2024 00:05:20 -0700 Subject: [PATCH 7/9] Update runner.node.mjs --- packages/bun-internal-test/src/runner.node.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bun-internal-test/src/runner.node.mjs b/packages/bun-internal-test/src/runner.node.mjs index 8b7796af17c026..d47254a0c9a7c3 100644 --- a/packages/bun-internal-test/src/runner.node.mjs +++ b/packages/bun-internal-test/src/runner.node.mjs @@ -558,7 +558,7 @@ function emojiTag(tag) { emojiText += ""; } - if (tag.includes("x86") || tag.includes("_64") || tag.includes("amd64")) { + if (tag.includes("x86") || tag.includes("x64") || tag.includes("_64") || tag.includes("amd64")) { if (!tag.includes("linux")) { emojiText += "💻"; } else { From 9342c2447ddc7be5916a198b2d22306768893f46 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 18 Apr 2024 00:33:22 -0700 Subject: [PATCH 8/9] Prefill process arch/platform in bun build --compile --- src/cli/build_command.zig | 39 ++++++++++++++++++++++++++++ test/bundler/bundler_compile.test.ts | 16 ++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index d1c058b1a5e87e..bef8ec202819bb 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -35,6 +35,26 @@ const BundleV2 = @import("../bundler/bundle_v2.zig").BundleV2; var estimated_input_lines_of_code_: usize = undefined; pub const BuildCommand = struct { + const compile_define_keys = &.{ + "process.platform", + "process.arch", + }; + const compile_define_values = &.{ + switch (Environment.os) { + .linux => "'linux'", + .mac => "'darwin'", + .windows => "'win32'", + else => "unknown", + }, + switch (Environment.isX64) { + true => "'x64'", + else => if (Environment.isAarch64) + "'arm64'" + else + "'unknown'", + }, + }; + pub fn exec( ctx: Command.Context, ) !void { @@ -47,6 +67,25 @@ pub const BuildCommand = struct { ctx.args.target = .bun; } + if (ctx.bundler_options.compile) { + if (ctx.args.define == null) { + ctx.args.define = .{ + .keys = compile_define_keys, + .values = compile_define_values, + }; + } else if (ctx.args.define) |*define| { + var keys = try std.ArrayList(string).initCapacity(bun.default_allocator, compile_define_keys.len + define.keys.len); + keys.appendSliceAssumeCapacity(compile_define_keys); + keys.appendSliceAssumeCapacity(define.keys); + var values = try std.ArrayList(string).initCapacity(bun.default_allocator, compile_define_values.len + define.values.len); + values.appendSliceAssumeCapacity(compile_define_values); + values.appendSliceAssumeCapacity(define.values); + + define.keys = keys.items; + define.values = values.items; + } + } + var this_bundler = try bundler.Bundler.init(allocator, log, ctx.args, null); this_bundler.options.source_map = options.SourceMapOption.fromApi(ctx.args.source_map); diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index ab1cc762a76742..842a7a8609e95e 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -187,8 +187,20 @@ describe("bundler", () => { }, compile: true, }); - for (const sourceMap of ["external", "inline", "none"] as const) { - for (const minify of [true, false] as const) { + for (const minify of [true, false] as const) { + itBundled("compile/platform-specific-binary" + (minify ? "-minify" : ""), { + minifySyntax: minify, + target: "bun", + compile: true, + files: { + "/entry.ts": /* js */ ` + await import(\`./platform.\${process.platform}.\${process.arch}.js\`); + `, + [`/platform.${process.platform}.${process.arch}.js`]: `console.log("${process.platform}", "${process.arch}");`, + }, + run: { stdout: `${process.platform} ${process.arch}` }, + }); + for (const sourceMap of ["external", "inline", "none"] as const) { // https://github.com/oven-sh/bun/issues/10344 itBundled("compile/10344+sourcemap=" + sourceMap + (minify ? "+minify" : ""), { minifyIdentifiers: minify, From f4c5274faf378d2318c2df2bf77b1f61f944da78 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Thu, 18 Apr 2024 12:34:29 -0700 Subject: [PATCH 9/9] nitpick --- src/cli/build_command.zig | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index bef8ec202819bb..0c696f714b9971 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -39,19 +39,14 @@ pub const BuildCommand = struct { "process.platform", "process.arch", }; + const compile_define_values = &.{ - switch (Environment.os) { - .linux => "'linux'", - .mac => "'darwin'", - .windows => "'win32'", - else => "unknown", - }, - switch (Environment.isX64) { - true => "'x64'", - else => if (Environment.isAarch64) - "'arm64'" - else - "'unknown'", + "\"" ++ Environment.os.nameString() ++ "\"", + + switch (@import("builtin").target.cpu.arch) { + .x86_64 => "\"x64\"", + .aarch64 => "\"arm64\"", + else => @compileError("TODO"), }, };