From 029eba397926832a1c5baf82cab2c33a5c12b4d5 Mon Sep 17 00:00:00 2001 From: Josh Lee Date: Fri, 17 May 2024 08:14:10 -0600 Subject: [PATCH 1/3] Added a strip_prefix to remove the BOM from the file content if it is present to bring it inline with how the C# lib works --- crates/compiler/src/compiler/run_compilation.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/compiler/src/compiler/run_compilation.rs b/crates/compiler/src/compiler/run_compilation.rs index 0fabec41..0fa96003 100644 --- a/crates/compiler/src/compiler/run_compilation.rs +++ b/crates/compiler/src/compiler/run_compilation.rs @@ -28,7 +28,16 @@ pub(crate) fn compile(compiler: &Compiler) -> Result { let chars: Vec> = compiler .files .iter() - .map(|file| file.source.chars().map(|c| c as u32).collect()) + .map(|file| { + // Strip the BOM from the source string if it is present before compiling. + // Rust does not do this by default + // https://github.com/rust-lang/rfcs/issues/2428 + let source = match file.source.strip_prefix("\u{feff}") { + None => file.source.as_str(), + Some(sanitized_string) => sanitized_string + }; + source.chars().map(|c| c as u32).collect() + }) .collect(); let chars: Vec<_> = chars.iter().map(|c| c.as_slice()).collect(); let initial = CompilationIntermediate::from_job(compiler, chars); From 1a94ee532d1dd0318ba6f3f4ef0607757dfeb227 Mon Sep 17 00:00:00 2001 From: Jan Hohenheim Date: Fri, 17 May 2024 17:36:02 +0200 Subject: [PATCH 2/3] Fix lints --- crates/compiler/src/compiler/run_compilation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compiler/src/compiler/run_compilation.rs b/crates/compiler/src/compiler/run_compilation.rs index 0fa96003..6babc0e5 100644 --- a/crates/compiler/src/compiler/run_compilation.rs +++ b/crates/compiler/src/compiler/run_compilation.rs @@ -34,7 +34,7 @@ pub(crate) fn compile(compiler: &Compiler) -> Result { // https://github.com/rust-lang/rfcs/issues/2428 let source = match file.source.strip_prefix("\u{feff}") { None => file.source.as_str(), - Some(sanitized_string) => sanitized_string + Some(sanitized_string) => sanitized_string, }; source.chars().map(|c| c as u32).collect() }) From 165eda862e0a33ecea891d8ef09231f479a7022a Mon Sep 17 00:00:00 2001 From: Jan Hohenheim Date: Fri, 17 May 2024 17:39:07 +0200 Subject: [PATCH 3/3] Fix lints --- crates/compiler/src/compiler/run_compilation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/compiler/src/compiler/run_compilation.rs b/crates/compiler/src/compiler/run_compilation.rs index 6babc0e5..66cc08fb 100644 --- a/crates/compiler/src/compiler/run_compilation.rs +++ b/crates/compiler/src/compiler/run_compilation.rs @@ -32,7 +32,7 @@ pub(crate) fn compile(compiler: &Compiler) -> Result { // Strip the BOM from the source string if it is present before compiling. // Rust does not do this by default // https://github.com/rust-lang/rfcs/issues/2428 - let source = match file.source.strip_prefix("\u{feff}") { + let source = match file.source.strip_prefix('\u{feff}') { None => file.source.as_str(), Some(sanitized_string) => sanitized_string, };