From 01dc1053b1cafbc00e5af3d0b9448b73796ac200 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 14:22:30 +0800 Subject: [PATCH 1/7] refactor: move current xwin implementation to `clang_cl` backend --- src/backend/clang_cl.rs | 477 +++++++++++++++++++++++++++++++ src/backend/mod.rs | 1 + src/{ => backend}/override.cmake | 0 src/common.rs | 470 ++---------------------------- src/lib.rs | 1 + 5 files changed, 506 insertions(+), 443 deletions(-) create mode 100644 src/backend/clang_cl.rs create mode 100644 src/backend/mod.rs rename src/{ => backend}/override.cmake (100%) diff --git a/src/backend/clang_cl.rs b/src/backend/clang_cl.rs new file mode 100644 index 0000000..6b80777 --- /dev/null +++ b/src/backend/clang_cl.rs @@ -0,0 +1,477 @@ +use std::collections::HashSet; +use std::convert::TryInto; +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result}; +use fs_err as fs; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use path_slash::PathExt; +use which::{which, which_in}; +use xwin::util::ProgressTarget; + +use crate::common::{ + adjust_canonicalization, default_build_target_from_config, get_rustflags, http_agent, + symlink_llvm_tool, XWinOptions, +}; + +#[derive(Debug)] +pub struct ClangCl<'a> { + xwin_options: &'a XWinOptions, +} + +impl<'a> ClangCl<'a> { + pub fn new(xwin_options: &'a XWinOptions) -> Self { + Self { xwin_options } + } + + pub fn apply_command_env( + &self, + manifest_path: Option<&Path>, + cargo: &cargo_options::CommonOptions, + cmd: &mut Command, + ) -> Result<()> { + let xwin_cache_dir = self + .xwin_options + .xwin_cache_dir + .clone() + .unwrap_or_else(|| { + dirs::cache_dir() + // If the really is no cache dir, cwd will also do + .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) + .join(env!("CARGO_PKG_NAME")) + }) + .join("xwin"); + fs::create_dir_all(&xwin_cache_dir)?; + let xwin_cache_dir = xwin_cache_dir.canonicalize()?; + + let env_path = env::var("PATH").unwrap_or_default(); + let mut env_paths: Vec<_> = env::split_paths(&env_path).collect(); + + let env_path = if cfg!(target_os = "macos") { + let mut new_path = env_path; + new_path.push_str(":/opt/homebrew/opt/llvm/bin"); + new_path.push_str(":/usr/local/opt/llvm/bin"); + new_path + } else { + env_path + }; + let cache_dir = xwin_cache_dir.parent().unwrap(); + env_paths.push(cache_dir.to_path_buf()); + + let workdir = manifest_path + .and_then(|p| p.parent().map(|x| x.to_path_buf())) + .or_else(|| env::current_dir().ok()) + .unwrap(); + let mut targets = cargo.target.clone(); + if targets.is_empty() { + if let Some(build_target) = default_build_target_from_config(&workdir)? { + // if no target is specified, use the default build target + // Note that this is required, otherwise it may fail with link errors + cmd.arg("--target").arg(&build_target); + targets.push(build_target); + } + } + + for target in &targets { + if target.contains("msvc") { + self.setup_msvc_crt(xwin_cache_dir.clone())?; + let env_target = target.to_lowercase().replace('-', "_"); + + if which_in("clang-cl", Some(env_path.clone()), env::current_dir()?).is_err() { + if let Ok(clang) = which("clang") { + #[cfg(windows)] + { + let symlink = cache_dir.join("clang-cl.exe"); + if symlink.exists() { + fs::remove_file(&symlink)?; + } + std::os::windows::fs::symlink_file(clang, symlink)?; + } + + #[cfg(unix)] + { + let symlink = cache_dir.join("clang-cl"); + if symlink.exists() { + fs::remove_file(&symlink)?; + } + std::os::unix::fs::symlink(clang, symlink)?; + } + } + } + symlink_llvm_tool("rust-lld", "lld-link", env_path.clone(), cache_dir)?; + symlink_llvm_tool("llvm-ar", "llvm-lib", env_path.clone(), cache_dir)?; + symlink_llvm_tool("llvm-ar", "llvm-dlltool", env_path.clone(), cache_dir)?; + + cmd.env("TARGET_CC", "clang-cl"); + cmd.env("TARGET_CXX", "clang-cl"); + cmd.env(format!("CC_{}", env_target), "clang-cl"); + cmd.env(format!("CXX_{}", env_target), "clang-cl"); + cmd.env("TARGET_AR", "llvm-lib"); + cmd.env(format!("AR_{}", env_target), "llvm-lib"); + + cmd.env( + format!("CARGO_TARGET_{}_LINKER", env_target.to_uppercase()), + "lld-link", + ); + + let user_set_cl_flags = env::var("CL_FLAGS").unwrap_or_default(); + let user_set_c_flags = env::var("CFLAGS").unwrap_or_default(); + let user_set_cxx_flags = env::var("CXXFLAGS").unwrap_or_default(); + + let xwin_dir = adjust_canonicalization(xwin_cache_dir.to_slash_lossy().to_string()); + let cl_flags = format!( + "--target={target} -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc{dir}/crt/include /imsvc{dir}/sdk/include/ucrt /imsvc{dir}/sdk/include/um /imsvc{dir}/sdk/include/shared {user_set_cl_flags}", + target = target, + dir = xwin_dir, + user_set_cl_flags = user_set_cl_flags, + ); + cmd.env("CL_FLAGS", &cl_flags); + cmd.env( + format!("CFLAGS_{}", env_target), + &format!( + "{cl_flags} {user_set_c_flags}", + cl_flags = cl_flags, + user_set_c_flags = user_set_c_flags + ), + ); + cmd.env( + format!("CXXFLAGS_{}", env_target), + &format!( + "{cl_flags} {user_set_cxx_flags}", + cl_flags = cl_flags, + user_set_cxx_flags = user_set_cxx_flags + ), + ); + + cmd.env( + format!("BINDGEN_EXTRA_CLANG_ARGS_{}", env_target), + format!( + "-I{dir}/crt/include -I{dir}/sdk/include/ucrt -I{dir}/sdk/include/um -I{dir}/sdk/include/shared", + dir = xwin_dir + ) + ); + + cmd.env( + "RCFLAGS", + format!( + "-I{dir}/crt/include -I{dir}/sdk/include/ucrt -I{dir}/sdk/include/um -I{dir}/sdk/include/shared", + dir = xwin_dir + ) + ); + + let target_arch = target + .split_once('-') + .map(|(x, _)| x) + .context("invalid target triple")?; + let xwin_arch = match target_arch { + "i586" | "i686" => "x86", + _ => target_arch, + }; + + let mut rustflags = get_rustflags(&workdir, target)?.unwrap_or_default(); + rustflags + .flags + .extend(["-C".to_string(), "linker-flavor=lld-link".to_string()]); + rustflags.push(format!( + "-Lnative={dir}/crt/lib/{arch}", + dir = xwin_dir, + arch = xwin_arch + )); + rustflags.push(format!( + "-Lnative={dir}/sdk/lib/um/{arch}", + dir = xwin_dir, + arch = xwin_arch + )); + rustflags.push(format!( + "-Lnative={dir}/sdk/lib/ucrt/{arch}", + dir = xwin_dir, + arch = xwin_arch + )); + cmd.env("CARGO_ENCODED_RUSTFLAGS", rustflags.encode()?); + + #[cfg(target_os = "macos")] + { + let usr_llvm = "/usr/local/opt/llvm/bin".into(); + let opt_llvm = "/opt/homebrew/opt/llvm/bin".into(); + if cfg!(target_arch = "x86_64") && !env_paths.contains(&usr_llvm) { + env_paths.push(usr_llvm); + } else if cfg!(target_arch = "aarch64") && !env_paths.contains(&opt_llvm) { + env_paths.push(opt_llvm); + } + } + + cmd.env("PATH", env::join_paths(env_paths.clone())?); + + // CMake support + let cmake_toolchain = self.setup_cmake_toolchain(target, &xwin_cache_dir)?; + cmd.env("CMAKE_GENERATOR", "Ninja") + .env("CMAKE_SYSTEM_NAME", "Windows") + .env( + format!("CMAKE_TOOLCHAIN_FILE_{}", env_target), + cmake_toolchain, + ); + } + } + Ok(()) + } + + fn setup_msvc_crt(&self, cache_dir: PathBuf) -> Result<()> { + let done_mark_file = cache_dir.join("DONE"); + let xwin_arches: HashSet<_> = self + .xwin_options + .xwin_arch + .iter() + .map(|x| x.as_str().to_string()) + .collect(); + let mut downloaded_arches = HashSet::new(); + if let Ok(content) = fs::read_to_string(&done_mark_file) { + for arch in content.split_whitespace() { + downloaded_arches.insert(arch.to_string()); + } + } + if xwin_arches.difference(&downloaded_arches).next().is_none() { + return Ok(()); + } + + let draw_target = ProgressTarget::Stdout; + + let agent = http_agent()?; + let xwin_dir = adjust_canonicalization(cache_dir.to_slash_lossy().to_string()); + // timeout defaults to 60s + let ctx = xwin::Ctx::with_dir(xwin::PathBuf::from(xwin_dir), draw_target, agent)?; + let ctx = std::sync::Arc::new(ctx); + let pkg_manifest = self.load_manifest(&ctx, draw_target)?; + + let arches = self + .xwin_options + .xwin_arch + .iter() + .fold(0, |acc, arch| acc | *arch as u32); + let variants = self + .xwin_options + .xwin_variant + .iter() + .fold(0, |acc, var| acc | *var as u32); + let pruned = xwin::prune_pkg_list(&pkg_manifest, arches, variants, false, None, None)?; + let op = xwin::Ops::Splat(xwin::SplatConfig { + include_debug_libs: self.xwin_options.xwin_include_debug_libs, + include_debug_symbols: self.xwin_options.xwin_include_debug_symbols, + enable_symlinks: !cfg!(target_os = "macos"), + preserve_ms_arch_notation: false, + use_winsysroot_style: false, + copy: false, + output: cache_dir.clone().try_into()?, + map: None, + }); + let pkgs = pkg_manifest.packages; + + let mp = MultiProgress::with_draw_target(draw_target.into()); + let work_items: Vec<_> = pruned.payloads + .into_iter() + .map(|pay| { + let prefix = match pay.kind { + xwin::PayloadKind::CrtHeaders => "CRT.headers".to_owned(), + xwin::PayloadKind::AtlHeaders => "ATL.headers".to_owned(), + xwin::PayloadKind::CrtLibs => { + format!( + "CRT.libs.{}.{}", + pay.target_arch.map(|ta| ta.as_str()).unwrap_or("all"), + pay.variant.map(|v| v.as_str()).unwrap_or("none") + ) + } + xwin::PayloadKind::AtlLibs => { + format!( + "ATL.libs.{}", + pay.target_arch.map(|ta| ta.as_str()).unwrap_or("all"), + ) + } + xwin::PayloadKind::SdkHeaders => { + format!( + "SDK.headers.{}.{}", + pay.target_arch.map(|v| v.as_str()).unwrap_or("all"), + pay.variant.map(|v| v.as_str()).unwrap_or("none") + ) + } + xwin::PayloadKind::SdkLibs => { + format!( + "SDK.libs.{}", + pay.target_arch.map(|ta| ta.as_str()).unwrap_or("all") + ) + } + xwin::PayloadKind::SdkStoreLibs => "SDK.libs.store.all".to_owned(), + xwin::PayloadKind::Ucrt => "SDK.ucrt.all".to_owned(), + }; + + let pb = mp.add( + ProgressBar::with_draw_target(Some(0), draw_target.into()).with_prefix(prefix).with_style( + ProgressStyle::default_bar() + .template("{spinner:.green} {prefix:.bold} [{elapsed}] {wide_bar:.green} {bytes}/{total_bytes} {msg}").unwrap() + .progress_chars("=> "), + ), + ); + xwin::WorkItem { + payload: std::sync::Arc::new(pay), + progress: pb, + } + }) + .collect(); + + mp.set_move_cursor(true); + ctx.execute( + pkgs, + work_items, + pruned.crt_version, + pruned.sdk_version, + arches, + variants, + op, + )?; + + let downloaded_arches: Vec<_> = self + .xwin_options + .xwin_arch + .iter() + .map(|x| x.as_str().to_string()) + .collect(); + fs::write(done_mark_file, downloaded_arches.join(" "))?; + + let dl = cache_dir.join("dl"); + if dl.exists() { + let _ = fs::remove_dir_all(dl); + } + let unpack = cache_dir.join("unpack"); + if unpack.exists() { + let _ = fs::remove_dir_all(unpack); + } + Ok(()) + } + + fn load_manifest( + &self, + ctx: &xwin::Ctx, + dt: ProgressTarget, + ) -> Result { + let manifest_pb = ProgressBar::with_draw_target(Some(0), dt.into()) + .with_style( + ProgressStyle::default_bar() + .template( + "{spinner:.green} {prefix:.bold} [{elapsed}] {wide_bar:.green} {bytes}/{total_bytes} {msg}", + )? + .progress_chars("=> "), + ); + manifest_pb.set_prefix("Manifest"); + manifest_pb.set_message("📥 downloading"); + + let manifest = xwin::manifest::get_manifest( + ctx, + &self.xwin_options.xwin_version, + "release", + manifest_pb.clone(), + )?; + let pkg_manifest = + xwin::manifest::get_package_manifest(ctx, &manifest, manifest_pb.clone())?; + manifest_pb.finish_with_message("📥 downloaded"); + Ok(pkg_manifest) + } + + fn setup_cmake_toolchain(&self, target: &str, xwin_cache_dir: &Path) -> Result { + let cmake_cache_dir = self + .xwin_options + .xwin_cache_dir + .clone() + .unwrap_or_else(|| { + dirs::cache_dir() + // If the really is no cache dir, cwd will also do + .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) + .join(env!("CARGO_PKG_NAME")) + }) + .join("cmake"); + fs::create_dir_all(&cmake_cache_dir)?; + + let override_file = cmake_cache_dir.join("override.cmake"); + fs::write(override_file, include_bytes!("override.cmake"))?; + + let toolchain_file = cmake_cache_dir.join(format!("{}-toolchain.cmake", target)); + let target_arch = target + .split_once('-') + .map(|(x, _)| x) + .context("invalid target triple")?; + let processor = match target_arch { + "i586" | "i686" => "X86", + "x86_64" => "AMD64", + "aarch64" => "ARM64", + _ => target_arch, + }; + let xwin_arch = match target_arch { + "i586" | "i686" => "x86", + _ => target_arch, + }; + + let content = format!( + r#" +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR {processor}) + +set(CMAKE_C_COMPILER clang-cl CACHE FILEPATH "") +set(CMAKE_CXX_COMPILER clang-cl CACHE FILEPATH "") +set(CMAKE_AR llvm-lib) +set(CMAKE_LINKER lld-link CACHE FILEPATH "") + +set(COMPILE_FLAGS + --target={target} + -Wno-unused-command-line-argument + -fuse-ld=lld-link + + /imsvc{xwin_dir}/crt/include + /imsvc{xwin_dir}/sdk/include/ucrt + /imsvc{xwin_dir}/sdk/include/um + /imsvc{xwin_dir}/sdk/include/shared) + +set(LINK_FLAGS + /manifest:no + + -libpath:"{xwin_dir}/crt/lib/{xwin_arch}" + -libpath:"{xwin_dir}/sdk/lib/um/{xwin_arch}" + -libpath:"{xwin_dir}/sdk/lib/ucrt/{xwin_arch}") + +string(REPLACE ";" " " COMPILE_FLAGS "${{COMPILE_FLAGS}}") + +set(_CMAKE_C_FLAGS_INITIAL "${{CMAKE_C_FLAGS}}" CACHE STRING "") +set(CMAKE_C_FLAGS "${{_CMAKE_C_FLAGS_INITIAL}} ${{COMPILE_FLAGS}}" CACHE STRING "" FORCE) + +set(_CMAKE_CXX_FLAGS_INITIAL "${{CMAKE_CXX_FLAGS}}" CACHE STRING "") +set(CMAKE_CXX_FLAGS "${{_CMAKE_CXX_FLAGS_INITIAL}} ${{COMPILE_FLAGS}}" CACHE STRING "" FORCE) + +string(REPLACE ";" " " LINK_FLAGS "${{LINK_FLAGS}}") + +set(_CMAKE_EXE_LINKER_FLAGS_INITIAL "${{CMAKE_EXE_LINKER_FLAGS}}" CACHE STRING "") +set(CMAKE_EXE_LINKER_FLAGS "${{_CMAKE_EXE_LINKER_FLAGS_INITIAL}} ${{LINK_FLAGS}}" CACHE STRING "" FORCE) + +set(_CMAKE_MODULE_LINKER_FLAGS_INITIAL "${{CMAKE_MODULE_LINKER_FLAGS}}" CACHE STRING "") +set(CMAKE_MODULE_LINKER_FLAGS "${{_CMAKE_MODULE_LINKER_FLAGS_INITIAL}} ${{LINK_FLAGS}}" CACHE STRING "" FORCE) + +set(_CMAKE_SHARED_LINKER_FLAGS_INITIAL "${{CMAKE_SHARED_LINKER_FLAGS}}" CACHE STRING "") +set(CMAKE_SHARED_LINKER_FLAGS "${{_CMAKE_SHARED_LINKER_FLAGS_INITIAL}} ${{LINK_FLAGS}}" CACHE STRING "" FORCE) + +# CMake populates these with a bunch of unnecessary libraries, which requires +# extra case-correcting symlinks and what not. Instead, let projects explicitly +# control which libraries they require. +set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE) +set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE) + +set(CMAKE_TRY_COMPILE_CONFIGURATION Release) + +# Allow clang-cl to work with macOS paths. +set(CMAKE_USER_MAKE_RULES_OVERRIDE "${{CMAKE_CURRENT_LIST_DIR}}/override.cmake") + "#, + target = target, + processor = processor, + xwin_dir = adjust_canonicalization(xwin_cache_dir.to_slash_lossy().to_string()), + xwin_arch = xwin_arch, + ); + fs::write(&toolchain_file, content)?; + Ok(toolchain_file) + } +} diff --git a/src/backend/mod.rs b/src/backend/mod.rs new file mode 100644 index 0000000..2b30d58 --- /dev/null +++ b/src/backend/mod.rs @@ -0,0 +1 @@ +pub mod clang_cl; diff --git a/src/override.cmake b/src/backend/override.cmake similarity index 100% rename from src/override.cmake rename to src/backend/override.cmake diff --git a/src/common.rs b/src/common.rs index 42b036a..09cd4c5 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,23 +1,30 @@ -use std::collections::HashSet; -use std::convert::TryInto; use std::env; use std::path::{Path, PathBuf}; use std::process::Command; -use anyhow::{Context, Result}; +use anyhow::Result; use clap::{ builder::{PossibleValuesParser, TypedValueParser as _}, - Parser, + Parser, ValueEnum, }; use fs_err as fs; -use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use path_slash::PathExt; -use which::{which, which_in}; -use xwin::util::ProgressTarget; +use which::which_in; + +/// MSVC cross compiler backend +#[derive(Clone, Debug, Default, ValueEnum)] +pub enum CompilerBackend { + /// clang-cl backend + #[default] + ClangCl, +} /// common xwin options #[derive(Clone, Debug, Parser)] pub struct XWinOptions { + /// The cross compiler backend to use + #[arg(long, env = "XWIN_COMPILER_BACKEND", default_value = "clang-cl")] + pub compiler_backend: CompilerBackend, + /// xwin cache directory #[arg(long, env = "XWIN_CACHE_DIR", hide = true)] pub xwin_cache_dir: Option, @@ -69,6 +76,7 @@ impl Default for XWinOptions { xwin_version: "16".to_string(), xwin_include_debug_libs: false, xwin_include_debug_symbols: false, + compiler_backend: CompilerBackend::ClangCl, } } } @@ -80,438 +88,14 @@ impl XWinOptions { cargo: &cargo_options::CommonOptions, cmd: &mut Command, ) -> Result<()> { - let xwin_cache_dir = self - .xwin_cache_dir - .clone() - .unwrap_or_else(|| { - dirs::cache_dir() - // If the really is no cache dir, cwd will also do - .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) - .join(env!("CARGO_PKG_NAME")) - }) - .join("xwin"); - fs::create_dir_all(&xwin_cache_dir)?; - let xwin_cache_dir = xwin_cache_dir.canonicalize()?; - - let env_path = env::var("PATH").unwrap_or_default(); - let mut env_paths: Vec<_> = env::split_paths(&env_path).collect(); - - let env_path = if cfg!(target_os = "macos") { - let mut new_path = env_path; - new_path.push_str(":/opt/homebrew/opt/llvm/bin"); - new_path.push_str(":/usr/local/opt/llvm/bin"); - new_path - } else { - env_path - }; - let cache_dir = xwin_cache_dir.parent().unwrap(); - env_paths.push(cache_dir.to_path_buf()); - - let workdir = manifest_path - .and_then(|p| p.parent().map(|x| x.to_path_buf())) - .or_else(|| env::current_dir().ok()) - .unwrap(); - let mut targets = cargo.target.clone(); - if targets.is_empty() { - if let Some(build_target) = default_build_target_from_config(&workdir)? { - // if no target is specified, use the default build target - // Note that this is required, otherwise it may fail with link errors - cmd.arg("--target").arg(&build_target); - targets.push(build_target); - } - } - - for target in &targets { - if target.contains("msvc") { - self.setup_msvc_crt(xwin_cache_dir.clone())?; - let env_target = target.to_lowercase().replace('-', "_"); - - if which_in("clang-cl", Some(env_path.clone()), env::current_dir()?).is_err() { - if let Ok(clang) = which("clang") { - #[cfg(windows)] - { - let symlink = cache_dir.join("clang-cl.exe"); - if symlink.exists() { - fs::remove_file(&symlink)?; - } - std::os::windows::fs::symlink_file(clang, symlink)?; - } - - #[cfg(unix)] - { - let symlink = cache_dir.join("clang-cl"); - if symlink.exists() { - fs::remove_file(&symlink)?; - } - std::os::unix::fs::symlink(clang, symlink)?; - } - } - } - symlink_llvm_tool("rust-lld", "lld-link", env_path.clone(), cache_dir)?; - symlink_llvm_tool("llvm-ar", "llvm-lib", env_path.clone(), cache_dir)?; - symlink_llvm_tool("llvm-ar", "llvm-dlltool", env_path.clone(), cache_dir)?; - - cmd.env("TARGET_CC", "clang-cl"); - cmd.env("TARGET_CXX", "clang-cl"); - cmd.env(format!("CC_{}", env_target), "clang-cl"); - cmd.env(format!("CXX_{}", env_target), "clang-cl"); - cmd.env("TARGET_AR", "llvm-lib"); - cmd.env(format!("AR_{}", env_target), "llvm-lib"); - - cmd.env( - format!("CARGO_TARGET_{}_LINKER", env_target.to_uppercase()), - "lld-link", - ); - - let user_set_cl_flags = env::var("CL_FLAGS").unwrap_or_default(); - let user_set_c_flags = env::var("CFLAGS").unwrap_or_default(); - let user_set_cxx_flags = env::var("CXXFLAGS").unwrap_or_default(); - - let xwin_dir = adjust_canonicalization(xwin_cache_dir.to_slash_lossy().to_string()); - let cl_flags = format!( - "--target={target} -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc{dir}/crt/include /imsvc{dir}/sdk/include/ucrt /imsvc{dir}/sdk/include/um /imsvc{dir}/sdk/include/shared {user_set_cl_flags}", - target = target, - dir = xwin_dir, - user_set_cl_flags = user_set_cl_flags, - ); - cmd.env("CL_FLAGS", &cl_flags); - cmd.env( - format!("CFLAGS_{}", env_target), - &format!( - "{cl_flags} {user_set_c_flags}", - cl_flags = cl_flags, - user_set_c_flags = user_set_c_flags - ), - ); - cmd.env( - format!("CXXFLAGS_{}", env_target), - &format!( - "{cl_flags} {user_set_cxx_flags}", - cl_flags = cl_flags, - user_set_cxx_flags = user_set_cxx_flags - ), - ); - - cmd.env( - format!("BINDGEN_EXTRA_CLANG_ARGS_{}", env_target), - format!( - "-I{dir}/crt/include -I{dir}/sdk/include/ucrt -I{dir}/sdk/include/um -I{dir}/sdk/include/shared", - dir = xwin_dir - ) - ); - - cmd.env( - "RCFLAGS", - format!( - "-I{dir}/crt/include -I{dir}/sdk/include/ucrt -I{dir}/sdk/include/um -I{dir}/sdk/include/shared", - dir = xwin_dir - ) - ); - - let target_arch = target - .split_once('-') - .map(|(x, _)| x) - .context("invalid target triple")?; - let xwin_arch = match target_arch { - "i586" | "i686" => "x86", - _ => target_arch, - }; - - let mut rustflags = get_rustflags(&workdir, target)?.unwrap_or_default(); - rustflags - .flags - .extend(["-C".to_string(), "linker-flavor=lld-link".to_string()]); - rustflags.push(format!( - "-Lnative={dir}/crt/lib/{arch}", - dir = xwin_dir, - arch = xwin_arch - )); - rustflags.push(format!( - "-Lnative={dir}/sdk/lib/um/{arch}", - dir = xwin_dir, - arch = xwin_arch - )); - rustflags.push(format!( - "-Lnative={dir}/sdk/lib/ucrt/{arch}", - dir = xwin_dir, - arch = xwin_arch - )); - cmd.env("CARGO_ENCODED_RUSTFLAGS", rustflags.encode()?); - - #[cfg(target_os = "macos")] - { - let usr_llvm = "/usr/local/opt/llvm/bin".into(); - let opt_llvm = "/opt/homebrew/opt/llvm/bin".into(); - if cfg!(target_arch = "x86_64") && !env_paths.contains(&usr_llvm) { - env_paths.push(usr_llvm); - } else if cfg!(target_arch = "aarch64") && !env_paths.contains(&opt_llvm) { - env_paths.push(opt_llvm); - } - } - - cmd.env("PATH", env::join_paths(env_paths.clone())?); - - // CMake support - let cmake_toolchain = self.setup_cmake_toolchain(target, &xwin_cache_dir)?; - cmd.env("CMAKE_GENERATOR", "Ninja") - .env("CMAKE_SYSTEM_NAME", "Windows") - .env( - format!("CMAKE_TOOLCHAIN_FILE_{}", env_target), - cmake_toolchain, - ); - } - } - Ok(()) - } - - fn setup_msvc_crt(&self, cache_dir: PathBuf) -> Result<()> { - let done_mark_file = cache_dir.join("DONE"); - let xwin_arches: HashSet<_> = self - .xwin_arch - .iter() - .map(|x| x.as_str().to_string()) - .collect(); - let mut downloaded_arches = HashSet::new(); - if let Ok(content) = fs::read_to_string(&done_mark_file) { - for arch in content.split_whitespace() { - downloaded_arches.insert(arch.to_string()); - } - } - if xwin_arches.difference(&downloaded_arches).next().is_none() { - return Ok(()); - } - - let draw_target = ProgressTarget::Stdout; - - let agent = http_agent()?; - let xwin_dir = adjust_canonicalization(cache_dir.to_slash_lossy().to_string()); - // timeout defaults to 60s - let ctx = xwin::Ctx::with_dir(xwin::PathBuf::from(xwin_dir), draw_target, agent)?; - let ctx = std::sync::Arc::new(ctx); - let pkg_manifest = self.load_manifest(&ctx, draw_target)?; - - let arches = self - .xwin_arch - .iter() - .fold(0, |acc, arch| acc | *arch as u32); - let variants = self - .xwin_variant - .iter() - .fold(0, |acc, var| acc | *var as u32); - let pruned = xwin::prune_pkg_list(&pkg_manifest, arches, variants, false, None, None)?; - let op = xwin::Ops::Splat(xwin::SplatConfig { - include_debug_libs: self.xwin_include_debug_libs, - include_debug_symbols: self.xwin_include_debug_symbols, - enable_symlinks: !cfg!(target_os = "macos"), - preserve_ms_arch_notation: false, - use_winsysroot_style: false, - copy: false, - output: cache_dir.clone().try_into()?, - map: None, - }); - let pkgs = pkg_manifest.packages; - - let mp = MultiProgress::with_draw_target(draw_target.into()); - let work_items: Vec<_> = pruned.payloads - .into_iter() - .map(|pay| { - let prefix = match pay.kind { - xwin::PayloadKind::CrtHeaders => "CRT.headers".to_owned(), - xwin::PayloadKind::AtlHeaders => "ATL.headers".to_owned(), - xwin::PayloadKind::CrtLibs => { - format!( - "CRT.libs.{}.{}", - pay.target_arch.map(|ta| ta.as_str()).unwrap_or("all"), - pay.variant.map(|v| v.as_str()).unwrap_or("none") - ) - } - xwin::PayloadKind::AtlLibs => { - format!( - "ATL.libs.{}", - pay.target_arch.map(|ta| ta.as_str()).unwrap_or("all"), - ) - } - xwin::PayloadKind::SdkHeaders => { - format!( - "SDK.headers.{}.{}", - pay.target_arch.map(|v| v.as_str()).unwrap_or("all"), - pay.variant.map(|v| v.as_str()).unwrap_or("none") - ) - } - xwin::PayloadKind::SdkLibs => { - format!( - "SDK.libs.{}", - pay.target_arch.map(|ta| ta.as_str()).unwrap_or("all") - ) - } - xwin::PayloadKind::SdkStoreLibs => "SDK.libs.store.all".to_owned(), - xwin::PayloadKind::Ucrt => "SDK.ucrt.all".to_owned(), - }; - - let pb = mp.add( - ProgressBar::with_draw_target(Some(0), draw_target.into()).with_prefix(prefix).with_style( - ProgressStyle::default_bar() - .template("{spinner:.green} {prefix:.bold} [{elapsed}] {wide_bar:.green} {bytes}/{total_bytes} {msg}").unwrap() - .progress_chars("=> "), - ), - ); - xwin::WorkItem { - payload: std::sync::Arc::new(pay), - progress: pb, + match self.compiler_backend { + CompilerBackend::ClangCl => { + let clang_cl = crate::backend::clang_cl::ClangCl::new(self); + clang_cl.apply_command_env(manifest_path, cargo, cmd)?; } - }) - .collect(); - - mp.set_move_cursor(true); - ctx.execute( - pkgs, - work_items, - pruned.crt_version, - pruned.sdk_version, - arches, - variants, - op, - )?; - - let downloaded_arches: Vec<_> = self - .xwin_arch - .iter() - .map(|x| x.as_str().to_string()) - .collect(); - fs::write(done_mark_file, downloaded_arches.join(" "))?; - - let dl = cache_dir.join("dl"); - if dl.exists() { - let _ = fs::remove_dir_all(dl); - } - let unpack = cache_dir.join("unpack"); - if unpack.exists() { - let _ = fs::remove_dir_all(unpack); } Ok(()) } - - fn load_manifest( - &self, - ctx: &xwin::Ctx, - dt: ProgressTarget, - ) -> Result { - let manifest_pb = ProgressBar::with_draw_target(Some(0), dt.into()) - .with_style( - ProgressStyle::default_bar() - .template( - "{spinner:.green} {prefix:.bold} [{elapsed}] {wide_bar:.green} {bytes}/{total_bytes} {msg}", - )? - .progress_chars("=> "), - ); - manifest_pb.set_prefix("Manifest"); - manifest_pb.set_message("📥 downloading"); - - let manifest = - xwin::manifest::get_manifest(ctx, &self.xwin_version, "release", manifest_pb.clone())?; - let pkg_manifest = - xwin::manifest::get_package_manifest(ctx, &manifest, manifest_pb.clone())?; - manifest_pb.finish_with_message("📥 downloaded"); - Ok(pkg_manifest) - } - - fn setup_cmake_toolchain(&self, target: &str, xwin_cache_dir: &Path) -> Result { - let cmake_cache_dir = self - .xwin_cache_dir - .clone() - .unwrap_or_else(|| { - dirs::cache_dir() - // If the really is no cache dir, cwd will also do - .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) - .join(env!("CARGO_PKG_NAME")) - }) - .join("cmake"); - fs::create_dir_all(&cmake_cache_dir)?; - - let override_file = cmake_cache_dir.join("override.cmake"); - fs::write(override_file, include_bytes!("override.cmake"))?; - - let toolchain_file = cmake_cache_dir.join(format!("{}-toolchain.cmake", target)); - let target_arch = target - .split_once('-') - .map(|(x, _)| x) - .context("invalid target triple")?; - let processor = match target_arch { - "i586" | "i686" => "X86", - "x86_64" => "AMD64", - "aarch64" => "ARM64", - _ => target_arch, - }; - let xwin_arch = match target_arch { - "i586" | "i686" => "x86", - _ => target_arch, - }; - - let content = format!( - r#" -set(CMAKE_SYSTEM_NAME Windows) -set(CMAKE_SYSTEM_PROCESSOR {processor}) - -set(CMAKE_C_COMPILER clang-cl CACHE FILEPATH "") -set(CMAKE_CXX_COMPILER clang-cl CACHE FILEPATH "") -set(CMAKE_AR llvm-lib) -set(CMAKE_LINKER lld-link CACHE FILEPATH "") - -set(COMPILE_FLAGS - --target={target} - -Wno-unused-command-line-argument - -fuse-ld=lld-link - - /imsvc{xwin_dir}/crt/include - /imsvc{xwin_dir}/sdk/include/ucrt - /imsvc{xwin_dir}/sdk/include/um - /imsvc{xwin_dir}/sdk/include/shared) - -set(LINK_FLAGS - /manifest:no - - -libpath:"{xwin_dir}/crt/lib/{xwin_arch}" - -libpath:"{xwin_dir}/sdk/lib/um/{xwin_arch}" - -libpath:"{xwin_dir}/sdk/lib/ucrt/{xwin_arch}") - -string(REPLACE ";" " " COMPILE_FLAGS "${{COMPILE_FLAGS}}") - -set(_CMAKE_C_FLAGS_INITIAL "${{CMAKE_C_FLAGS}}" CACHE STRING "") -set(CMAKE_C_FLAGS "${{_CMAKE_C_FLAGS_INITIAL}} ${{COMPILE_FLAGS}}" CACHE STRING "" FORCE) - -set(_CMAKE_CXX_FLAGS_INITIAL "${{CMAKE_CXX_FLAGS}}" CACHE STRING "") -set(CMAKE_CXX_FLAGS "${{_CMAKE_CXX_FLAGS_INITIAL}} ${{COMPILE_FLAGS}}" CACHE STRING "" FORCE) - -string(REPLACE ";" " " LINK_FLAGS "${{LINK_FLAGS}}") - -set(_CMAKE_EXE_LINKER_FLAGS_INITIAL "${{CMAKE_EXE_LINKER_FLAGS}}" CACHE STRING "") -set(CMAKE_EXE_LINKER_FLAGS "${{_CMAKE_EXE_LINKER_FLAGS_INITIAL}} ${{LINK_FLAGS}}" CACHE STRING "" FORCE) - -set(_CMAKE_MODULE_LINKER_FLAGS_INITIAL "${{CMAKE_MODULE_LINKER_FLAGS}}" CACHE STRING "") -set(CMAKE_MODULE_LINKER_FLAGS "${{_CMAKE_MODULE_LINKER_FLAGS_INITIAL}} ${{LINK_FLAGS}}" CACHE STRING "" FORCE) - -set(_CMAKE_SHARED_LINKER_FLAGS_INITIAL "${{CMAKE_SHARED_LINKER_FLAGS}}" CACHE STRING "") -set(CMAKE_SHARED_LINKER_FLAGS "${{_CMAKE_SHARED_LINKER_FLAGS_INITIAL}} ${{LINK_FLAGS}}" CACHE STRING "" FORCE) - -# CMake populates these with a bunch of unnecessary libraries, which requires -# extra case-correcting symlinks and what not. Instead, let projects explicitly -# control which libraries they require. -set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE) -set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE) - -set(CMAKE_TRY_COMPILE_CONFIGURATION Release) - -# Allow clang-cl to work with macOS paths. -set(CMAKE_USER_MAKE_RULES_OVERRIDE "${{CMAKE_CURRENT_LIST_DIR}}/override.cmake") - "#, - target = target, - processor = processor, - xwin_dir = adjust_canonicalization(xwin_cache_dir.to_slash_lossy().to_string()), - xwin_arch = xwin_arch, - ); - fs::write(&toolchain_file, content)?; - Ok(toolchain_file) - } } #[cfg(target_family = "unix")] @@ -529,7 +113,7 @@ pub fn adjust_canonicalization(p: String) -> String { } } -fn rustc_target_bin_dir() -> Result { +pub fn rustc_target_bin_dir() -> Result { let output = Command::new("rustc") .args(["--print", "target-libdir"]) .output()?; @@ -540,7 +124,7 @@ fn rustc_target_bin_dir() -> Result { } /// Symlink Rust provided llvm tool component -fn symlink_llvm_tool( +pub fn symlink_llvm_tool( tool: &str, link_name: &str, env_path: String, @@ -572,7 +156,7 @@ fn symlink_llvm_tool( Ok(()) } -fn default_build_target_from_config(workdir: &Path) -> Result> { +pub fn default_build_target_from_config(workdir: &Path) -> Result> { let output = Command::new("cargo") .current_dir(workdir) .args([ @@ -598,7 +182,7 @@ fn default_build_target_from_config(workdir: &Path) -> Result> { /// /// 1. `RUSTFLAGS` environment variable. /// 2. `rustflags` cargo configuration -fn get_rustflags(workdir: &Path, target: &str) -> Result> { +pub fn get_rustflags(workdir: &Path, target: &str) -> Result> { let cargo_config = cargo_config2::Config::load_with_cwd(workdir)?; let rustflags = cargo_config.rustflags(target)?; Ok(rustflags) @@ -612,7 +196,7 @@ fn tls_ca_bundle() -> Option { } #[cfg(all(feature = "native-tls", not(feature = "rustls")))] -fn http_agent() -> Result { +pub fn http_agent() -> Result { use std::fs::File; use std::io; use std::sync::Arc; @@ -631,7 +215,7 @@ fn http_agent() -> Result { } #[cfg(feature = "rustls")] -fn http_agent() -> Result { +pub fn http_agent() -> Result { use std::fs::File; use std::io; use std::sync::Arc; @@ -652,7 +236,7 @@ fn http_agent() -> Result { } #[cfg(not(any(feature = "native-tls", feature = "rustls")))] -fn http_agent() -> Result { +pub fn http_agent() -> Result { let builder = ureq::builder().try_proxy_from_env(true); Ok(builder.build()) } diff --git a/src/lib.rs b/src/lib.rs index 9776a5c..7570b05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +mod backend; mod common; mod macros; mod run; From ab11dca03b2f9499183c4b2d8cea4a3323bbcd87 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 14:25:21 +0800 Subject: [PATCH 2/7] feat: add a clang backend --- Cargo.lock | 69 ++++++++++++- Cargo.toml | 26 ++++- src/backend/clang.rs | 208 ++++++++++++++++++++++++++++++++++++++++ src/backend/clang_cl.rs | 20 +--- src/backend/mod.rs | 1 + src/common.rs | 6 ++ 6 files changed, 307 insertions(+), 23 deletions(-) create mode 100644 src/backend/clang.rs diff --git a/Cargo.lock b/Cargo.lock index 3b662ec..c78e1d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,10 +182,14 @@ dependencies = [ "path-slash", "rustls", "rustls-pemfile", + "serde", + "serde_json", + "tar", "tracing-subscriber", "ureq", "which", "xwin", + "xz2", ] [[package]] @@ -458,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -467,6 +471,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "flate2" version = "1.0.35" @@ -781,6 +797,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags", "libc", + "redox_syscall", ] [[package]] @@ -817,6 +834,17 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "lzxd" version = "0.2.5" @@ -1216,7 +1244,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1455,6 +1483,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tar" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.14.0" @@ -1465,7 +1504,7 @@ dependencies = [ "fastrand", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1728,6 +1767,8 @@ dependencies = [ "once_cell", "rustls", "rustls-pki-types", + "serde", + "serde_json", "socks", "url", "webpki-roots", @@ -1919,7 +1960,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -2103,6 +2144,17 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "xwin" version = "0.6.5" @@ -2137,6 +2189,15 @@ dependencies = [ "zip", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index ce97d32..721433c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,23 +18,41 @@ pkg-url = "{ repo }/releases/download/v{ version }/{ name }-v{ version }.{ targe anyhow = "1.0.53" cargo-config2 = "0.1.4" cargo-options = "0.7.1" -clap = { version = "4.3.0", features = ["derive", "env", "wrap_help", "unstable-styles"] } +clap = { version = "4.3.0", features = [ + "derive", + "env", + "wrap_help", + "unstable-styles", +] } dirs = "5.0.0" fs-err = "3.0.0" indicatif = "0.17.2" native-tls-crate = { package = "native-tls", version = "0.2.11", optional = true } paste = "1.0.12" path-slash = "0.2.0" -rustls = { version = "0.23.10", default-features = false, features = ["std", "tls12", "logging", "ring"], optional = true } +rustls = { version = "0.23.10", default-features = false, features = [ + "std", + "tls12", + "logging", + "ring", +], optional = true } rustls-pemfile = { version = "2.0.0", optional = true } +serde = { version = "1.0.216", features = ["derive"] } +serde_json = "1.0.133" +tar = "0.4.43" tracing-subscriber = { version = "0.3.17", features = ["fmt"] } -ureq = { version = "2.11.0", default-features = false, features = ["gzip", "socks-proxy"] } +ureq = { version = "2.11.0", default-features = false, features = [ + "gzip", + "json", + "socks-proxy", +] } which = "7.0.0" xwin = { version = "0.6.3", default-features = false } +xz2 = "0.1.7" [features] # By default we use rustls for TLS -default = ["rustls-tls"] +default = ["rustls-tls", "xz2/static"] rustls-tls = ["ureq/tls", "rustls", "rustls-pemfile"] # If this feature is enabled we instead use the native TLS implementation for the # target platform diff --git a/src/backend/clang.rs b/src/backend/clang.rs new file mode 100644 index 0000000..3bde688 --- /dev/null +++ b/src/backend/clang.rs @@ -0,0 +1,208 @@ +use std::env; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{Context, Result}; +use fs_err as fs; +use path_slash::PathExt; +use serde::Deserialize; + +use crate::common::{ + adjust_canonicalization, default_build_target_from_config, get_rustflags, http_agent, + symlink_llvm_tool, XWinOptions, +}; + +const MSVC_SYSROOT_REPOSITORY: &str = "trcrsired/windows-msvc-sysroot"; +const MSVC_SYSROOT_ASSET_NAME: &str = "windows-msvc-sysroot.tar.xz"; + +#[derive(Debug)] +pub struct Clang<'a> { + xwin_options: &'a XWinOptions, +} + +impl<'a> Clang<'a> { + pub fn new(xwin_options: &'a XWinOptions) -> Self { + Self { xwin_options } + } + + pub fn apply_command_env( + &self, + manifest_path: Option<&Path>, + cargo: &cargo_options::CommonOptions, + cmd: &mut Command, + ) -> Result<()> { + let cache_dir = self.xwin_options.xwin_cache_dir.clone().unwrap_or_else(|| { + dirs::cache_dir() + // If the really is no cache dir, cwd will also do + .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) + .join(env!("CARGO_PKG_NAME")) + }); + fs::create_dir_all(&cache_dir)?; + let cache_dir = cache_dir.canonicalize()?; + + let env_path = env::var("PATH").unwrap_or_default(); + let mut env_paths: Vec<_> = env::split_paths(&env_path).collect(); + + let env_path = if cfg!(target_os = "macos") { + let mut new_path = env_path; + new_path.push_str(":/opt/homebrew/opt/llvm/bin"); + new_path.push_str(":/usr/local/opt/llvm/bin"); + new_path + } else { + env_path + }; + env_paths.push(cache_dir.clone()); + + let workdir = manifest_path + .and_then(|p| p.parent().map(|x| x.to_path_buf())) + .or_else(|| env::current_dir().ok()) + .unwrap(); + let mut targets = cargo.target.clone(); + if targets.is_empty() { + if let Some(build_target) = default_build_target_from_config(&workdir)? { + // if no target is specified, use the default build target + // Note that this is required, otherwise it may fail with link errors + cmd.arg("--target").arg(&build_target); + targets.push(build_target); + } + } + + for target in &targets { + if target.contains("msvc") { + let msvc_sysroot_dir = self.setup_msvc_sysroot(cache_dir.clone())?; + // x86_64-pc-windows-msvc -> x86_64-windows-msvc + let target_no_vendor = target.replace("-pc-", "-"); + let target_unknown_vendor = target.replace("-pc-", "-unknown-"); + let env_target = target.to_lowercase().replace('-', "_"); + + symlink_llvm_tool("rust-lld", "lld-link", env_path.clone(), &cache_dir)?; + symlink_llvm_tool("llvm-ar", "llvm-lib", env_path.clone(), &cache_dir)?; + symlink_llvm_tool("llvm-ar", "llvm-dlltool", env_path.clone(), &cache_dir)?; + + cmd.env("TARGET_CC", "clang"); + cmd.env("TARGET_CXX", "clang++"); + cmd.env(format!("CC_{}", env_target), "clang"); + cmd.env(format!("CXX_{}", env_target), "clang++"); + cmd.env("TARGET_AR", "llvm-lib"); + cmd.env(format!("AR_{}", env_target), "llvm-lib"); + cmd.env( + format!("CARGO_TARGET_{}_LINKER", env_target.to_uppercase()), + "lld-link", + ); + + let user_set_c_flags = env::var("CFLAGS").unwrap_or_default(); + let user_set_cxx_flags = env::var("CXXFLAGS").unwrap_or_default(); + let sysroot_dir = + adjust_canonicalization(msvc_sysroot_dir.to_slash_lossy().to_string()); + let clang_flags = format!( + "--target={target_no_vendor} -fuse-ld=lld-link -I{dir}/include -I{dir}/include/c++/stl -L{dir}/lib/{target_unknown_vendor}", + dir = sysroot_dir, + ); + cmd.env( + format!("CFLAGS_{env_target}"), + &format!("{clang_flags} {user_set_c_flags}",), + ); + cmd.env( + format!("CXXFLAGS_{env_target}"), + &format!("{clang_flags} {user_set_cxx_flags}",), + ); + cmd.env( + format!("BINDGEN_EXTRA_CLANG_ARGS_{env_target}"), + format!("-I{dir}/include -I{dir}/include/c++/stl", dir = sysroot_dir), + ); + cmd.env( + "RCFLAGS", + format!("-I{dir}/include -I{dir}/include/c++/stl", dir = sysroot_dir), + ); + + let mut rustflags = get_rustflags(&workdir, target)?.unwrap_or_default(); + rustflags + .flags + .extend(["-C".to_string(), "linker-flavor=lld-link".to_string()]); + rustflags.push(format!( + "-Lnative={dir}/lib/{target_unknown_vendor}", + dir = sysroot_dir, + )); + cmd.env("CARGO_ENCODED_RUSTFLAGS", rustflags.encode()?); + + #[cfg(target_os = "macos")] + { + let usr_llvm = "/usr/local/opt/llvm/bin".into(); + let opt_llvm = "/opt/homebrew/opt/llvm/bin".into(); + if cfg!(target_arch = "x86_64") && !env_paths.contains(&usr_llvm) { + env_paths.push(usr_llvm); + } else if cfg!(target_arch = "aarch64") && !env_paths.contains(&opt_llvm) { + env_paths.push(opt_llvm); + } + } + + cmd.env("PATH", env::join_paths(env_paths.clone())?); + + // TODO: CMake support + } + } + Ok(()) + } + + fn setup_msvc_sysroot(&self, cache_dir: PathBuf) -> Result { + let msvc_sysroot_dir = cache_dir.join("windows-msvc-sysroot"); + if msvc_sysroot_dir.is_dir() { + // Already downloaded and unpacked + return Ok(msvc_sysroot_dir); + } + + let agent = http_agent()?; + let gh_token = env::var("GITHUB_TOKEN").ok(); + // fetch release info to get download url + let mut request = agent + .get(&format!( + "https://api.github.com/repos/{}/releases/latest", + MSVC_SYSROOT_REPOSITORY + )) + .set("X-GitHub-Api-Version", "2022-11-28"); + if let Some(token) = &gh_token { + request = request.set("Authorization", &format!("Bearer {token}")); + } + let response = request.call().context("Failed to get GitHub release")?; + let release: GitHubRelease = response + .into_json() + .context("Failed to deserialize GitHub release")?; + let asset = release + .assets + .iter() + .find(|x| x.name == MSVC_SYSROOT_ASSET_NAME) + .with_context(|| { + format!("Failed to find {MSVC_SYSROOT_ASSET_NAME} in GitHub release") + })?; + let download_url = &asset.browser_download_url; + self.download_msvc_sysroot(&cache_dir, agent, download_url) + .context("Failed to unpack msvc sysroot")?; + Ok(msvc_sysroot_dir) + } + + fn download_msvc_sysroot( + &self, + cache_dir: &Path, + agent: ureq::Agent, + download_url: &str, + ) -> Result<()> { + use xz2::read::XzDecoder; + + let response = agent.get(download_url).call()?; + let tar = XzDecoder::new(response.into_reader()); + let mut archive = tar::Archive::new(tar); + archive.unpack(cache_dir)?; + Ok(()) + } +} + +#[derive(Debug, Deserialize)] +struct GitHubRelease { + assets: Vec, +} + +#[derive(Debug, Deserialize)] +struct GitHubReleaseAsset { + browser_download_url: String, + name: String, +} diff --git a/src/backend/clang_cl.rs b/src/backend/clang_cl.rs index 6b80777..31a2baa 100644 --- a/src/backend/clang_cl.rs +++ b/src/backend/clang_cl.rs @@ -123,30 +123,20 @@ impl<'a> ClangCl<'a> { let xwin_dir = adjust_canonicalization(xwin_cache_dir.to_slash_lossy().to_string()); let cl_flags = format!( "--target={target} -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc{dir}/crt/include /imsvc{dir}/sdk/include/ucrt /imsvc{dir}/sdk/include/um /imsvc{dir}/sdk/include/shared {user_set_cl_flags}", - target = target, dir = xwin_dir, - user_set_cl_flags = user_set_cl_flags, ); cmd.env("CL_FLAGS", &cl_flags); cmd.env( - format!("CFLAGS_{}", env_target), - &format!( - "{cl_flags} {user_set_c_flags}", - cl_flags = cl_flags, - user_set_c_flags = user_set_c_flags - ), + format!("CFLAGS_{env_target}"), + &format!("{cl_flags} {user_set_c_flags}",), ); cmd.env( - format!("CXXFLAGS_{}", env_target), - &format!( - "{cl_flags} {user_set_cxx_flags}", - cl_flags = cl_flags, - user_set_cxx_flags = user_set_cxx_flags - ), + format!("CXXFLAGS_{env_target}"), + &format!("{cl_flags} {user_set_cxx_flags}",), ); cmd.env( - format!("BINDGEN_EXTRA_CLANG_ARGS_{}", env_target), + format!("BINDGEN_EXTRA_CLANG_ARGS_{env_target}"), format!( "-I{dir}/crt/include -I{dir}/sdk/include/ucrt -I{dir}/sdk/include/um -I{dir}/sdk/include/shared", dir = xwin_dir diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 2b30d58..aa2447a 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -1 +1,2 @@ +pub mod clang; pub mod clang_cl; diff --git a/src/common.rs b/src/common.rs index 09cd4c5..e183bf5 100644 --- a/src/common.rs +++ b/src/common.rs @@ -16,6 +16,8 @@ pub enum CompilerBackend { /// clang-cl backend #[default] ClangCl, + /// clang backend + Clang, } /// common xwin options @@ -93,6 +95,10 @@ impl XWinOptions { let clang_cl = crate::backend::clang_cl::ClangCl::new(self); clang_cl.apply_command_env(manifest_path, cargo, cmd)?; } + CompilerBackend::Clang => { + let clang = crate::backend::clang::Clang::new(self); + clang.apply_command_env(manifest_path, cargo, cmd)?; + } } Ok(()) } From 2be11a76d80cd5ee77d497e52d9c75b6ffe9b1ef Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 15:43:02 +0800 Subject: [PATCH 3/7] Rename `CompilerBackend` to `CrossCompiler` --- .github/workflows/CI.yml | 4 ++++ src/common.rs | 22 +++++++++++----------- src/{backend => compiler}/clang.rs | 0 src/{backend => compiler}/clang_cl.rs | 0 src/{backend => compiler}/mod.rs | 0 src/{backend => compiler}/override.cmake | 0 src/lib.rs | 2 +- 7 files changed, 16 insertions(+), 12 deletions(-) rename src/{backend => compiler}/clang.rs (100%) rename src/{backend => compiler}/clang_cl.rs (100%) rename src/{backend => compiler}/mod.rs (100%) rename src/{backend => compiler}/override.cmake (100%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 968a00e..c055218 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -27,6 +27,10 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] toolchain: [stable, nightly] + cross-compiler: [clang-cl, clang] + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + XWIN_CROSS_COMPILER: ${{ matrix.cross-compiler }} steps: - uses: actions/checkout@v4 - uses: actions/checkout@v4 diff --git a/src/common.rs b/src/common.rs index e183bf5..067d483 100644 --- a/src/common.rs +++ b/src/common.rs @@ -10,9 +10,9 @@ use clap::{ use fs_err as fs; use which::which_in; -/// MSVC cross compiler backend +/// MSVC cross compiler #[derive(Clone, Debug, Default, ValueEnum)] -pub enum CompilerBackend { +pub enum CrossCompiler { /// clang-cl backend #[default] ClangCl, @@ -23,9 +23,9 @@ pub enum CompilerBackend { /// common xwin options #[derive(Clone, Debug, Parser)] pub struct XWinOptions { - /// The cross compiler backend to use - #[arg(long, env = "XWIN_COMPILER_BACKEND", default_value = "clang-cl")] - pub compiler_backend: CompilerBackend, + /// The cross compiler to use + #[arg(long, env = "XWIN_CROSS_COMPILER", default_value = "clang-cl")] + pub cross_compiler: CrossCompiler, /// xwin cache directory #[arg(long, env = "XWIN_CACHE_DIR", hide = true)] @@ -78,7 +78,7 @@ impl Default for XWinOptions { xwin_version: "16".to_string(), xwin_include_debug_libs: false, xwin_include_debug_symbols: false, - compiler_backend: CompilerBackend::ClangCl, + cross_compiler: CrossCompiler::ClangCl, } } } @@ -90,13 +90,13 @@ impl XWinOptions { cargo: &cargo_options::CommonOptions, cmd: &mut Command, ) -> Result<()> { - match self.compiler_backend { - CompilerBackend::ClangCl => { - let clang_cl = crate::backend::clang_cl::ClangCl::new(self); + match self.cross_compiler { + CrossCompiler::ClangCl => { + let clang_cl = crate::compiler::clang_cl::ClangCl::new(self); clang_cl.apply_command_env(manifest_path, cargo, cmd)?; } - CompilerBackend::Clang => { - let clang = crate::backend::clang::Clang::new(self); + CrossCompiler::Clang => { + let clang = crate::compiler::clang::Clang::new(self); clang.apply_command_env(manifest_path, cargo, cmd)?; } } diff --git a/src/backend/clang.rs b/src/compiler/clang.rs similarity index 100% rename from src/backend/clang.rs rename to src/compiler/clang.rs diff --git a/src/backend/clang_cl.rs b/src/compiler/clang_cl.rs similarity index 100% rename from src/backend/clang_cl.rs rename to src/compiler/clang_cl.rs diff --git a/src/backend/mod.rs b/src/compiler/mod.rs similarity index 100% rename from src/backend/mod.rs rename to src/compiler/mod.rs diff --git a/src/backend/override.cmake b/src/compiler/override.cmake similarity index 100% rename from src/backend/override.cmake rename to src/compiler/override.cmake diff --git a/src/lib.rs b/src/lib.rs index 7570b05..3247003 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ -mod backend; mod common; +mod compiler; mod macros; mod run; mod test; From 0f6a1068cd53e3dc821091d6da332cc18a14efa3 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 15:58:21 +0800 Subject: [PATCH 4/7] Try to add cmake support --- src/compiler/clang.rs | 65 +++++++++++++++++++++++++++++++++++++++- src/compiler/clang_cl.rs | 3 +- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs index 3bde688..139eac6 100644 --- a/src/compiler/clang.rs +++ b/src/compiler/clang.rs @@ -138,7 +138,14 @@ impl<'a> Clang<'a> { cmd.env("PATH", env::join_paths(env_paths.clone())?); - // TODO: CMake support + // CMake support + let cmake_toolchain = self.setup_cmake_toolchain(target, &sysroot_dir)?; + cmd.env("CMAKE_GENERATOR", "Ninja") + .env("CMAKE_SYSTEM_NAME", "Windows") + .env( + format!("CMAKE_TOOLCHAIN_FILE_{}", env_target), + cmake_toolchain, + ); } } Ok(()) @@ -194,6 +201,62 @@ impl<'a> Clang<'a> { archive.unpack(cache_dir)?; Ok(()) } + + fn setup_cmake_toolchain(&self, target: &str, sysroot_dir: &str) -> Result { + // x86_64-pc-windows-msvc -> x86_64-windows-msvc + let target_no_vendor = target.replace("-pc-", "-"); + let target_unknown_vendor = target.replace("-pc-", "-unknown-"); + let cmake_cache_dir = self + .xwin_options + .xwin_cache_dir + .clone() + .unwrap_or_else(|| { + dirs::cache_dir() + // If the really is no cache dir, cwd will also do + .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) + .join(env!("CARGO_PKG_NAME")) + }) + .join("cmake") + .join("clang"); + fs::create_dir_all(&cmake_cache_dir)?; + + let toolchain_file = cmake_cache_dir.join(format!("{}-toolchain.cmake", target)); + let target_arch = target + .split_once('-') + .map(|(x, _)| x) + .context("invalid target triple")?; + let processor = match target_arch { + "i586" | "i686" => "X86", + "x86_64" => "AMD64", + "aarch64" => "ARM64", + _ => target_arch, + }; + + let content = format!( + r#" +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR {processor}) + +set(CMAKE_C_COMPILER clang CACHE FILEPATH "") +set(CMAKE_CXX_COMPILER clang++ CACHE FILEPATH "") +set(CMAKE_LINKER lld-link CACHE FILEPATH "") +set(CMAKE_RC_COMPILER llvm-rc CACHE FILEPATH "") + +set(COMPILE_FLAGS + --target={target_no_vendor} + -fuse-ld=lld-link + -I{dir}/include + -I{dir}/include/c++/stl) + +set(LINK_FLAGS + /manifest:no + -libpath:"{dir}/lib/{target_unknown_vendor}") + "#, + dir = sysroot_dir, + ); + fs::write(&toolchain_file, content)?; + Ok(toolchain_file) + } } #[derive(Debug, Deserialize)] diff --git a/src/compiler/clang_cl.rs b/src/compiler/clang_cl.rs index 31a2baa..b8ccff9 100644 --- a/src/compiler/clang_cl.rs +++ b/src/compiler/clang_cl.rs @@ -377,7 +377,8 @@ impl<'a> ClangCl<'a> { .unwrap_or_else(|| env::current_dir().expect("Failed to get current dir")) .join(env!("CARGO_PKG_NAME")) }) - .join("cmake"); + .join("cmake") + .join("clang-cl"); fs::create_dir_all(&cmake_cache_dir)?; let override_file = cmake_cache_dir.join("override.cmake"); From bf8301a01741a542ea6bb39b70a2394e06cd1498 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 16:28:06 +0800 Subject: [PATCH 5/7] Try use lld instead of lld-link when using clang --- src/compiler/clang.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs index 139eac6..d27ccab 100644 --- a/src/compiler/clang.rs +++ b/src/compiler/clang.rs @@ -75,7 +75,7 @@ impl<'a> Clang<'a> { let target_unknown_vendor = target.replace("-pc-", "-unknown-"); let env_target = target.to_lowercase().replace('-', "_"); - symlink_llvm_tool("rust-lld", "lld-link", env_path.clone(), &cache_dir)?; + symlink_llvm_tool("rust-lld", "lld", env_path.clone(), &cache_dir)?; symlink_llvm_tool("llvm-ar", "llvm-lib", env_path.clone(), &cache_dir)?; symlink_llvm_tool("llvm-ar", "llvm-dlltool", env_path.clone(), &cache_dir)?; @@ -87,7 +87,7 @@ impl<'a> Clang<'a> { cmd.env(format!("AR_{}", env_target), "llvm-lib"); cmd.env( format!("CARGO_TARGET_{}_LINKER", env_target.to_uppercase()), - "lld-link", + "lld", ); let user_set_c_flags = env::var("CFLAGS").unwrap_or_default(); @@ -95,7 +95,7 @@ impl<'a> Clang<'a> { let sysroot_dir = adjust_canonicalization(msvc_sysroot_dir.to_slash_lossy().to_string()); let clang_flags = format!( - "--target={target_no_vendor} -fuse-ld=lld-link -I{dir}/include -I{dir}/include/c++/stl -L{dir}/lib/{target_unknown_vendor}", + "--target={target_no_vendor} -fuse-ld=lld -I{dir}/include -I{dir}/include/c++/stl -L{dir}/lib/{target_unknown_vendor}", dir = sysroot_dir, ); cmd.env( @@ -116,9 +116,6 @@ impl<'a> Clang<'a> { ); let mut rustflags = get_rustflags(&workdir, target)?.unwrap_or_default(); - rustflags - .flags - .extend(["-C".to_string(), "linker-flavor=lld-link".to_string()]); rustflags.push(format!( "-Lnative={dir}/lib/{target_unknown_vendor}", dir = sysroot_dir, @@ -234,17 +231,18 @@ impl<'a> Clang<'a> { let content = format!( r#" +cmake_minimum_required(VERSION 3.29) set(CMAKE_SYSTEM_NAME Windows) set(CMAKE_SYSTEM_PROCESSOR {processor}) set(CMAKE_C_COMPILER clang CACHE FILEPATH "") set(CMAKE_CXX_COMPILER clang++ CACHE FILEPATH "") -set(CMAKE_LINKER lld-link CACHE FILEPATH "") set(CMAKE_RC_COMPILER llvm-rc CACHE FILEPATH "") +set(CMAKE_LINKER_TYPE LLD CACHE STRING "") set(COMPILE_FLAGS --target={target_no_vendor} - -fuse-ld=lld-link + -fuse-ld=lld -I{dir}/include -I{dir}/include/c++/stl) From e5d2f665d82b13de6d0708930798a80f08512a2f Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 16:54:13 +0800 Subject: [PATCH 6/7] Revert "Try use lld instead of lld-link when using clang" This reverts commit bf8301a01741a542ea6bb39b70a2394e06cd1498. It still tries to find `lld-link` ``` clang: error: unable to execute command: Executable "lld-link" doesn't exist! clang: error: linker command failed with exit code 1 (use -v to see invocation) ninja: build stopped: subcommand failed. ``` --- src/compiler/clang.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs index d27ccab..139eac6 100644 --- a/src/compiler/clang.rs +++ b/src/compiler/clang.rs @@ -75,7 +75,7 @@ impl<'a> Clang<'a> { let target_unknown_vendor = target.replace("-pc-", "-unknown-"); let env_target = target.to_lowercase().replace('-', "_"); - symlink_llvm_tool("rust-lld", "lld", env_path.clone(), &cache_dir)?; + symlink_llvm_tool("rust-lld", "lld-link", env_path.clone(), &cache_dir)?; symlink_llvm_tool("llvm-ar", "llvm-lib", env_path.clone(), &cache_dir)?; symlink_llvm_tool("llvm-ar", "llvm-dlltool", env_path.clone(), &cache_dir)?; @@ -87,7 +87,7 @@ impl<'a> Clang<'a> { cmd.env(format!("AR_{}", env_target), "llvm-lib"); cmd.env( format!("CARGO_TARGET_{}_LINKER", env_target.to_uppercase()), - "lld", + "lld-link", ); let user_set_c_flags = env::var("CFLAGS").unwrap_or_default(); @@ -95,7 +95,7 @@ impl<'a> Clang<'a> { let sysroot_dir = adjust_canonicalization(msvc_sysroot_dir.to_slash_lossy().to_string()); let clang_flags = format!( - "--target={target_no_vendor} -fuse-ld=lld -I{dir}/include -I{dir}/include/c++/stl -L{dir}/lib/{target_unknown_vendor}", + "--target={target_no_vendor} -fuse-ld=lld-link -I{dir}/include -I{dir}/include/c++/stl -L{dir}/lib/{target_unknown_vendor}", dir = sysroot_dir, ); cmd.env( @@ -116,6 +116,9 @@ impl<'a> Clang<'a> { ); let mut rustflags = get_rustflags(&workdir, target)?.unwrap_or_default(); + rustflags + .flags + .extend(["-C".to_string(), "linker-flavor=lld-link".to_string()]); rustflags.push(format!( "-Lnative={dir}/lib/{target_unknown_vendor}", dir = sysroot_dir, @@ -231,18 +234,17 @@ impl<'a> Clang<'a> { let content = format!( r#" -cmake_minimum_required(VERSION 3.29) set(CMAKE_SYSTEM_NAME Windows) set(CMAKE_SYSTEM_PROCESSOR {processor}) set(CMAKE_C_COMPILER clang CACHE FILEPATH "") set(CMAKE_CXX_COMPILER clang++ CACHE FILEPATH "") +set(CMAKE_LINKER lld-link CACHE FILEPATH "") set(CMAKE_RC_COMPILER llvm-rc CACHE FILEPATH "") -set(CMAKE_LINKER_TYPE LLD CACHE STRING "") set(COMPILE_FLAGS --target={target_no_vendor} - -fuse-ld=lld + -fuse-ld=lld-link -I{dir}/include -I{dir}/include/c++/stl) From 71e4ddbc38282435ed896d3328c294bac6cdc969 Mon Sep 17 00:00:00 2001 From: messense Date: Sat, 21 Dec 2024 17:00:45 +0800 Subject: [PATCH 7/7] Add a hard coded fallback download url for msvc sysroot --- src/compiler/clang.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs index 139eac6..85c5dd5 100644 --- a/src/compiler/clang.rs +++ b/src/compiler/clang.rs @@ -14,6 +14,7 @@ use crate::common::{ const MSVC_SYSROOT_REPOSITORY: &str = "trcrsired/windows-msvc-sysroot"; const MSVC_SYSROOT_ASSET_NAME: &str = "windows-msvc-sysroot.tar.xz"; +const FALLBACK_DOWNLOAD_URL: &str = "https://github.com/trcrsired/windows-msvc-sysroot/releases/download/20241217/windows-msvc-sysroot.tar.xz"; #[derive(Debug)] pub struct Clang<'a> { @@ -159,15 +160,26 @@ impl<'a> Clang<'a> { } let agent = http_agent()?; - let gh_token = env::var("GITHUB_TOKEN").ok(); // fetch release info to get download url + let download_url = self + .get_latest_msvc_sysroot_download_url(agent.clone()) + .unwrap_or_else(|_| FALLBACK_DOWNLOAD_URL.to_string()); + self.download_msvc_sysroot(&cache_dir, agent, &download_url) + .context("Failed to unpack msvc sysroot")?; + Ok(msvc_sysroot_dir) + } + + fn get_latest_msvc_sysroot_download_url(&self, agent: ureq::Agent) -> Result { + if let Ok(url) = env::var("XWIN_MSVC_SYSROOT_DOWNLOAD_URL") { + return Ok(url); + } let mut request = agent .get(&format!( "https://api.github.com/repos/{}/releases/latest", MSVC_SYSROOT_REPOSITORY )) .set("X-GitHub-Api-Version", "2022-11-28"); - if let Some(token) = &gh_token { + if let Ok(token) = env::var("GITHUB_TOKEN") { request = request.set("Authorization", &format!("Bearer {token}")); } let response = request.call().context("Failed to get GitHub release")?; @@ -181,10 +193,8 @@ impl<'a> Clang<'a> { .with_context(|| { format!("Failed to find {MSVC_SYSROOT_ASSET_NAME} in GitHub release") })?; - let download_url = &asset.browser_download_url; - self.download_msvc_sysroot(&cache_dir, agent, download_url) - .context("Failed to unpack msvc sysroot")?; - Ok(msvc_sysroot_dir) + let download_url = asset.browser_download_url.clone(); + Ok(download_url) } fn download_msvc_sysroot(