From be0260611849d7090facfc354642061e53ead288 Mon Sep 17 00:00:00 2001 From: Douglas Dwyer Date: Sun, 1 Dec 2024 23:14:10 -0500 Subject: [PATCH] Static linking for DXC via mach-dxcompiler (#6574) --- CHANGELOG.md | 1 + Cargo.lock | 7 + Cargo.toml | 1 + README.md | 2 +- wgpu-hal/Cargo.toml | 3 + wgpu-hal/examples/ray-traced-triangle/main.rs | 2 +- wgpu-hal/src/dx12/instance.rs | 25 +++- wgpu-hal/src/dx12/shader_compilation.rs | 138 ++++++++++++------ wgpu-types/src/lib.rs | 5 +- wgpu/Cargo.toml | 12 ++ wgpu/src/util/init.rs | 4 +- 11 files changed, 150 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f60dff7eb1..23741c882a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,7 @@ By @ErichDonGubler in [#6456](https://github.com/gfx-rs/wgpu/pull/6456), [#6148] - Return submission index in `map_async` and `on_submitted_work_done` to track down completion of async callbacks. By @eliemichel in [#6360](https://github.com/gfx-rs/wgpu/pull/6360). - Move raytracing alignments into HAL instead of in core. By @Vecvec in [#6563](https://github.com/gfx-rs/wgpu/pull/6563). +- Allow for statically linking DXC rather than including separate `.dll` files. By @DouglasDwyer in [#6574](https://github.com/gfx-rs/wgpu/pull/6574). ### Changes diff --git a/Cargo.lock b/Cargo.lock index 1990c31cdd..c521a5ed0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1778,6 +1778,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "mach-dxcompiler-rs" +version = "0.1.2+2024.11.22-df583a3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a30a333be477f592794a1c1c5739b1a64021ec0e5fb10a1bb5f0feab5b913f5f" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -3692,6 +3698,7 @@ dependencies = [ "libc", "libloading", "log", + "mach-dxcompiler-rs", "metal", "naga", "ndk-sys", diff --git a/Cargo.toml b/Cargo.toml index 24b57c5c36..db2f6f0ae8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,7 @@ gpu-descriptor = "0.3" bit-set = "0.8" gpu-allocator = { version = "0.27", default-features = false } range-alloc = "0.1" +mach-dxcompiler-rs = { version = "0.1.2", default-features = false } windows-core = { version = "0.58", default-features = false } # Gles dependencies diff --git a/README.md b/README.md index bdd587b573..fb571bd016 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ All testing and example infrastructure share the same set of environment variabl - `WGPU_ADAPTER_NAME` with a substring of the name of the adapter you want to use (ex. `1080` will match `NVIDIA GeForce 1080ti`). - `WGPU_BACKEND` with a comma-separated list of the backends you want to use (`vulkan`, `metal`, `dx12`, or `gl`). - `WGPU_POWER_PREF` with the power preference to choose when a specific adapter name isn't specified (`high`, `low` or `none`) -- `WGPU_DX12_COMPILER` with the DX12 shader compiler you wish to use (`dxc` or `fxc`, note that `dxc` requires `dxil.dll` and `dxcompiler.dll` to be in the working directory otherwise it will fall back to `fxc`) +- `WGPU_DX12_COMPILER` with the DX12 shader compiler you wish to use (`dxc`, `static-dxc`, or `fxc`). Note that `dxc` requires `dxil.dll` and `dxcompiler.dll` to be in the working directory, and `static-dxc` requires the `static-dxc` crate feature to be enabled. Otherwise, it will fall back to `fxc`. - `WGPU_GLES_MINOR_VERSION` with the minor OpenGL ES 3 version number to request (`0`, `1`, `2` or `automatic`). - `WGPU_ALLOW_UNDERLYING_NONCOMPLIANT_ADAPTER` with a boolean whether non-compliant drivers are enumerated (`0` for false, `1` for true). diff --git a/wgpu-hal/Cargo.toml b/wgpu-hal/Cargo.toml index e4a2943af1..b175b9905a 100644 --- a/wgpu-hal/Cargo.toml +++ b/wgpu-hal/Cargo.toml @@ -91,6 +91,8 @@ dx12 = [ "windows/Win32_System_Threading", "windows/Win32_UI_WindowsAndMessaging", ] +## Enables statically linking DXC. +static-dxc = ["dep:mach-dxcompiler-rs"] renderdoc = ["dep:libloading", "dep:renderdoc-sys"] fragile-send-sync-non-atomic-wasm = ["wgt/fragile-send-sync-non-atomic-wasm"] # Panic when running into an out-of-memory error (for debugging purposes). @@ -158,6 +160,7 @@ windows = { workspace = true, optional = true } bit-set = { workspace = true, optional = true } range-alloc = { workspace = true, optional = true } gpu-allocator = { workspace = true, optional = true } +mach-dxcompiler-rs = { workspace = true, optional = true } # For core macros. This crate is also reexported as windows::core. windows-core = { workspace = true, optional = true } diff --git a/wgpu-hal/examples/ray-traced-triangle/main.rs b/wgpu-hal/examples/ray-traced-triangle/main.rs index 7212988f48..6754dc36a9 100644 --- a/wgpu-hal/examples/ray-traced-triangle/main.rs +++ b/wgpu-hal/examples/ray-traced-triangle/main.rs @@ -239,7 +239,7 @@ impl Example { let instance_desc = hal::InstanceDescriptor { name: "example", flags: wgt::InstanceFlags::default(), - dx12_shader_compiler: wgt::Dx12Compiler::Dxc { + dx12_shader_compiler: wgt::Dx12Compiler::DynamicDxc { dxil_path: None, dxc_path: None, }, diff --git a/wgpu-hal/src/dx12/instance.rs b/wgpu-hal/src/dx12/instance.rs index 31d0511d39..389843a21c 100644 --- a/wgpu-hal/src/dx12/instance.rs +++ b/wgpu-hal/src/dx12/instance.rs @@ -76,14 +76,29 @@ impl crate::Instance for super::Instance { // Initialize DXC shader compiler let dxc_container = match desc.dx12_shader_compiler.clone() { - wgt::Dx12Compiler::Dxc { + wgt::Dx12Compiler::DynamicDxc { dxil_path, dxc_path, } => { - let container = super::shader_compilation::get_dxc_container(dxc_path, dxil_path) - .map_err(|e| { - crate::InstanceError::with_source(String::from("Failed to load DXC"), e) - })?; + let container = + super::shader_compilation::get_dynamic_dxc_container(dxc_path, dxil_path) + .map_err(|e| { + crate::InstanceError::with_source( + String::from("Failed to load dynamic DXC"), + e, + ) + })?; + + container.map(Arc::new) + } + wgt::Dx12Compiler::StaticDxc => { + let container = + super::shader_compilation::get_static_dxc_container().map_err(|e| { + crate::InstanceError::with_source( + String::from("Failed to load static DXC"), + e, + ) + })?; container.map(Arc::new) } diff --git a/wgpu-hal/src/dx12/shader_compilation.rs b/wgpu-hal/src/dx12/shader_compilation.rs index b95ccaa5a5..96865b9080 100644 --- a/wgpu-hal/src/dx12/shader_compilation.rs +++ b/wgpu-hal/src/dx12/shader_compilation.rs @@ -97,7 +97,10 @@ struct DxcLib { } impl DxcLib { - fn new(lib_path: Option, lib_name: &'static str) -> Result { + fn new_dynamic( + lib_path: Option, + lib_name: &'static str, + ) -> Result { let lib_path = if let Some(lib_path) = lib_path { if lib_path.is_file() { lib_path @@ -111,37 +114,54 @@ impl DxcLib { } pub fn create_instance(&self) -> Result { - type Fun = extern "system" fn( - rclsid: *const windows_core::GUID, - riid: *const windows_core::GUID, - ppv: *mut *mut core::ffi::c_void, - ) -> windows_core::HRESULT; - let func: libloading::Symbol = unsafe { self.lib.get(b"DxcCreateInstance\0") }?; - - let mut result__ = None; - (func)(&T::CLSID, &T::IID, <*mut _>::cast(&mut result__)) - .ok() - .into_device_result("DxcCreateInstance")?; - result__.ok_or(crate::DeviceError::Unexpected) + unsafe { + type DxcCreateInstanceFn = unsafe extern "system" fn( + rclsid: *const windows_core::GUID, + riid: *const windows_core::GUID, + ppv: *mut *mut core::ffi::c_void, + ) + -> windows_core::HRESULT; + + let func: libloading::Symbol = + self.lib.get(b"DxcCreateInstance\0")?; + dxc_create_instance::(|clsid, iid, ppv| func(clsid, iid, ppv)) + } } } +/// Invokes the provided library function to create a DXC object. +unsafe fn dxc_create_instance( + f: impl Fn( + *const windows_core::GUID, + *const windows_core::GUID, + *mut *mut core::ffi::c_void, + ) -> windows_core::HRESULT, +) -> Result { + let mut result__ = None; + f(&T::CLSID, &T::IID, <*mut _>::cast(&mut result__)) + .ok() + .into_device_result("DxcCreateInstance")?; + result__.ok_or(crate::DeviceError::Unexpected) +} + // Destructor order should be fine since _dxil and _dxc don't rely on each other. pub(super) struct DxcContainer { compiler: Dxc::IDxcCompiler3, utils: Dxc::IDxcUtils, - validator: Dxc::IDxcValidator, + validator: Option, // Has to be held onto for the lifetime of the device otherwise shaders will fail to compile. - _dxc: DxcLib, + // Only needed when using dynamic linking. + _dxc: Option, // Also Has to be held onto for the lifetime of the device otherwise shaders will fail to validate. - _dxil: DxcLib, + // Only needed when using dynamic linking. + _dxil: Option, } -pub(super) fn get_dxc_container( +pub(super) fn get_dynamic_dxc_container( dxc_path: Option, dxil_path: Option, ) -> Result, crate::DeviceError> { - let dxc = match DxcLib::new(dxc_path, "dxcompiler.dll") { + let dxc = match DxcLib::new_dynamic(dxc_path, "dxcompiler.dll") { Ok(dxc) => dxc, Err(e) => { log::warn!( @@ -153,7 +173,7 @@ pub(super) fn get_dxc_container( } }; - let dxil = match DxcLib::new(dxil_path, "dxil.dll") { + let dxil = match DxcLib::new_dynamic(dxil_path, "dxil.dll") { Ok(dxil) => dxil, Err(e) => { log::warn!( @@ -172,12 +192,47 @@ pub(super) fn get_dxc_container( Ok(Some(DxcContainer { compiler, utils, - validator, - _dxc: dxc, - _dxil: dxil, + validator: Some(validator), + _dxc: Some(dxc), + _dxil: Some(dxil), })) } +/// Creates a [`DxcContainer`] that delegates to the statically-linked version of DXC. +pub(super) fn get_static_dxc_container() -> Result, crate::DeviceError> { + #[cfg(feature = "static-dxc")] + { + unsafe { + let compiler = dxc_create_instance::(|clsid, iid, ppv| { + windows_core::HRESULT(mach_dxcompiler_rs::DxcCreateInstance( + clsid.cast(), + iid.cast(), + ppv, + )) + })?; + let utils = dxc_create_instance::(|clsid, iid, ppv| { + windows_core::HRESULT(mach_dxcompiler_rs::DxcCreateInstance( + clsid.cast(), + iid.cast(), + ppv, + )) + })?; + + Ok(Some(DxcContainer { + compiler, + utils, + validator: None, + _dxc: None, + _dxil: None, + })) + } + } + #[cfg(not(feature = "static-dxc"))] + { + panic!("Attempted to create a static DXC shader compiler, but the static-dxc feature was not enabled") + } +} + /// Owned PCWSTR #[allow(clippy::upper_case_acronyms)] struct OPCWSTR { @@ -245,9 +300,12 @@ pub(super) fn compile_dxc( windows::core::w!("2018"), // Use HLSL 2018, Naga doesn't supported 2021 yet. windows::core::w!("-no-warnings"), Dxc::DXC_ARG_ENABLE_STRICTNESS, - Dxc::DXC_ARG_SKIP_VALIDATION, // Disable implicit validation to work around bugs when dxil.dll isn't in the local directory. ]); + if dxc_container.validator.is_some() { + compile_args.push(Dxc::DXC_ARG_SKIP_VALIDATION); // Disable implicit validation to work around bugs when dxil.dll isn't in the local directory.) + } + if device .private_caps .instance_flags @@ -288,26 +346,24 @@ pub(super) fn compile_dxc( let blob = get_output::(&compile_res, Dxc::DXC_OUT_OBJECT)?; - let err_blob = { - let res = unsafe { - dxc_container - .validator - .Validate(&blob, Dxc::DxcValidatorFlags_InPlaceEdit) - } - .into_device_result("Validate")?; + if let Some(validator) = &dxc_container.validator { + let err_blob = { + let res = unsafe { validator.Validate(&blob, Dxc::DxcValidatorFlags_InPlaceEdit) } + .into_device_result("Validate")?; - unsafe { res.GetErrorBuffer() }.into_device_result("GetErrorBuffer")? - }; + unsafe { res.GetErrorBuffer() }.into_device_result("GetErrorBuffer")? + }; - let size = unsafe { err_blob.GetBufferSize() }; - if size != 0 { - let err_blob = unsafe { dxc_container.utils.GetBlobAsUtf8(&err_blob) } - .into_device_result("GetBlobAsUtf8")?; - let err = as_err_str(&err_blob)?; - return Err(crate::PipelineError::Linkage( - stage_bit, - format!("DXC validation error: {err}"), - )); + let size = unsafe { err_blob.GetBufferSize() }; + if size != 0 { + let err_blob = unsafe { dxc_container.utils.GetBlobAsUtf8(&err_blob) } + .into_device_result("GetBlobAsUtf8")?; + let err = as_err_str(&err_blob)?; + return Err(crate::PipelineError::Linkage( + stage_bit, + format!("DXC validation error: {err}"), + )); + } } Ok(crate::dx12::CompiledShader::Dxc(blob)) diff --git a/wgpu-types/src/lib.rs b/wgpu-types/src/lib.rs index 17f8e43f34..e56aa31d2f 100644 --- a/wgpu-types/src/lib.rs +++ b/wgpu-types/src/lib.rs @@ -7431,12 +7431,15 @@ pub enum Dx12Compiler { /// Minimum supported version: [v1.5.2010](https://github.com/microsoft/DirectXShaderCompiler/releases/tag/v1.5.2010) /// /// It also requires WDDM 2.1 (Windows 10 version 1607). - Dxc { + DynamicDxc { /// Path to the `dxil.dll` file, or path to the directory containing `dxil.dll` file. Passing `None` will use standard platform specific dll loading rules. dxil_path: Option, /// Path to the `dxcompiler.dll` file, or path to the directory containing `dxcompiler.dll` file. Passing `None` will use standard platform specific dll loading rules. dxc_path: Option, }, + /// The statically-linked variant of Dxc. + /// The `static-dxc` feature is required to use this. + StaticDxc, } /// Selects which OpenGL ES 3 minor version to request. diff --git a/wgpu/Cargo.toml b/wgpu/Cargo.toml index ed630133e2..d05731958d 100644 --- a/wgpu/Cargo.toml +++ b/wgpu/Cargo.toml @@ -117,6 +117,18 @@ fragile-send-sync-non-atomic-wasm = [ "wgt/fragile-send-sync-non-atomic-wasm", ] + +#! ### External libraries +# -------------------------------------------------------------------- +#! The following features facilitate integration with third-party supporting libraries. + +## Enables statically linking DXC. +## Normally, to use the modern DXC shader compiler with WGPU, the final application +## must be shipped alongside `dxcompiler.dll` and `dxil.dll` (which can be downloaded from Microsoft's GitHub). +## This feature statically links a version of DXC so that no external binaries are required +## to compile DX12 shaders. +static-dxc = ["hal/static-dxc"] + # wgpu-core is always available as an optional dependency, "wgc". # Whenever wgpu-core is selected, we want raw window handle support. [dependencies.wgc] diff --git a/wgpu/src/util/init.rs b/wgpu/src/util/init.rs index 573fe3bd80..2419f4be9c 100644 --- a/wgpu/src/util/init.rs +++ b/wgpu/src/util/init.rs @@ -107,10 +107,12 @@ pub fn dx12_shader_compiler_from_env() -> Option { .map(str::to_lowercase) .as_deref() { - Ok("dxc") => wgt::Dx12Compiler::Dxc { + Ok("dxc") => wgt::Dx12Compiler::DynamicDxc { dxil_path: None, dxc_path: None, }, + #[cfg(feature = "static-dxc")] + Ok("static-dxc") => wgt::Dx12Compiler::StaticDxc, Ok("fxc") => wgt::Dx12Compiler::Fxc, _ => return None, },