From 3e2fb690aca11d010b892b2f5ad6e46f22345a09 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Fri, 24 May 2024 14:50:35 -0700 Subject: [PATCH] Support zstd-compressed ELF sections. zstd has been introduced as an alternative to zlib for the compression of debug sections.[0] Toolchain support is widely present at this time but lack of support in backtrace is a severe limitation on using this feature in Rust programs. This uses a Rust reimplementation of zstd (the ruzstd crate). This has the benefit of simplifying the build process, but this crate is less used and admittedly slower than the zstd crate that binds to the C libzstd. [0] https://maskray.me/blog/2022-09-09-zstd-compressed-debug-sections --- .github/workflows/main.yml | 16 ++++++++++++++ Cargo.lock | 42 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + crates/as-if-std/Cargo.toml | 3 ++- src/symbolize/gimli/elf.rs | 35 +++++++++++++++++++++++-------- 5 files changed, 86 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ed2863c..ca8a4b81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,6 +22,12 @@ jobs: rust: beta - os: ubuntu-20.04 rust: nightly + - os: ubuntu-24.04 + rust: stable + - os: ubuntu-24.04 + rust: beta + - os: ubuntu-24.04 + rust: nightly - os: macos-latest rust: stable - os: macos-latest @@ -48,6 +54,12 @@ jobs: - run: echo RUSTFLAGS=-Dwarnings >> $GITHUB_ENV shell: bash + # Starting with Ubuntu 22.04 libc6-dbg is needed. + - name: Install libc debug info + run: sudo apt-get install -y libc6-dbg + shell: bash + if: contains(matrix.os, 'ubuntu-24.04') + # full fidelity of backtraces on 32-bit msvc requires frame pointers, so # enable that for our tests - name: Force frame pointers @@ -85,6 +97,10 @@ jobs: if: contains(matrix.os, 'ubuntu') env: RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib-gnu" + - run: cargo test + if: contains(matrix.os, 'ubuntu-24.04') + env: + RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zstd" # Test that, on macOS, packed/unpacked debuginfo both work - run: cargo clean && cargo test diff --git a/Cargo.lock b/Cargo.lock index 52472f32..b0c9ecf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", + "ruzstd", ] [[package]] @@ -44,10 +45,17 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", + "ruzstd", "serde", "winapi", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" version = "1.0.90" @@ -77,6 +85,17 @@ dependencies = [ "cc", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "dylib-dep" version = "0.1.0" @@ -151,6 +170,16 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "ruzstd" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5174a470eeb535a721ae9fdd6e291c2411a906b96592182d05217591d5c5cf7b" +dependencies = [ + "byteorder", + "derive_more", +] + [[package]] name = "serde" version = "1.0.188" @@ -168,7 +197,18 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.29", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bf8cf869..2228c8f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ cpp_demangle = { default-features = false, version = "0.4.0", optional = true, f [target.'cfg(not(all(windows, target_env = "msvc", not(target_vendor = "uwp"))))'.dependencies] miniz_oxide = { version = "0.7.0", default-features = false } +ruzstd = { version = "0.6.0", default-features = false } addr2line = { version = "0.22.0", default-features = false } libc = { version = "0.2.146", default-features = false } diff --git a/crates/as-if-std/Cargo.toml b/crates/as-if-std/Cargo.toml index b451c024..8f5c8055 100644 --- a/crates/as-if-std/Cargo.toml +++ b/crates/as-if-std/Cargo.toml @@ -18,6 +18,7 @@ libc = { version = "0.2.146", default-features = false } [target.'cfg(not(all(windows, target_env = "msvc", not(target_vendor = "uwp"))))'.dependencies] miniz_oxide = { version = "0.7.0", optional = true, default-features = false } +ruzstd = { version = "0.6.0", optional = true, default-features = false } addr2line = { version = "0.22.0", optional = true, default-features = false } [target.'cfg(not(all(windows, target_env = "msvc", not(target_vendor = "uwp"))))'.dependencies.object] @@ -32,7 +33,7 @@ cc = "1.0.90" [features] default = ['backtrace'] -backtrace = ['addr2line', 'miniz_oxide', 'object'] +backtrace = ['addr2line', 'miniz_oxide', 'object', 'ruzstd'] std = [] [lints.rust] diff --git a/src/symbolize/gimli/elf.rs b/src/symbolize/gimli/elf.rs index 906a3005..056d42f6 100644 --- a/src/symbolize/gimli/elf.rs +++ b/src/symbolize/gimli/elf.rs @@ -7,7 +7,9 @@ use super::{gimli, Context, Endian, EndianSlice, Mapping, Stash, Vec}; use alloc::sync::Arc; use core::convert::{TryFrom, TryInto}; use core::str; -use object::elf::{ELFCOMPRESS_ZLIB, ELF_NOTE_GNU, NT_GNU_BUILD_ID, SHF_COMPRESSED}; +use object::elf::{ + ELFCOMPRESS_ZLIB, ELFCOMPRESS_ZSTD, ELF_NOTE_GNU, NT_GNU_BUILD_ID, SHF_COMPRESSED, +}; use object::read::elf::{CompressionHeader, FileHeader, SectionHeader, SectionTable, Sym}; use object::read::StringTable; use object::{BigEndian, Bytes, NativeEndian}; @@ -170,7 +172,8 @@ impl<'a> Object<'a> { let mut data = Bytes(section.data(self.endian, self.data).ok()?); // Check for DWARF-standard (gABI) compression, i.e., as generated - // by ld's `--compress-debug-sections=zlib-gabi` flag. + // by ld's `--compress-debug-sections=zlib-gabi` and + // `--compress-debug-sections=zstd` flags. let flags: u64 = section.sh_flags(self.endian).into(); if (flags & u64::from(SHF_COMPRESSED)) == 0 { // Not compressed. @@ -178,14 +181,21 @@ impl<'a> Object<'a> { } let header = data.read::<::CompressionHeader>().ok()?; - if header.ch_type(self.endian) != ELFCOMPRESS_ZLIB { - // Zlib compression is the only known type. - return None; + match header.ch_type(self.endian) { + ELFCOMPRESS_ZLIB => { + let size = usize::try_from(header.ch_size(self.endian)).ok()?; + let buf = stash.allocate(size); + decompress_zlib(data.0, buf)?; + return Some(buf); + } + ELFCOMPRESS_ZSTD => { + let size = usize::try_from(header.ch_size(self.endian)).ok()?; + let buf = stash.allocate(size); + decompress_zstd(data.0, buf)?; + return Some(buf); + } + _ => return None, // Unknown compression type. } - let size = usize::try_from(header.ch_size(self.endian)).ok()?; - let buf = stash.allocate(size); - decompress_zlib(data.0, buf)?; - return Some(buf); } // Check for the nonstandard GNU compression format, i.e., as generated @@ -304,6 +314,13 @@ fn decompress_zlib(input: &[u8], output: &mut [u8]) -> Option<()> { } } +fn decompress_zstd(input: &[u8], output: &mut [u8]) -> Option<()> { + use ruzstd::io::Read; + + let mut decoder = ruzstd::StreamingDecoder::new(input).ok()?; + decoder.read_exact(output).ok() +} + const DEBUG_PATH: &[u8] = b"/usr/lib/debug"; fn debug_path_exists() -> bool {