feat: add 256bit vpclmulqdq support

Cargo.toml

@@ -13,6 +13,7 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+lazy_static = { version = "1.4.0", optional = true }
 
 [dev-dependencies]
 crc = "1"
@@ -22,6 +23,7 @@ rand = "0.8"
 
 [features]
 pmull = []
+vpclmulqdq = ["lazy_static"]
 fake-simd = []
 
 [[bench]]

src/lib.rs

@@ -22,11 +22,20 @@
     feature = "pmull",
     feature(stdsimd, platform_intrinsics, aarch64_target_feature, llvm_asm)
 )]
+#![cfg_attr(
+    feature = "vpclmulqdq",
+    feature(
+        simd_ffi,
+        link_llvm_intrinsics,
+        avx512_target_feature,
+        target_feature_11
+    )
+)]
 
 mod pclmulqdq;
 mod table;
 
-type UpdateFn = fn(u64, &[u8]) -> u64;
+type UpdateFn = unsafe fn(u64, &[u8]) -> u64;
 
 /// Represents an in-progress CRC-64 computation.
 #[derive(Clone)]
@@ -57,7 +66,9 @@ impl Digest {
 
     /// Writes some data into the digest.
     pub fn write(&mut self, bytes: &[u8]) {
-        self.state = (self.computer)(self.state, bytes);
+        unsafe {
+            self.state = (self.computer)(self.state, bytes);
+        }
     }
 
     /// Computes the current CRC-64-ECMA value.

src/pclmulqdq/mod.rs

@@ -7,21 +7,23 @@
 //!
 //! [white paper]: https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
 
+use std::{
+    fmt::Debug,
+    ops::{BitXor, BitXorAssign},
+};
+
+use super::table;
+
+use self::arch::Simd;
+
 #[cfg(not(feature = "fake-simd"))]
-#[cfg_attr(any(target_arch = "x86", target_arch = "x86_64"), path = "x86.rs")]
+#[cfg_attr(any(target_arch = "x86", target_arch = "x86_64"), path = "x86/mod.rs")]
 #[cfg_attr(all(target_arch = "aarch64", feature = "pmull"), path = "aarch64.rs")]
 mod arch;
 
 #[cfg(feature = "fake-simd")]
 mod arch;
 
-use self::arch::Simd;
-use super::table;
-use std::{
-    fmt::Debug,
-    ops::{BitXor, BitXorAssign},
-};
-
 /// This trait must be implemented on `self::arch::Simd` to provide the
 /// platform-specific SIMD implementations.
 trait SimdExt: Copy + Debug + BitXor {
@@ -70,24 +72,47 @@ impl BitXorAssign for Simd {
 }
 
 pub fn get_update() -> super::UpdateFn {
+    #[cfg(all(feature = "vpclmulqdq"))]
+    {
+        use arch::vpclmulqdq::*;
+        if Simd256::is_supported() {
+            return update_256_batch;
+        }
+    }
+
     if Simd::is_supported() {
-        update
+        update_128_batch
     } else {
         table::update
     }
 }
 
-fn update(mut state: u64, bytes: &[u8]) -> u64 {
-    let (left, middle, right) = unsafe { bytes.align_to::<[Simd; 8]>() };
+// This function is unsafe because it uses platform dependent functions.
+unsafe fn update_128_batch(mut state: u64, bytes: &[u8]) -> u64 {
+    let (left, middle, right) = bytes.align_to::<[Simd; 8]>();
     if let Some((first, rest)) = middle.split_first() {
         state = table::update(state, left);
-        state = unsafe { update_simd(state, first, rest) };
+        state = update_simd(state, first, rest);
         table::update(state, right)
     } else {
         table::update(state, bytes)
     }
 }
 
+#[cfg(feature = "vpclmulqdq")]
+#[target_feature(enable = "avx2", enable = "avx512vpclmulqdq")]
+unsafe fn update_256_batch(mut state: u64, bytes: &[u8]) -> u64 {
+    use arch::vpclmulqdq::*;
+    let (left, middle, right) = bytes.align_to::<[[Simd256; 4]; 2]>();
+    if let Some((first, rest)) = middle.split_first() {
+        state = update_128_batch(state, left);
+        state = update_vpclmulqdq(state, first, rest);
+        update_128_batch(state, right)
+    } else {
+        update_128_batch(state, bytes)
+    }
+}
+
 #[cfg_attr(
     any(target_arch = "x86", target_arch = "x86_64"),
     target_feature(enable = "pclmulqdq", enable = "sse2", enable = "sse4.1")
@@ -111,6 +136,11 @@ unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]]) -> u64
         }
     }
 
+    fold_tail(x)
+}
+
+#[inline(always)]
+unsafe fn fold_tail(x: [Simd; 8]) -> u64 {
     let coeffs = [
         Simd::new(table::K_895, table::K_959), // fold by distance of 112 bytes
         Simd::new(table::K_767, table::K_831), // fold by distance of 96 bytes

src/pclmulqdq/x86.rs → src/pclmulqdq/x86/mod.rs

@@ -8,6 +8,9 @@ use std::arch::x86::*;
 use std::arch::x86_64::*;
 use std::ops::BitXor;
 
+#[cfg(all(feature = "vpclmulqdq"))]
+pub mod vpclmulqdq;
+
 #[repr(transparent)]
 #[derive(Copy, Clone, Debug)]
 pub struct Simd(__m128i);
@@ -28,29 +31,29 @@ impl super::SimdExt for Simd {
     #[inline]
     #[target_feature(enable = "sse2", enable = "pclmulqdq")]
     unsafe fn fold_16(self, coeff: Self) -> Self {
-        let h = Self(_mm_clmulepi64_si128(self.0, coeff.0, 0x11));
-        let l = Self(_mm_clmulepi64_si128(self.0, coeff.0, 0x00));
+        let h = Self(_mm_clmulepi64_si128::<0x11>(self.0, coeff.0));
+        let l = Self(_mm_clmulepi64_si128::<0x00>(self.0, coeff.0));
         h ^ l
     }
 
     #[inline]
     #[target_feature(enable = "sse2", enable = "pclmulqdq")]
     unsafe fn fold_8(self, coeff: u64) -> Self {
         let coeff = Self::new(0, coeff);
-        let h = Self(_mm_clmulepi64_si128(self.0, coeff.0, 0x00));
-        let l = Self(_mm_srli_si128(self.0, 8));
+        let h = Self(_mm_clmulepi64_si128::<0x00>(self.0, coeff.0));
+        let l = Self(_mm_srli_si128::<8>(self.0));
         h ^ l
     }
 
     #[inline]
     #[target_feature(enable = "sse2", enable = "sse4.1", enable = "pclmulqdq")]
     unsafe fn barrett(self, poly: u64, mu: u64) -> u64 {
         let polymu = Self::new(poly, mu);
-        let t1 = _mm_clmulepi64_si128(self.0, polymu.0, 0x00);
-        let h = Self(_mm_slli_si128(t1, 8));
-        let l = Self(_mm_clmulepi64_si128(t1, polymu.0, 0x10));
+        let t1 = _mm_clmulepi64_si128::<0x00>(self.0, polymu.0);
+        let h = Self(_mm_slli_si128::<8>(t1));
+        let l = Self(_mm_clmulepi64_si128::<0x10>(t1, polymu.0));
         let reduced = h ^ l ^ self;
-        _mm_extract_epi64(reduced.0, 1) as u64
+        _mm_extract_epi64::<1>(reduced.0) as u64
     }
 }