Skip to content

Commit

Permalink
fix crc32
Browse files Browse the repository at this point in the history
  • Loading branch information
andygrove committed Jun 9, 2024
1 parent ad12c9b commit 35b3585
Showing 1 changed file with 6 additions and 8 deletions.
14 changes: 6 additions & 8 deletions core/src/parquet/util/hash_util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

use byteorder::{LittleEndian, ReadBytesExt};

fn hash_(data: &[u8], seed: u32) -> u32 {
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
unsafe {
Expand Down Expand Up @@ -106,16 +108,12 @@ unsafe fn crc32_hash(bytes: &[u8], seed: u32) -> u32 {
let num_words = num_bytes / u32_num_bytes;
num_bytes %= u32_num_bytes;

let bytes_u32: &[u32] = std::slice::from_raw_parts(
&bytes[0..num_words * u32_num_bytes] as *const [u8] as *const u32,
num_words,
);

let mut offset = 0;
let mut hash = seed;
while offset < num_words {
hash = _mm_crc32_u32(hash, bytes_u32[offset]);
offset += 1;
for _ in 0..num_words {
let mut buf = &bytes[offset..offset + 4];
hash = _mm_crc32_u32(hash, buf.read_u32::<LittleEndian>().unwrap());
offset += 4;
}

offset = num_words * u32_num_bytes;
Expand Down

0 comments on commit 35b3585

Please sign in to comment.