Mirrors branch main from private at 9c7cceefc844b0fa24e163690bf064b7c54a8435

.cargo/config.toml

@@ -0,0 +1,5 @@
+[build]
+# The following configs are in an increasing performance order for x86_64.
+# rustflags = ["-Ctarget-feature=+avx2"]
+# rustflags = ["-Ctarget-feature=+avx512vl,+avx512f"]
+# rustflags = ["-Ctarget-feature=+avx512vl,+avx512f,+avx512bw,+avx512cd,+avx512dq"]

.gitignore

@@ -1,4 +1,5 @@
 /.vscode/
 
 /target
+/profiling-target/target
 /Cargo.lock

Cargo.toml

@@ -23,10 +23,9 @@ itertools = "0.10"
 blake2 = "0.10"
 sha2 = "0.10"
 num-modular = "0.5.1"
-# packed_simd = { version = "0.3.8", package = "packed_simd_2" }
 packed_simd = { version = "0.3.9" }
 pairing = { package = "pairing_ce", git = "https://github.com/matter-labs/pairing.git" }
-crypto-bigint = "*"
+crypto-bigint = "0.5"
 convert_case = "*"
 firestorm = "*"
 tracing = { version = "0.1.37", optional = true }

benches/benchmarks.rs

@@ -1,3 +1,8 @@
+#![allow(
+    dead_code, // OK for benches
+    clippy::let_unit_value, // False positives
+    clippy::needless_range_loop, // Suggests less readable code
+)]
 #![feature(iter_collect_into)]
 #![feature(allocator_api)]
 
@@ -16,13 +21,13 @@ const MUL_UPPER_BOUND: usize = 1024;
 fn criterion_benchmark_multiplication(c: &mut Criterion) {
     let aa: [GoldilocksField; MUL_UPPER_BOUND] = (0..MUL_UPPER_BOUND)
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect::<Vec<_>>()
         .try_into()
         .unwrap();
     let bb: [GoldilocksField; MUL_UPPER_BOUND] = (0..MUL_UPPER_BOUND)
         .map(|x| x as u64 + MUL_UPPER_BOUND as u64)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect::<Vec<_>>()
         .try_into()
         .unwrap();
@@ -82,7 +87,7 @@ fn criterion_benchmark_poseidon2_statevec(c: &mut Criterion) {
 fn criterion_benchmark_keccak(c: &mut Criterion) {
     let input = [0u8; 64];
     c.bench_function("Keccak256", |b| {
-        b.iter(|| Keccak256::digest(&mut black_box(input)))
+        b.iter(|| Keccak256::digest(black_box(input)))
     });
 }
 
@@ -206,11 +211,11 @@ fn criterion_benchmark_add_vectors_naive(c: &mut Criterion) {
 
     let mut aa: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
     let bb: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
 
     c.bench_function("Naive Vec add", |b| {
@@ -223,11 +228,11 @@ fn criterion_benchmark_add_vectors_vectorized(c: &mut Criterion) {
 
     let aa: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
     let bb: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -247,11 +252,11 @@ fn criterion_benchmark_add_vectors_simd(c: &mut Criterion) {
 
     (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut aa);
     (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut bb);
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -271,11 +276,11 @@ fn criterion_benchmark_add_vectors_portable_simd(c: &mut Criterion) {
 
     (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut aa);
     (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut bb);
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -293,11 +298,11 @@ fn criterion_benchmark_add_vectors_mixedgl(c: &mut Criterion) {
 
     let aa: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
     let bb: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
 
     // let mut aa: Vec<MixedGL> = boojum::utils::cast_check_alignment(aa);
@@ -323,11 +328,11 @@ fn criterion_benchmark_mul_vectors_naive(c: &mut Criterion) {
 
     let aa: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
     let bb: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
 
     let mut aa: Vec<MixedGL> =
@@ -350,11 +355,11 @@ fn criterion_benchmark_mul_vectors_vectorized(c: &mut Criterion) {
 
     let aa: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
     let bb: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -374,11 +379,11 @@ fn criterion_benchmark_mul_vectors_simd(c: &mut Criterion) {
 
     (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut aa);
     (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut bb);
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -398,11 +403,11 @@ fn criterion_benchmark_mul_vectors_portable_simd_long(c: &mut Criterion) {
 
     (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut aa);
     (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut bb);
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -424,11 +429,11 @@ fn criterion_benchmark_mul_vectors_portable_simd(c: &mut Criterion) {
 
     (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut aa);
     (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect_into(&mut bb);
 
     let mut aa = boojum::utils::cast_check_alignment(aa);
@@ -444,11 +449,11 @@ fn criterion_benchmark_mul_vectors_mixedgl(c: &mut Criterion) {
 
     let aa: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 1)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
     let bb: Vec<GoldilocksField> = (0..(degree * 2))
         .map(|x| x as u64 + 2)
-        .map(|el| GoldilocksField::from_u64_with_reduction(el))
+        .map(GoldilocksField::from_u64_with_reduction)
         .collect();
 
     let mut aa: Vec<MixedGL> =

src/algebraic_props/round_function.rs

@@ -176,7 +176,7 @@ pub fn absorb_into_state_vararg<
     if to_absorb.array_chunks::<AW>().remainder().is_empty() == false {
         let mut tmp = [F::ZERO; AW];
         let remainder = to_absorb.array_chunks::<AW>().remainder();
-        tmp[..remainder.len()].copy_from_slice(&remainder);
+        tmp[..remainder.len()].copy_from_slice(remainder);
         T::absorb_into_state::<M>(state, &tmp);
         T::round_function(state);
     }

src/cs/cs_builder.rs

@@ -53,11 +53,11 @@ pub trait CsBuilderImpl<F: SmallField, TImpl> {
 
 pub struct CsBuilder<TImpl, F: SmallField, GC: GateConfigurationHolder<F>, TB: StaticToolboxHolder>
 {
-    pub(crate) phantom: std::marker::PhantomData<F>,
+    pub phantom: std::marker::PhantomData<F>,
 
-    pub(crate) implementation: TImpl,
-    pub(crate) gates_config: GC,
-    pub(crate) toolbox: TB,
+    pub implementation: TImpl,
+    pub gates_config: GC,
+    pub toolbox: TB,
 }
 
 pub fn new_builder<TImpl: CsBuilderImpl<F, TImpl>, F: SmallField>(

src/cs/cs_builder_reference.rs

@@ -444,20 +444,20 @@ impl<F: SmallField, P: PrimeFieldLikeVectorized<Base = F>, CFG: CSConfig>
             evaluation_data_over_specialized_columns,
             specialized_gates_rough_stats: HashMap::with_capacity(16),
             gates_application_sets,
-            copy_permutation_data: copy_permutation_data,
-            witness_placement_data: witness_placement_data,
+            copy_permutation_data,
+            witness_placement_data,
             next_available_row: 0,
             next_available_place_idx: 0,
             next_lookup_table_index: INITIAL_LOOKUP_TABLE_ID_VALUE,
-            lookup_marker_gate_idx: lookup_marker_gate_idx,
-            constants_requested_per_row: constants_requested_per_row,
-            constants_for_gates_in_specialized_mode: constants_for_gates_in_specialized_mode,
+            lookup_marker_gate_idx,
+            constants_requested_per_row,
+            constants_for_gates_in_specialized_mode,
             lookup_table_marker_into_id: HashMap::with_capacity(32),
             lookup_tables: Vec::with_capacity(32),
             lookup_multiplicities: Vec::with_capacity(8),
             table_ids_as_variables: Vec::with_capacity(32),
             public_inputs: Vec::with_capacity(8),
-            max_trace_len: max_trace_len,
+            max_trace_len,
             static_toolbox: builder.toolbox,
             row_cleanups,
             columns_cleanups,

src/cs/cs_builder_verifier.rs

@@ -43,7 +43,7 @@ pub struct CsVerifierBuilder<F: SmallField, EXT: FieldExtension<2, BaseField = F
 impl<F: SmallField, EXT: FieldExtension<2, BaseField = F>> CsVerifierBuilder<F, EXT> {
     pub fn new_from_parameters(parameters: CSGeometry) -> Self {
         Self {
-            parameters: parameters,
+            parameters,
             lookup_parameters: LookupParameters::NoLookup,
 
             gate_type_ids_for_specialized_columns: Vec::with_capacity(16),

src/cs/gates/boolean_allocator.rs

@@ -30,9 +30,7 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BooleanConstraitEvaluator {
     }
 
     #[inline(always)]
-    fn unique_params(&self) -> Self::UniqueParameterizationParams {
-        ()
-    }
+    fn unique_params(&self) -> Self::UniqueParameterizationParams {}
 
     #[inline]
     fn type_name() -> std::borrow::Cow<'static, str> {
@@ -83,7 +81,6 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BooleanConstraitEvaluator {
         &self,
         _ctx: &mut P::Context,
     ) -> Self::GlobalConstants<P> {
-        ()
     }
 
     type RowSharedConstants<P: field::traits::field_like::PrimeFieldLike<Base = F>> = ();
@@ -97,7 +94,6 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BooleanConstraitEvaluator {
         _trace_source: &S,
         _ctx: &mut P::Context,
     ) -> Self::RowSharedConstants<P> {
-        ()
     }
 
     #[inline(always)]
@@ -125,7 +121,7 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BooleanConstraitEvaluator {
     }
 }
 
-const UNIQUE_IDENTIFIER: &'static str = "Boolean constraint gate";
+const UNIQUE_IDENTIFIER: &str = "Boolean constraint gate";
 const PRINCIPAL_WIDTH: usize = 1;
 
 impl<F: SmallField> Gate<F> for BooleanConstraintGate {

src/cs/gates/bounded_boolean_allocator.rs

@@ -88,7 +88,6 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BoundedBooleanConstraitEvalua
         &self,
         _ctx: &mut P::Context,
     ) -> Self::GlobalConstants<P> {
-        ()
     }
 
     type RowSharedConstants<P: field::traits::field_like::PrimeFieldLike<Base = F>> = ();
@@ -102,7 +101,6 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BoundedBooleanConstraitEvalua
         _trace_source: &S,
         _ctx: &mut P::Context,
     ) -> Self::RowSharedConstants<P> {
-        ()
     }
 
     #[inline(always)]
@@ -130,7 +128,7 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BoundedBooleanConstraitEvalua
     }
 }
 
-const UNIQUE_IDENTIFIER: &'static str = "Boolean constraint gate";
+const UNIQUE_IDENTIFIER: &str = "Boolean constraint gate";
 const PRINCIPAL_WIDTH: usize = 1;
 
 // Just keep a position that we can use next

src/cs/gates/bounded_constant_allocator.rs

@@ -98,7 +98,6 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BoundedConstantAllocatorConst
         &self,
         _ctx: &mut P::Context,
     ) -> Self::GlobalConstants<P> {
-        ()
     }
 
     // there are no constants that would be shared between instances
@@ -114,7 +113,6 @@ impl<F: PrimeField> GateConstraintEvaluator<F> for BoundedConstantAllocatorConst
         _trace_source: &S,
         _ctx: &mut P::Context,
     ) -> Self::RowSharedConstants<P> {
-        ()
     }
 
     #[inline(always)]

src/cs/gates/conditional_swap.rs

@@ -38,9 +38,7 @@ impl<F: PrimeField, const N: usize> GateConstraintEvaluator<F>
     }
 
     #[inline(always)]
-    fn unique_params(&self) -> Self::UniqueParameterizationParams {
-        ()
-    }
+    fn unique_params(&self) -> Self::UniqueParameterizationParams {}
 
     #[inline]
     fn type_name() -> std::borrow::Cow<'static, str> {
@@ -91,7 +89,6 @@ impl<F: PrimeField, const N: usize> GateConstraintEvaluator<F>
         &self,
         _ctx: &mut P::Context,
     ) -> Self::GlobalConstants<P> {
-        ()
     }
 
     type RowSharedConstants<P: field::traits::field_like::PrimeFieldLike<Base = F>> = ();
@@ -105,7 +102,6 @@ impl<F: PrimeField, const N: usize> GateConstraintEvaluator<F>
         _trace_source: &S,
         _ctx: &mut P::Context,
     ) -> Self::RowSharedConstants<P> {
-        ()
     }
 
     #[inline(always)]