Memory verify (#162)

* Memory verify

* Removing distributed from cargo.toml and changing debug to diagnostic

* Adding diagnostic to all tests

* Updating Cargo.toml

* Cargo.lock update

common/src/distribution_ctx.rs

@@ -43,6 +43,49 @@ pub struct DistributionCtx {
     pub balance_distribution: bool,
 }
 
+impl std::fmt::Debug for DistributionCtx {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        #[cfg(feature = "distributed")]
+        {
+            f.debug_struct("DistributionCtx")
+                .field("rank", &self.rank)
+                .field("n_processes", &self.n_processes)
+                .field("n_instances", &self.n_instances)
+                .field("my_instances", &self.my_instances)
+                .field("instances", &self.instances)
+                .field("instances_owner", &self.instances_owner)
+                .field("owners_count", &self.owners_count)
+                .field("owners_weight", &self.owners_weight)
+                .field("my_groups", &self.my_groups)
+                .field("my_air_groups", &self.my_air_groups)
+                .field("airgroup_instances", &self.airgroup_instances)
+                .field("glob2loc", &self.glob2loc)
+                .field("balance_distribution", &self.balance_distribution)
+                .field("roots_gatherv_count", &self.roots_gatherv_count)
+                .field("roots_gatherv_displ", &self.roots_gatherv_displ)
+                .finish()
+        }
+        #[cfg(not(feature = "distributed"))]
+        {
+            f.debug_struct("DistributionCtx")
+                .field("rank", &self.rank)
+                .field("n_processes", &self.n_processes)
+                .field("n_instances", &self.n_instances)
+                .field("my_instances", &self.my_instances)
+                .field("instances", &self.instances)
+                .field("instances_owner", &self.instances_owner)
+                .field("owners_count", &self.owners_count)
+                .field("owners_weight", &self.owners_weight)
+                .field("my_groups", &self.my_groups)
+                .field("my_air_groups", &self.my_air_groups)
+                .field("airgroup_instances", &self.airgroup_instances)
+                .field("glob2loc", &self.glob2loc)
+                .field("balance_distribution", &self.balance_distribution)
+                .finish()
+        }
+    }
+}
+
 impl DistributionCtx {
     pub fn new() -> Self {
         #[cfg(feature = "distributed")]

common/src/setup.rs

@@ -87,7 +87,7 @@ impl<F: Clone> Setup<F> {
             let recursive = &ProofType::Basic != setup_type;
             let prover_buffer_size = if verify_constraints {
                 let mut mem_instance = 0;
-                for stage in 0..stark_info.n_stages + 1 {
+                for stage in 0..stark_info.n_stages {
                     let n_cols = stark_info.map_sections_n[&format!("cm{}", stage + 1)];
                     mem_instance += n_cols * (1 << (stark_info.stark_struct.n_bits));
                 }

examples/fibonacci-square/Cargo.toml

@@ -24,4 +24,4 @@ num-bigint = "0.4"
 
 [features]
 default = []
-debug = []
+diagnostic = []

hints/src/hints.rs

@@ -822,6 +822,7 @@ pub fn acc_hint_field<F: Field>(
     acc_hint_field_c(
         (&setup.p_setup).into(),
         (&steps_params).into(),
+        pctx.get_buff_helper_ptr(),
         hint_id as u64,
         hint_field_dest,
         hint_field_airgroupvalue,
@@ -870,6 +871,7 @@ pub fn acc_mul_hint_fields<F: Field>(
     acc_mul_hint_fields_c(
         (&setup.p_setup).into(),
         (&steps_params).into(),
+        pctx.get_buff_helper_ptr(),
         hint_id as u64,
         hint_field_dest,
         hint_field_airgroupvalue,

macros/Cargo.toml

@@ -14,4 +14,4 @@ rayon = "1.10"
 
 [features]
 default = []
-debug = []
+diagnostic = []

macros/src/lib.rs

@@ -79,7 +79,7 @@ fn trace_impl(input: TokenStream2) -> Result<TokenStream2> {
                 assert!(num_rows >= 2);
                 assert!(num_rows & (num_rows - 1) == 0);
 
-                let buffer: Vec<#row_struct_name::<#generics>> = if cfg!(feature = "debug") {
+                let buffer: Vec<#row_struct_name::<#generics>> = if cfg!(feature = "diagnostic") {
                     let mut buffer_u64 = vec![u64::MAX - 1; num_rows * #row_struct_name::<#generics>::ROW_SIZE];
 
                     // Convert safely by properly managing size & alignment

pil2-components/test/simple/rs/Cargo.toml

@@ -25,4 +25,5 @@ rayon.workspace = true
 
 [features]
 default = []
-no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-components/test/std/connection/rs/Cargo.toml

@@ -27,4 +27,5 @@ proofman-cli.workspace = true
 
 [features]
 default = []
-no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-components/test/std/diff_buses/rs/Cargo.toml

@@ -25,3 +25,8 @@ rayon.workspace = true
 
 [build-dependencies]
 proofman-cli.workspace = true
+
+[features]
+default = []
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-components/test/std/direct_update/rs/Cargo.toml

@@ -28,3 +28,8 @@ rayon.workspace = true
 
 [build-dependencies]
 proofman-cli.workspace = true
+
+[features]
+default = []
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-components/test/std/lookup/rs/Cargo.toml

@@ -27,4 +27,5 @@ proofman-cli.workspace = true
 
 [features]
 default = []
-no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-components/test/std/permutation/rs/Cargo.toml

@@ -27,4 +27,5 @@ proofman-cli.workspace = true
 
 [features]
 default = []
-no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-components/test/std/range_check/rs/Cargo.toml

@@ -25,3 +25,8 @@ rayon.workspace = true
 
 [build-dependencies]
 proofman-cli.workspace = true
+
+[features]
+default = []
+no_lib_link = ["proofman-common/no_lib_link", "proofman/no_lib_link"]
+diagnostic = ["proofman-macros/diagnostic"]

pil2-stark/lib/include/starks_lib.h

@@ -57,8 +57,8 @@
     uint64_t get_hint_field_values(void *pSetupCtx, uint64_t hintId, char* hintFieldName);
     void get_hint_field_sizes(void *pSetupCtx, void* hintFieldValues, uint64_t hintId, char* hintFieldName, void* hintOptions);
     uint64_t mul_hint_fields(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2); 
-    void acc_hint_field(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName, bool add);
-    void acc_mul_hint_fields(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2,  void* hintOptions1, void *hintOptions2, bool add);
+    void acc_hint_field(void *pSetupCtx, void* stepsParams, void *pBuffHelper, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName, bool add);
+    void acc_mul_hint_fields(void *pSetupCtx, void* stepsParams, void *pBuffHelper, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2,  void* hintOptions1, void *hintOptions2, bool add);
     uint64_t update_airgroupvalue(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2, bool add);
     uint64_t set_hint_field(void *pSetupCtx, void* stepsParams, void *values, uint64_t hintId, char* hintFieldName);
     uint64_t get_hint_id(void *pSetupCtx, uint64_t hintId, char * hintFieldName);

pil2-stark/src/api/starks_api.cpp

@@ -368,12 +368,12 @@ uint64_t mul_hint_fields(void *pSetupCtx, void* stepsParams, uint64_t hintId, ch
     return multiplyHintFields(*(SetupCtx *)pSetupCtx, *(StepsParams *)stepsParams, hintId, string(hintFieldNameDest), string(hintFieldName1), string(hintFieldName2), *(HintFieldOptions *)hintOptions1,  *(HintFieldOptions *)hintOptions2);
 }
 
-void acc_hint_field(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName, bool add) {
-    accHintField(*(SetupCtx *)pSetupCtx, *(StepsParams *)stepsParams, hintId, string(hintFieldNameDest), string(hintFieldNameAirgroupVal), string(hintFieldName), add);
+void acc_hint_field(void *pSetupCtx, void* stepsParams, void *pBuffHelper, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName, bool add) {
+    accHintField(*(SetupCtx *)pSetupCtx, *(StepsParams *)stepsParams, (Goldilocks::Element *)pBuffHelper, hintId, string(hintFieldNameDest), string(hintFieldNameAirgroupVal), string(hintFieldName), add);
 }
 
-void acc_mul_hint_fields(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2, bool add) {
-    accMulHintFields(*(SetupCtx *)pSetupCtx, *(StepsParams *)stepsParams, hintId, string(hintFieldNameDest), string(hintFieldNameAirgroupVal), string(hintFieldName1), string(hintFieldName2),*(HintFieldOptions *)hintOptions1,  *(HintFieldOptions *)hintOptions2, add);
+void acc_mul_hint_fields(void *pSetupCtx, void* stepsParams, void *pBuffHelper, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2, bool add) {
+    accMulHintFields(*(SetupCtx *)pSetupCtx, *(StepsParams *)stepsParams, (Goldilocks::Element *)pBuffHelper, hintId, string(hintFieldNameDest), string(hintFieldNameAirgroupVal), string(hintFieldName1), string(hintFieldName2),*(HintFieldOptions *)hintOptions1,  *(HintFieldOptions *)hintOptions2, add);
 }
 
 uint64_t update_airgroupvalue(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2, bool add) {

pil2-stark/src/api/starks_api.hpp

@@ -57,8 +57,8 @@
     uint64_t get_hint_field_values(void *pSetupCtx, uint64_t hintId, char* hintFieldName);
     void get_hint_field_sizes(void *pSetupCtx, void* hintFieldValues, uint64_t hintId, char* hintFieldName, void* hintOptions);
     uint64_t mul_hint_fields(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2); 
-    void acc_hint_field(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName, bool add);
-    void acc_mul_hint_fields(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2,  void* hintOptions1, void *hintOptions2, bool add);
+    void acc_hint_field(void *pSetupCtx, void* stepsParams, void *pBuffHelper, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName, bool add);
+    void acc_mul_hint_fields(void *pSetupCtx, void* stepsParams, void *pBuffHelper, uint64_t hintId, char *hintFieldNameDest, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2,  void* hintOptions1, void *hintOptions2, bool add);
     uint64_t update_airgroupvalue(void *pSetupCtx, void* stepsParams, uint64_t hintId, char *hintFieldNameAirgroupVal, char *hintFieldName1, char *hintFieldName2, void* hintOptions1, void *hintOptions2, bool add);
     uint64_t set_hint_field(void *pSetupCtx, void* stepsParams, void *values, uint64_t hintId, char* hintFieldName);
     uint64_t get_hint_id(void *pSetupCtx, uint64_t hintId, char * hintFieldName);

pil2-stark/src/starkpil/hints.hpp

@@ -538,7 +538,7 @@ uint64_t multiplyHintFields(SetupCtx& setupCtx, StepsParams &params, uint64_t hi
     return hintFieldDestVal.id;
 }
 
-void accHintField(SetupCtx& setupCtx, StepsParams &params, uint64_t hintId, std::string hintFieldNameDest, std::string hintFieldNameAirgroupVal, std::string hintFieldName, bool add) {
+void accHintField(SetupCtx& setupCtx, StepsParams &params, Goldilocks::Element *pBuffHelper, uint64_t hintId, std::string hintFieldNameDest, std::string hintFieldNameAirgroupVal, std::string hintFieldName, bool add) {
     uint64_t N = 1 << setupCtx.starkInfo.starkStruct.nBits;
 
     Hint hint = setupCtx.expressionsBin.hints[hintId];
@@ -551,7 +551,7 @@ void accHintField(SetupCtx& setupCtx, StepsParams &params, uint64_t hintId, std:
     HintFieldValue hintFieldDestVal = hintFieldDest->values[0];
 
     uint64_t dim = setupCtx.starkInfo.cmPolsMap[hintFieldDestVal.id].dim;
-    Goldilocks::Element *vals = &params.aux_trace[setupCtx.starkInfo.mapOffsets[std::make_pair("q", true)]];
+    Goldilocks::Element *vals = pBuffHelper;
 
     Dest destStruct(vals, 0);
     addHintField(setupCtx, params, hintId, destStruct, hintFieldName, hintOptions);
@@ -589,7 +589,7 @@ uint64_t getHintId(SetupCtx& setupCtx, uint64_t hintId, std::string name) {
     return hintField->values[0].id;
 }
 
-void accMulHintFields(SetupCtx& setupCtx, StepsParams &params, uint64_t hintId, std::string hintFieldNameDest, std::string hintFieldNameAirgroupVal, std::string hintFieldName1, std::string hintFieldName2, HintFieldOptions &hintOptions1, HintFieldOptions &hintOptions2, bool add) {
+void accMulHintFields(SetupCtx& setupCtx, StepsParams &params, Goldilocks::Element *pBuffHelper, uint64_t hintId, std::string hintFieldNameDest, std::string hintFieldNameAirgroupVal, std::string hintFieldName1, std::string hintFieldName2, HintFieldOptions &hintOptions1, HintFieldOptions &hintOptions2, bool add) {
     uint64_t N = 1 << setupCtx.starkInfo.starkStruct.nBits;
 
     Hint hint = setupCtx.expressionsBin.hints[hintId];
@@ -600,7 +600,7 @@ void accMulHintFields(SetupCtx& setupCtx, StepsParams &params, uint64_t hintId,
     HintFieldValue hintFieldDestVal = hintFieldDest->values[0];
 
     uint64_t dim = setupCtx.starkInfo.cmPolsMap[hintFieldDestVal.id].dim;
-    Goldilocks::Element *vals = &params.aux_trace[setupCtx.starkInfo.mapOffsets[std::make_pair("q", true)]];
+    Goldilocks::Element *vals = pBuffHelper;
 
     Dest destStruct(vals, 0);
     addHintField(setupCtx, params, hintId, destStruct, hintFieldName1, hintOptions1);

proofman/Cargo.toml

@@ -28,4 +28,4 @@ rayon = "1.7"
 default = []
 no_lib_link = ["proofman-starks-lib-c/no_lib_link"]
 distributed = ["proofman-common/distributed", "dep:mpi"]
-debug = ["proofman-macros/debug"]
+diagnostic = ["proofman-macros/diagnostic"]

proofman/src/proofman.rs

@@ -101,7 +101,7 @@ impl<F: PrimeField + 'static> ProofMan<F> {
 
         timer_stop_and_log_info!(GENERATING_WITNESS);
 
-        #[cfg(feature = "debug")]
+        #[cfg(feature = "diagnostic")]
         {
             let air_instances = pctx.air_instance_repo.air_instances.read().unwrap();
             let instances = pctx.dctx_get_instances();

provers/stark/src/stark_prover.rs

@@ -33,6 +33,7 @@ pub struct StarkProver<F: Field> {
     merkle_tree_custom: bool,
     p_proof: *mut c_void,
     constraints_to_check: Vec<usize>,
+    prover_buffer_size: u64,
     _marker: PhantomData<F>, // Add PhantomData to track the type F
 }
 
@@ -52,6 +53,8 @@ impl<F: Field> StarkProver<F> {
     ) -> Self {
         let setup = sctx.get_setup(airgroup_id, air_id);
 
+        let prover_buffer_size = setup.prover_buffer_size;
+
         let p_stark = starks_new_c((&setup.p_setup).into(), setup.get_const_tree_ptr());
 
         let stark_info = setup.stark_info.clone();
@@ -81,6 +84,7 @@ impl<F: Field> StarkProver<F> {
             merkle_tree_arity,
             merkle_tree_custom,
             constraints_to_check,
+            prover_buffer_size,
             _marker: PhantomData,
         }
     }
@@ -90,7 +94,7 @@ impl<F: Field> Prover<F> for StarkProver<F> {
     fn build(&mut self, pctx: Arc<ProofCtx<F>>) {
         let mut air_instances = pctx.air_instance_repo.air_instances.write().unwrap();
         let air_instance = air_instances.get_mut(&self.global_idx).unwrap();
-        air_instance.init_aux_trace(get_map_totaln_c(self.p_stark_info, false) as usize);
+        air_instance.init_aux_trace(self.prover_buffer_size as usize);
         air_instance.init_evals(self.stark_info.ev_map.len() * Self::FIELD_EXTENSION);
 
         let n_custom_commits = self.stark_info.custom_commits.len();

provers/starks-lib-c/bindings_starks.rs

@@ -221,10 +221,11 @@ extern "C" {
     ) -> u64;
 }
 extern "C" {
-    #[link_name = "\u{1}_Z14acc_hint_fieldPvS_mPcS0_S0_b"]
+    #[link_name = "\u{1}_Z14acc_hint_fieldPvS_S_mPcS0_S0_b"]
     pub fn acc_hint_field(
         pSetupCtx: *mut ::std::os::raw::c_void,
         stepsParams: *mut ::std::os::raw::c_void,
+        pBuffHelper: *mut ::std::os::raw::c_void,
         hintId: u64,
         hintFieldNameDest: *mut ::std::os::raw::c_char,
         hintFieldNameAirgroupVal: *mut ::std::os::raw::c_char,
@@ -233,10 +234,11 @@ extern "C" {
     );
 }
 extern "C" {
-    #[link_name = "\u{1}_Z19acc_mul_hint_fieldsPvS_mPcS0_S0_S0_S_S_b"]
+    #[link_name = "\u{1}_Z19acc_mul_hint_fieldsPvS_S_mPcS0_S0_S0_S_S_b"]
     pub fn acc_mul_hint_fields(
         pSetupCtx: *mut ::std::os::raw::c_void,
         stepsParams: *mut ::std::os::raw::c_void,
+        pBuffHelper: *mut ::std::os::raw::c_void,
         hintId: u64,
         hintFieldNameDest: *mut ::std::os::raw::c_char,
         hintFieldNameAirgroupVal: *mut ::std::os::raw::c_char,

provers/starks-lib-c/src/ffi_starks.rs

@@ -361,9 +361,11 @@ pub fn mul_hint_fields_c(
 }
 
 #[cfg(not(feature = "no_lib_link"))]
+#[allow(clippy::too_many_arguments)]
 pub fn acc_hint_field_c(
     p_setup_ctx: *mut c_void,
     p_steps_params: *mut u8,
+    p_buff_helper: *mut u8,
     hint_id: u64,
     hint_field_dest: &str,
     hint_field_airgroupvalue: &str,
@@ -378,6 +380,7 @@ pub fn acc_hint_field_c(
         acc_hint_field(
             p_setup_ctx,
             p_steps_params as *mut std::os::raw::c_void,
+            p_buff_helper as *mut std::os::raw::c_void,
             hint_id,
             field_dest.as_ptr() as *mut std::os::raw::c_char,
             field_airgroupvalue.as_ptr() as *mut std::os::raw::c_char,
@@ -392,6 +395,7 @@ pub fn acc_hint_field_c(
 pub fn acc_mul_hint_fields_c(
     p_setup_ctx: *mut c_void,
     p_steps_params: *mut u8,
+    p_buff_helper: *mut u8,
     hint_id: u64,
     hint_field_dest: &str,
     hint_field_airgroupvalue: &str,
@@ -410,6 +414,7 @@ pub fn acc_mul_hint_fields_c(
         acc_mul_hint_fields(
             p_setup_ctx,
             p_steps_params as *mut std::os::raw::c_void,
+            p_buff_helper as *mut std::os::raw::c_void,
             hint_id,
             field_dest.as_ptr() as *mut std::os::raw::c_char,
             field_airgroupvalue.as_ptr() as *mut std::os::raw::c_char,
@@ -1439,9 +1444,11 @@ pub fn mul_hint_fields_c(
 }
 
 #[cfg(feature = "no_lib_link")]
+#[allow(clippy::too_many_arguments)]
 pub fn acc_hint_field_c(
     _p_setup_ctx: *mut c_void,
     _p_steps_params: *mut u8,
+    _p_buff_helper: *mut u8,
     _hint_id: u64,
     _hint_field_dest: &str,
     _hint_field_airgroupvalue: &str,
@@ -1456,6 +1463,7 @@ pub fn acc_hint_field_c(
 pub fn acc_mul_hint_fields_c(
     _p_setup_ctx: *mut c_void,
     _p_steps_params: *mut u8,
+    _p_buff_helper: *mut u8,
     _hint_id: u64,
     _hint_field_dest: &str,
     _hint_field_airgroupvalue: &str,