From 7b01f5d65be4087c76110a345c6d28a8548c8389 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 15 Jul 2024 16:49:51 +0800 Subject: [PATCH 001/107] Implement JumpDest fetching from RPC. Supports CREATE1/CREATE2 Supports transaction deployments Includes test scripts for randomised testing --- .gitignore | 1 + Cargo.lock | 4 + Cargo.toml | 4 +- evm_arithmetization/Cargo.toml | 1 + .../benches/fibonacci_25m_gas.rs | 1 + .../src/cpu/kernel/interpreter.rs | 60 ++- .../src/cpu/kernel/tests/add11.rs | 2 + .../kernel/tests/core/jumpdest_analysis.rs | 24 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 1 + .../src/cpu/kernel/tests/mod.rs | 2 +- .../src/generation/jumpdest.rs | 203 +++++++++ evm_arithmetization/src/generation/mod.rs | 11 + .../src/generation/prover_input.rs | 48 +- evm_arithmetization/src/generation/state.rs | 3 +- evm_arithmetization/src/lib.rs | 3 + evm_arithmetization/tests/add11_yml.rs | 1 + evm_arithmetization/tests/erc20.rs | 1 + evm_arithmetization/tests/erc721.rs | 1 + evm_arithmetization/tests/global_exit_root.rs | 1 + evm_arithmetization/tests/log_opcode.rs | 1 + evm_arithmetization/tests/selfdestruct.rs | 1 + evm_arithmetization/tests/simple_transfer.rs | 1 + evm_arithmetization/tests/withdrawals.rs | 1 + test_jerigon.sh | 88 ++++ test_native.sh | 83 ++++ trace_decoder/src/core.rs | 12 + trace_decoder/src/interface.rs | 20 +- trace_decoder/src/jumpdest.rs | 1 + trace_decoder/src/lib.rs | 1 + zero_bin/prover/src/cli.rs | 2 +- zero_bin/rpc/Cargo.toml | 3 + zero_bin/rpc/src/jerigon.rs | 134 +++++- zero_bin/rpc/src/jumpdest.rs | 417 ++++++++++++++++++ zero_bin/rpc/src/lib.rs | 1 + zero_bin/rpc/src/main.rs | 3 + zero_bin/rpc/src/native/mod.rs | 6 +- zero_bin/rpc/src/native/txn.rs | 72 ++- zero_bin/tools/prove_stdio.sh | 22 +- 38 files changed, 1170 insertions(+), 71 deletions(-) create mode 100644 evm_arithmetization/src/generation/jumpdest.rs create mode 100755 test_jerigon.sh create mode 100755 test_native.sh create mode 100644 trace_decoder/src/jumpdest.rs create mode 100644 zero_bin/rpc/src/jumpdest.rs diff --git a/.gitignore b/.gitignore index f035f7002..dd5a27331 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .vscode /**/*.ignoreme **/output.log +witnesses diff --git a/Cargo.lock b/Cargo.lock index 1791c16ed..0b4835ef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2066,6 +2066,7 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plonky2_util", + "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", @@ -4169,6 +4170,8 @@ version = "0.1.0" dependencies = [ "alloy", "alloy-compat", + "alloy-primitives", + "alloy-serde", "anyhow", "cargo_metadata", "clap", @@ -4177,6 +4180,7 @@ dependencies = [ "futures", "hex", "itertools 0.13.0", + "keccak-hash 0.10.0", "mpt_trie", "primitive-types 0.12.2", "proof_gen", diff --git a/Cargo.toml b/Cargo.toml index 9628a345c..84a438806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ alloy = { version = '0.3.0', default-features = false, features = [ "transport-http", "rpc-types-debug", ] } +alloy-primitives = "0.8.0" +alloy-serde = "0.3.0" anyhow = "1.0.86" async-stream = "0.3.5" axum = "0.7.5" @@ -57,7 +59,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -94,7 +95,6 @@ serde = "1.0.203" serde-big-array = "0.5.1" serde_json = "1.0.118" serde_path_to_error = "0.1.16" -serde_with = "3.8.1" sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "1.0.61" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index ffdb8d2f5..e8e832c2d 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,6 +15,7 @@ homepage.workspace = true keywords.workspace = true [dependencies] +__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index f6d5fc39f..57b902c63 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,6 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 10af8d495..d0cb867fa 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -5,10 +5,12 @@ //! the future execution and generate nondeterministically the corresponding //! jumpdest table, before the actual CPU carries on with contract execution. +use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; +use keccak_hash::H256; use log::Level; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; @@ -19,7 +21,9 @@ use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; +use crate::generation::jumpdest::{ContextJumpDests, JumpDestTableProcessed, JumpDestTableWitness}; use crate::generation::mpt::{load_linked_lists_and_txn_and_receipt_mpts, TrieRootPtrs}; +use crate::generation::prover_input::get_proofs_and_jumpdests; use crate::generation::rlp::all_rlp_prover_inputs_reversed; use crate::generation::state::{ all_ger_prover_inputs, all_withdrawals_prover_inputs_reversed, GenerationState, @@ -56,6 +60,7 @@ pub(crate) struct Interpreter { /// Counts the number of appearances of each opcode. For debugging purposes. #[allow(unused)] pub(crate) opcode_count: [usize; 0x100], + /// A table of contexts and their reached JUMPDESTs. jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -71,9 +76,9 @@ pub(crate) struct Interpreter { pub(crate) fn simulate_cpu_and_get_user_jumps( final_label: &str, state: &GenerationState, -) -> Option>> { +) -> Option<(JumpDestTableProcessed, JumpDestTableWitness)> { match state.jumpdest_table { - Some(_) => None, + Some(_) => Default::default(), None => { let halt_pc = KERNEL.global_labels[final_label]; let initial_context = state.registers.context; @@ -92,16 +97,15 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - interpreter + let jdtw = interpreter .generation_state - .set_jumpdest_analysis_inputs(interpreter.jumpdest_table); + .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - - interpreter.generation_state.jumpdest_table + (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) } } } @@ -114,9 +118,9 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, - pub(crate) jumpdest_table: Option>>, pub(crate) accounts: BTreeMap, pub(crate) storage: BTreeMap<(U256, U256), usize>, + pub(crate) jumpdest_table: Option, pub(crate) next_txn_index: usize, } @@ -150,6 +154,48 @@ pub(crate) fn set_registers_and_run( interpreter.run() } +/// Computes the JUMPDEST proofs for each context. +/// +/// # Arguments +/// +/// - `jumpdest_table_rpc`: The raw table received from RPC. +/// - `code_db`: The corresponding database of contract code used in the trace. +pub(crate) fn set_jumpdest_analysis_inputs_rpc( + jumpdest_table_rpc: &JumpDestTableWitness, + code_map: &HashMap>, +) -> JumpDestTableProcessed { + let ctx_proofs = (*jumpdest_table_rpc) + .iter() + .flat_map(|(code_addr, ctx_jumpdests)| { + prove_context_jumpdests(&code_map[code_addr], ctx_jumpdests) + }) + .collect(); + JumpDestTableProcessed::new(ctx_proofs) +} + +/// Orchestrates the proving of all contexts in a specific bytecode. +/// +/// # Arguments +/// +/// - `ctx_jumpdests`: Map from `ctx` to its list of offsets to reached +/// `JUMPDEST`s. +/// - `code`: The bytecode for the contexts. This is the same for all contexts. +fn prove_context_jumpdests( + code: &[u8], + ctx_jumpdests: &ContextJumpDests, +) -> HashMap> { + ctx_jumpdests + .0 + .iter() + .map(|(&ctx, jumpdests)| { + let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { + get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) + }); + (ctx, proofs) + }) + .collect() +} + impl Interpreter { /// Returns an instance of `Interpreter` given `GenerationInputs`, and /// assuming we are initializing with the `KERNEL` code. diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index fe36b40e3..1cc5588e1 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,6 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let initial_stack = vec![]; @@ -371,6 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index b0ef17033..1d93e0e31 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -7,6 +7,7 @@ use plonky2::field::goldilocks_field::GoldilocksField as F; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::interpreter::Interpreter; use crate::cpu::kernel::opcodes::{get_opcode, get_push_opcode}; +use crate::generation::jumpdest::JumpDestTableProcessed; use crate::witness::operation::CONTEXT_SCALING_FACTOR; #[test] @@ -67,7 +68,10 @@ fn test_jumpdest_analysis() -> Result<()> { interpreter.generation_state.jumpdest_table, // Context 3 has jumpdest 1, 5, 7. All have proof 0 and hence // the list [proof_0, jumpdest_0, ... ] is [0, 1, 0, 5, 0, 7, 8, 40] - Some(HashMap::from([(3, vec![0, 1, 0, 5, 0, 7, 8, 40])])) + Some(JumpDestTableProcessed::new(HashMap::from([( + 3, + vec![0, 1, 0, 5, 0, 7, 8, 40] + )]))) ); // Run jumpdest analysis with context = 3 @@ -84,14 +88,14 @@ fn test_jumpdest_analysis() -> Result<()> { // We need to manually pop the jumpdest_table and push its value on the top of // the stack - interpreter + (*interpreter .generation_state .jumpdest_table .as_mut() - .unwrap() - .get_mut(&CONTEXT) - .unwrap() - .pop(); + .unwrap()) + .get_mut(&CONTEXT) + .unwrap() + .pop(); interpreter .push(41.into()) .expect("The stack should not overflow"); @@ -136,7 +140,9 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(HashMap::from([(3, vec![1, 33])])); + interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from( + [(3, vec![1, 33])], + ))); interpreter.run()?; @@ -149,7 +155,9 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(HashMap::from([(3, vec![1, 33])])); + interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new( + HashMap::from([(3, vec![1, 33])]), + )); assert!(interpreter.run().is_err()); diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index be88021de..08fe57b1a 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,6 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, + jumpdest_tables: vec![], }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index 39810148b..08b17d495 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -249,7 +249,7 @@ impl Interpreter { } pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { - self.generation_state.set_jumpdest_analysis_inputs(jumps); + let _ = self.generation_state.set_jumpdest_analysis_inputs(jumps); } pub(crate) fn extract_kernel_memory(self, segment: Segment, range: Range) -> Vec { diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs new file mode 100644 index 000000000..fc2a8e1a6 --- /dev/null +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -0,0 +1,203 @@ +use std::cmp::max; +use std::{ + collections::{BTreeSet, HashMap}, + fmt::Display, + ops::{Deref, DerefMut}, +}; + +use itertools::{sorted, Itertools}; +use keccak_hash::H256; +use serde::{Deserialize, Serialize}; + +/// Each `CodeAddress` can be called one or more times, each time creating a new +/// `Context`. Each `Context` will contain one or more offsets of `JUMPDEST`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct ContextJumpDests(pub HashMap>); + +/// The result after proving a `JumpDestTableWitness`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct JumpDestTableProcessed(HashMap>); + +/// Map `CodeAddress -> (Context -> [JumpDests])` +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableWitness(HashMap); + +impl ContextJumpDests { + pub fn insert(&mut self, ctx: usize, offset: usize) { + self.entry(ctx).or_default().insert(offset); + } + + pub fn get(&self, ctx: usize) -> Option<&BTreeSet> { + self.0.get(&ctx) + } +} + +impl JumpDestTableProcessed { + pub fn new(ctx_map: HashMap>) -> Self { + Self(ctx_map) + } +} + +impl JumpDestTableWitness { + pub fn new(ctx_map: HashMap) -> Self { + Self(ctx_map) + } + + pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { + self.0.get(code_hash) + } + + /// Insert `offset` into `ctx` under the corresponding `code_hash`. + /// Creates the required `ctx` keys and `code_hash`. Idempotent. + pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { + (*self).entry(code_hash).or_default().insert(ctx, offset); + + // TODO(einar) remove before publishing PR. + assert!(self.0.contains_key(&code_hash)); + assert!(self.0[&code_hash].0.contains_key(&ctx)); + assert!(self.0[&code_hash].0[&ctx].contains(&offset)); + } + + pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { + let mut curr_max_ctx = prev_max_ctx; + + // TODO: Opportunity for optimization: Simulate to generate only missing + // JUMPDEST tables. + for (code_hash, ctx_tbl) in (*other).iter() { + for (ctx, jumpdests) in ctx_tbl.0.iter() { + let batch_ctx = prev_max_ctx + ctx; + curr_max_ctx = max(curr_max_ctx, batch_ctx); + + for offset in jumpdests { + self.insert(*code_hash, batch_ctx, *offset); + + assert!(self.0.contains_key(code_hash)); + assert!(self.0[code_hash].0.contains_key(&batch_ctx)); + assert!(self.0[code_hash].0[&batch_ctx].contains(offset)); + } + // dbg!(&self); + } + } + + (self, curr_max_ctx) + } + + pub fn merge<'a>(jdts: impl IntoIterator) -> (Self, usize) { + jdts.into_iter() + .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) + } +} + +impl Display for JumpDestTableWitness { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "=== JumpDestTableWitness ===")?; + + for (code, ctxtbls) in &self.0 { + write!(f, "codehash: {:#x}\n{}", code, ctxtbls)?; + } + Ok(()) + } +} + +impl Display for ContextJumpDests { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let v: Vec<_> = self.0.iter().sorted().collect(); + for (ctx, offsets) in v.into_iter() { + write!(f, " ctx: {:>4}: [", ctx)?; + for offset in offsets { + write!(f, "{:#}, ", offset)?; + } + writeln!(f, "]")?; + } + Ok(()) + } +} + +impl Display for JumpDestTableProcessed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "=== JumpDestTableProcessed ===")?; + + let v = sorted(self.0.clone()); + for (ctx, code) in v { + writeln!(f, "ctx: {:?} {:?}", ctx, code)?; + } + Ok(()) + } +} + +impl Deref for ContextJumpDests { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for ContextJumpDests { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for JumpDestTableProcessed { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JumpDestTableProcessed { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for JumpDestTableWitness { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JumpDestTableWitness { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(test)] +mod test { + use keccak_hash::H256; + + use super::JumpDestTableWitness; + + #[test] + fn test_extend() { + let code_hash = H256::default(); + + let mut table1 = JumpDestTableWitness::default(); + table1.insert(code_hash, 1, 1); + table1.insert(code_hash, 2, 2); + table1.insert(code_hash, 42, 3); + table1.insert(code_hash, 43, 4); + let table2 = table1.clone(); + + let jdts = [&table1, &table2]; + let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); + + let mut expected = JumpDestTableWitness::default(); + expected.insert(code_hash, 1, 1); + expected.insert(code_hash, 2, 2); + expected.insert(code_hash, 42, 3); + expected.insert(code_hash, 43, 4); + expected.insert(code_hash, 44, 1); + expected.insert(code_hash, 45, 2); + expected.insert(code_hash, 85, 3); + expected.insert(code_hash, 86, 4); + + assert_eq!(86, max_ctx); + assert_eq!(expected, actual) + } +} diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 9c7625d2b..87d9ed1f7 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use anyhow::anyhow; use ethereum_types::{Address, BigEndianHash, H256, U256}; +use jumpdest::JumpDestTableWitness; use keccak_hash::keccak; use log::log_enabled; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; @@ -32,6 +33,7 @@ use crate::util::{h2u, u256_to_usize}; use crate::witness::memory::{MemoryAddress, MemoryChannel, MemoryState}; use crate::witness::state::RegistersState; +pub mod jumpdest; pub(crate) mod linked_list; pub mod mpt; pub(crate) mod prover_input; @@ -106,6 +108,10 @@ pub struct GenerationInputs { /// /// This is specific to `cdk-erigon`. pub ger_data: Option<(H256, H256)>, + + /// A table listing each JUMPDESTs reached in each call context under + /// associated code hash. + pub jumpdest_tables: Vec>, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -156,6 +162,10 @@ pub struct TrimmedGenerationInputs { /// The hash of the current block, and a list of the 256 previous block /// hashes. pub block_hashes: BlockHashes, + + /// A list of tables listing each JUMPDESTs reached in each call context + /// under associated code hash. + pub jumpdest_tables: Vec>, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -230,6 +240,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), + jumpdest_tables: self.jumpdest_tables.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 16c1e5310..ae49366b3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -6,10 +6,12 @@ use std::str::FromStr; use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; +use keccak_hash::keccak; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; +use super::jumpdest::{JumpDestTableProcessed, JumpDestTableWitness}; use super::linked_list::{ LinkedList, ACCOUNTS_LINKED_LIST_NODE_SIZE, STORAGE_LINKED_LIST_NODE_SIZE, }; @@ -20,7 +22,9 @@ use crate::cpu::kernel::constants::cancun_constants::{ POINT_EVALUATION_PRECOMPILE_RETURN_VALUE, }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; -use crate::cpu::kernel::interpreter::simulate_cpu_and_get_user_jumps; +use crate::cpu::kernel::interpreter::{ + set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, +}; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; use crate::generation::prover_input::EvmField::{ @@ -38,6 +42,10 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; +/// A set of contract code as a byte arrays. From this a mapping: hash -> +/// contract can be built. +pub type CodeDb = BTreeSet>; + /// Prover input function represented as a scoped function name. /// Example: `PROVER_INPUT(ff::bn254_base::inverse)` is represented as /// `ProverInputFn([ff, bn254_base, inverse])`. @@ -364,12 +372,12 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { Ok((next_jumpdest_address + 1).into()) } else { - jumpdest_table.remove(&context); + (*jumpdest_table).remove(&context); Ok(U256::zero()) } } @@ -383,7 +391,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { Ok(next_jumpdest_proof.into()) @@ -757,7 +765,22 @@ impl GenerationState { fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - self.jumpdest_table = simulate_cpu_and_get_user_jumps("terminate_common", self); + + dbg!(&self.inputs.jumpdest_tables); + eprintln!("Generating JUMPDEST tables"); + // w for witness + let txn_idx = self.next_txn_index - 1; + let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref(); + let rpc = rpcw.map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + + if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { + if rpcw.is_some() && rpcw.unwrap() != &simw.clone() { + println!("SIMW {}", simw.clone()); + println!("RPCW {}", rpcw.unwrap()); + assert_eq!(simw.clone(), *rpcw.unwrap()); + } + } + self.jumpdest_table = rpc; Ok(()) } @@ -768,18 +791,23 @@ impl GenerationState { pub(crate) fn set_jumpdest_analysis_inputs( &mut self, jumpdest_table: HashMap>, - ) { - self.jumpdest_table = Some(HashMap::from_iter(jumpdest_table.into_iter().map( - |(ctx, jumpdest_table)| { + ) -> JumpDestTableWitness { + let mut jdtw = JumpDestTableWitness::default(); + self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( + jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); + for offset in jumpdest_table.clone() { + jdtw.insert(keccak(code.clone()), ctx, offset); + } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); (ctx, proofs) } else { (ctx, vec![]) } - }, + }), ))); + jdtw } pub(crate) fn get_current_code(&self) -> Result, ProgramError> { @@ -850,7 +878,7 @@ impl GenerationState { /// for which none of the previous 32 bytes in the code (including opcodes /// and pushed bytes) is a PUSHXX and the address is in its range. It returns /// a vector of even size containing proofs followed by their addresses. -fn get_proofs_and_jumpdests( +pub(crate) fn get_proofs_and_jumpdests( code: &[u8], largest_address: usize, jumpdest_table: std::collections::BTreeSet, diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index b094c15f9..39744ddf2 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -8,6 +8,7 @@ use keccak_hash::keccak; use log::Level; use plonky2::hash::hash_types::RichField; +use super::jumpdest::JumpDestTableProcessed; use super::linked_list::{AccountsLinkedList, StorageLinkedList}; use super::mpt::TrieRootPtrs; use super::segments::GenerationSegmentData; @@ -374,7 +375,7 @@ pub struct GenerationState { /// "proof" for a jump destination is either 0 or an address i > 32 in /// the code (not necessarily pointing to an opcode) such that for every /// j in [i, i+32] it holds that code[j] < 0x7f - j + i. - pub(crate) jumpdest_table: Option>>, + pub(crate) jumpdest_table: Option, /// Each entry contains the pair (key, ptr) where key is the (hashed) key /// of an account in the accounts linked list, and ptr is the respective diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index ca2cf2bd7..f813b18cf 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -210,6 +210,9 @@ pub mod verifier; pub mod generation; pub mod witness; +pub use generation::jumpdest; +pub use generation::prover_input::CodeDb; + // Utility modules pub mod curve_pairings; pub mod extension_tower; diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 5b8290229..51ac662df 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,6 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 2c1a56829..f13f541b4 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,6 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 170a19df4..6386dbe69 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,6 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/global_exit_root.rs b/evm_arithmetization/tests/global_exit_root.rs index a41d79e6f..8c18b8e47 100644 --- a/evm_arithmetization/tests/global_exit_root.rs +++ b/evm_arithmetization/tests/global_exit_root.rs @@ -114,6 +114,7 @@ fn test_global_exit_root() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + batch_jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 44453ae22..d9fe6f234 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,6 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 7d8ecba5b..27d68af7b 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,6 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 7dbf91399..4109ae09c 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,6 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index fcdd73f49..3dfe13e43 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,6 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/test_jerigon.sh b/test_jerigon.sh new file mode 100755 index 000000000..49f88ed12 --- /dev/null +++ b/test_jerigon.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +#export CARGO_LOG=cargo::core::compiler::fingerprint=debug +export RPC= +if [ -z $RPC ]; then + # You must set an RPC endpoint + exit 1 +fi +mkdir -p witnesses + +export RAYON_NUM_THREADS=4 +export TOKIO_WORKER_THREADS=4 +export RUST_BACKTRACE=full +export RUST_LOG=info +export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUST_MIN_STACK=33554432 + +GITHASH=`git rev-parse --short HEAD` +echo "Testing against jergion, current revision: $GITHASH." + +TESTNETBLOCKS=" +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +" + + +KNOWNFAILED=" +28 +444 +" + + + +# 470..663 from Robin +for i in {470..663} +do + ROBIN+=" $i" +done + +# Pick random blocks +for i in {1..10} +do + RANDOMBLOCKS+=" $((1 + $RANDOM % 688))" +done + +# TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +TESTNETBLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" + +TESTNETBLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` +SHUF=`shuf -e $TESTNETBLOCKS` +echo $SHUF + + + +for BLOCK in $TESTNETBLOCKS; do + GITHASH=`git rev-parse --short HEAD` + WITNESS="$witnesses/$BLOCK.jerigon.$GITHASH.witness.json" + echo "Fetching block $BLOCK" + cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Checking block $BLOCK" + zero_bin/tools/prove_stdio.sh $WITNESS test_only + EXITCODE=$? + if [ -n $EXITCODE ] + then + RESULT="success" + else + RESULT="failure" + fi + printf "%10i %s %s\n" $BLOCK $GITHASH $RESULT | tee -a result.txt +done + +exit 0 diff --git a/test_native.sh b/test_native.sh new file mode 100755 index 000000000..913f27f13 --- /dev/null +++ b/test_native.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +export RPC= +if [ -z $RPC ]; then + # You must set an RPC endpoint + exit 1 +fi +mkdir -p witnesses + +# Must match the values in prove_stdio.sh or build is dirty. +export RAYON_NUM_THREADS=4 +export TOKIO_WORKER_THREADS=4 +export RUST_BACKTRACE=full +export RUST_LOG=info +export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUST_MIN_STACK=33554432 + + + +MAINNETBLOCKS=" +20548415 +20240058 +19665756 +20634472 +19807080 +20634403 +19096840 +19240700 +" + +CANCUN=19426587 +TIP=`cast block-number --rpc-url $RPC` +STATICTIP=20721266 +NUMRANDOMBLOCKS=10 +RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` + +GITHASH=`git rev-parse --short HEAD` +echo "Testing against mainnet, current revision: $GITHASH." + +#BLOCKS="$MAINNETBLOCKS $RANDOMBLOCKS" +BLOCKS="$MAINNETBLOCKS" +echo "Testing blocks: $BLOCKS" + +echo "Downloading witnesses.." + +for BLOCK in $BLOCKS; do + WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" + until [ -f $WITNESS -a -s $WITNESS ]; do + echo "Fetching block $BLOCK" + cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + EXITCODE=$? + + if [ -n $EXITCODE -a -f $WITNESS -a -s $WITNESS ] + then + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt + break + fi + + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure| tee -a witnesses/native_results.txt + done + + echo "Witness for block $BLOCK ($WITNESS) prepared." +done + +echo "Finished downloading witnesses." +echo "Testing prepared witnesses.." + +for WITNESS in witnesses/*.native.$GITHASH.witness.json; do + echo "Testing $WITNESS" + zero_bin/tools/prove_stdio.sh $WITNESS test_only + EXITCODE=$? + if [ -n $EXITCODE ] + then + RESULT="success" + else + RESULT="failure" + fi + printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt +done + +echo "Finished testing witnesses." diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 64c0dc2ee..243bacf29 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -1,3 +1,4 @@ +use core::option::Option::None; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -10,6 +11,7 @@ use anyhow::{anyhow, bail, ensure, Context as _}; use ethereum_types::{Address, U256}; use evm_arithmetization::{ generation::{mpt::AccountRlp, TrieInputs}, + jumpdest::JumpDestTableWitness, proof::TrieRoots, testing_utils::{BEACON_ROOTS_CONTRACT_ADDRESS, HISTORY_BUFFER_LENGTH}, GenerationInputs, @@ -87,6 +89,7 @@ pub fn entrypoint( }, after, withdrawals, + jumpdest_tables, }| GenerationInputs:: { txn_number_before: first_txn_ix.into(), gas_used_before: running_gas_used.into(), @@ -113,6 +116,7 @@ pub fn entrypoint( block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, + jumpdest_tables, }, ) .collect()) @@ -256,6 +260,8 @@ struct Batch { /// Empty for all but the final batch pub withdrawals: Vec<(Address, U256)>, + + pub jumpdest_tables: Vec>, } /// [`evm_arithmetization::generation::TrieInputs`], @@ -336,6 +342,8 @@ fn middle( )?; } + let mut jumpdest_tables = vec![]; + for txn in batch { let do_increment_txn_ix = txn.is_some(); let TxnInfo { @@ -345,6 +353,7 @@ fn middle( byte_code, new_receipt_trie_node_byte, gas_used: txn_gas_used, + jumpdest_table, }, } = txn.unwrap_or_default(); @@ -475,6 +484,8 @@ fn middle( // the transaction calling them reverted. } + jumpdest_tables.push(jumpdest_table); + if do_increment_txn_ix { txn_ix += 1; } @@ -527,6 +538,7 @@ fn middle( transactions_root: transaction_trie.root(), receipts_root: receipt_trie.root(), }, + jumpdest_tables, }); } // batch in batches diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 901fef89a..5c72d6b3f 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -5,8 +5,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use ethereum_types::{Address, U256}; -use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; -use keccak_hash::H256; +use evm_arithmetization::{ + jumpdest::JumpDestTableWitness, + proof::{BlockHashes, BlockMetadata}, +}; +use keccak_hash::{keccak, H256}; use mpt_trie::partial_trie::HashedPartialTrie; use plonky2::hash::hash_types::NUM_HASH_OUT_ELTS; use serde::{Deserialize, Serialize}; @@ -113,6 +116,9 @@ pub struct TxnMeta { /// Gas used by this txn (Note: not cumulative gas used). pub gas_used: u64, + + /// JumpDest table + pub jumpdest_table: Option, } /// A "trace" specific to an account for a txn. @@ -164,6 +170,16 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } +// Question: Why has this has been removed upstream. Proably unused. +impl ContractCodeUsage { + /// Get code hash from a read or write operation of contract code. + pub fn get_code_hash(&self) -> H256 { + match self { + ContractCodeUsage::Read(hash) => *hash, + ContractCodeUsage::Write(bytes) => keccak(bytes), + } + } +} /// Other data that is needed for proof gen. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/trace_decoder/src/jumpdest.rs b/trace_decoder/src/jumpdest.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/trace_decoder/src/jumpdest.rs @@ -0,0 +1 @@ + diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index fba82d8f7..75146fe23 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -62,6 +62,7 @@ use plonky2::field::goldilocks_field::GoldilocksField; mod type1; // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 // add backend/prod support for type 2 +mod jumpdest; #[cfg(test)] #[allow(dead_code)] mod type2; diff --git a/zero_bin/prover/src/cli.rs b/zero_bin/prover/src/cli.rs index 694194ad9..ba3147f5f 100644 --- a/zero_bin/prover/src/cli.rs +++ b/zero_bin/prover/src/cli.rs @@ -18,7 +18,7 @@ pub struct CliProverConfig { #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 19)] max_cpu_len_log: usize, /// Number of transactions in a batch to process at once. - #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 10)] + #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 1)] batch_size: usize, /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] diff --git a/zero_bin/rpc/Cargo.toml b/zero_bin/rpc/Cargo.toml index c694335d4..0215090eb 100644 --- a/zero_bin/rpc/Cargo.toml +++ b/zero_bin/rpc/Cargo.toml @@ -35,6 +35,9 @@ prover = { workspace = true } trace_decoder = { workspace = true } zero_bin_common = { workspace = true } zk_evm_common = { workspace = true } +alloy-serde = { workspace = true } +alloy-primitives = { workspace = true } +keccak-hash = { workspace = true } [build-dependencies] anyhow = { workspace = true } diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 00d56cf48..3a9f7f866 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -1,12 +1,29 @@ -use alloy::{providers::Provider, rpc::types::eth::BlockId, transports::Transport}; +use std::collections::BTreeMap; +use std::ops::Deref as _; + +use alloy::providers::ext::DebugApi; +use alloy::rpc::types::trace::geth::StructLog; +use alloy::{ + providers::Provider, + rpc::types::{eth::BlockId, trace::geth::GethTrace, Block, BlockTransactionsKind, Transaction}, + transports::Transport, +}; +use alloy_primitives::Address; use anyhow::Context as _; +use evm_arithmetization::jumpdest::JumpDestTableWitness; +use futures::stream::FuturesOrdered; +use futures::StreamExt as _; use prover::BlockProverInput; use serde::Deserialize; use serde_json::json; -use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; +use trace_decoder::{ + BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, +}; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; +use crate::jumpdest; +use crate::jumpdest::structlogprime::try_reserialize; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] @@ -33,16 +50,63 @@ where "debug_traceBlockByNumber".into(), (target_block_id, json!({"tracer": "zeroTracer"})), ) - .await?; + .await? + .into_iter() + .map(|ztr| ztr.result) + .collect::>(); // Grab block witness info (packed as combined trie pre-images) - let block_witness = cached_provider .get_provider() .await? .raw_request::<_, String>("eth_getWitness".into(), vec![target_block_id]) .await?; + let block = cached_provider + .get_block(target_block_id, BlockTransactionsKind::Full) + .await?; + + let tx_traces: Vec<&BTreeMap<__compat_primitive_types::H160, TxnTrace>> = tx_results + .iter() + .map(|TxnInfo { traces, meta: _ }| traces) + .collect::>(); + + let jdts: Vec> = process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + &tx_traces, + ) + .await?; + + // weave in the JDTs + let txn_info = tx_results + .into_iter() + .zip(jdts) + .map( + |( + TxnInfo { + traces, + meta: + TxnMeta { + byte_code, + new_receipt_trie_node_byte, + gas_used, + jumpdest_table: _, + }, + }, + jdt, + )| TxnInfo { + traces, + meta: TxnMeta { + byte_code, + new_receipt_trie_node_byte, + gas_used, + jumpdest_table: jdt, + }, + }, + ) + .collect(); + let other_data = fetch_other_block_data(cached_provider, target_block_id, checkpoint_block_number).await?; @@ -53,9 +117,69 @@ where compact: hex::decode(block_witness.strip_prefix("0x").unwrap_or(&block_witness)) .context("invalid hex returned from call to eth_getWitness")?, }), - txn_info: tx_results.into_iter().map(|it| it.result).collect(), + txn_info, code_db: Default::default(), }, other_data, }) } + +/// Processes the transactions in the given block and updates the code db. +pub async fn process_transactions( + block: &Block, + provider: &ProviderT, + tx_traces: &[&BTreeMap<__compat_primitive_types::H160, TxnTrace>], +) -> anyhow::Result>> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let futures = block + .transactions + .as_transactions() + .context("No transactions in block")? + .iter() + .zip(tx_traces) + .map(|(tx, &tx_trace)| process_transaction(provider, tx, tx_trace)) + .collect::>(); + + let vec_of_res = futures.collect::>().await; + vec_of_res.into_iter().collect::, _>>() +} + +/// Processes the transaction with the given transaction hash and updates the +/// accounts state. +pub async fn process_transaction( + provider: &ProviderT, + tx: &Transaction, + tx_trace: &BTreeMap<__compat_primitive_types::H160, TxnTrace>, +) -> anyhow::Result> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let structlog_trace = provider + .debug_trace_transaction(tx.hash, jumpdest::structlog_tracing_options()) + .await?; + + let struct_logs_opt: Option> = match structlog_trace { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + }; + + let tx_traces = tx_trace + .iter() + .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) + .collect(); + + let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { + jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) + .map(Some) + .unwrap_or_default() + }); + + Ok(jumpdest_table) +} diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs new file mode 100644 index 000000000..7e9bde277 --- /dev/null +++ b/zero_bin/rpc/src/jumpdest.rs @@ -0,0 +1,417 @@ +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; +use std::ops::Not as _; +use std::sync::OnceLock; + +use __compat_primitive_types::H256; +use alloy::primitives::Address; +use alloy::primitives::U160; +use alloy::rpc::types::eth::Transaction; +use alloy::rpc::types::trace::geth::StructLog; +use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; +use alloy_primitives::U256; +use anyhow::ensure; +use evm_arithmetization::jumpdest::JumpDestTableWitness; +use keccak_hash::keccak; +use trace_decoder::TxnTrace; +use tracing::trace; + +/// Tracing options for the `debug_traceTransaction` call to get structlog. +/// Used for filling JUMPDEST table. +pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { + GethDebugTracingOptions { + config: GethDefaultTracingOptions { + disable_stack: Some(false), + // needed for CREATE2 + disable_memory: Some(false), + disable_storage: Some(true), + ..GethDefaultTracingOptions::default() + }, + tracer: None, + ..GethDebugTracingOptions::default() + } +} + +/// Provides a way to check in constant time if an address points to a +/// precompile. +fn precompiles() -> &'static HashSet
{ + static PRECOMPILES: OnceLock> = OnceLock::new(); + PRECOMPILES.get_or_init(|| { + HashSet::
::from_iter((1..=0xa).map(|x| Address::from(U160::from(x)))) + }) +} + +/// Generate at JUMPDEST table by simulating the call stack in EVM, +/// using a Geth structlog as input. +pub(crate) fn generate_jumpdest_table( + tx: &Transaction, + struct_log: &[StructLog], + tx_traces: &BTreeMap, +) -> anyhow::Result { + trace!("Generating JUMPDEST table for tx: {}", tx.hash); + ensure!(struct_log.is_empty().not(), "Structlog is empty."); + + let mut jumpdest_table = JumpDestTableWitness::default(); + + let callee_addr_to_code_hash: HashMap = tx_traces + .iter() + .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) + .filter(|(_callee_addr, code_usage)| code_usage.is_some()) + .map(|(callee_addr, code_usage)| { + (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + }) + .collect(); + + trace!( + "Transaction: {} is a {}.", + tx.hash, + if tx.to.is_some() { + "message call" + } else { + "contract creation" + } + ); + + let entrypoint_code_hash: H256 = if let Some(to_address) = tx.to { + // Guard against transactions to a non-contract address. + ensure!( + callee_addr_to_code_hash.contains_key(&to_address), + format!("Callee addr {} is not at contract address", to_address) + ); + callee_addr_to_code_hash[&to_address] + } else { + let init = tx.input.clone(); + keccak(init) + }; + + // `None` encodes that previous `entry`` was not a JUMP or JUMPI with true + // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with + // true condition and target `jump_target`. + let mut prev_jump = None; + + // Call depth of the previous `entry`. We initialize to 0 as this compares + // smaller to 1. + //let mut prev_depth = 0; + // The next available context. Starts at 1. Never decrements. + let mut next_ctx_available = 1; + // Immediately use context 1; + let mut call_stack = vec![(entrypoint_code_hash, next_ctx_available)]; + next_ctx_available += 1; + + for (step, entry) in struct_log.iter().enumerate() { + let op = entry.op.as_str(); + let curr_depth: usize = entry.depth.try_into().unwrap(); + + ensure!(curr_depth <= next_ctx_available, "Structlog is malformed."); + + // ensure!(call_stack.is_empty().not(), "Call stack was empty."); + while curr_depth < call_stack.len() { + call_stack.pop(); + } + + let (code_hash, ctx) = call_stack.last().unwrap(); + + trace!("TX: {:?}", tx.hash); + trace!("STEP: {:?}", step); + trace!("STEPS: {:?}", struct_log.len()); + trace!("OPCODE: {}", entry.op.as_str()); + trace!("CODE: {:?}", code_hash); + trace!("CTX: {:?}", ctx); + trace!("CURR_DEPTH: {:?}", curr_depth); + trace!("{:#?}\n", entry); + + match op { + "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 2; // actually 6 or 7. + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 + let [_gas, address, ..] = evm_stack[..] else { + unreachable!() + }; + + let callee_address = { + // Clear the upper half of the operand. + let callee_raw = *address; + // let (callee_raw, _overflow) = callee_raw.overflowing_shl(128); + // let (callee_raw, _overflow) = callee_raw.overflowing_shr(128); + + ensure!(callee_raw <= U256::from(U160::MAX)); + let lower_20_bytes = U160::from(callee_raw); + Address::from(lower_20_bytes) + }; + + if precompiles().contains(&callee_address) { + trace!("Called precompile at address {}.", &callee_address); + } else if callee_addr_to_code_hash.contains_key(&callee_address) { + let code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((code_hash, next_ctx_available)); + } else { + // This case happens if calling an EOA. This is described + // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun + trace!( + "Callee address {} has no associated `code_hash`.", + &callee_address + ); + } + next_ctx_available += 1; + prev_jump = None; + } + "CREATE" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 3; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [_value, _offset, _size, _salt, ..] = evm_stack[..] else { + unreachable!() + }; + + let contract_address = tx.from.create(tx.nonce); + let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((code_hash, next_ctx_available)); + + next_ctx_available += 1; + prev_jump = None; + } + "CREATE2" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 4; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [_value, offset, size, salt, ..] = evm_stack[..] else { + unreachable!() + }; + ensure!(*offset < U256::from(usize::MAX)); + let offset: usize = offset.to(); + ensure!(*size < U256::from(usize::MAX)); + let size: usize = size.to(); + let salt: [u8; 32] = salt.to_be_bytes(); + + ensure!( + size == 0 + || (entry.memory.is_some() && entry.memory.as_ref().unwrap().len() >= size), + "No or insufficient memory available for {op}." + ); + let memory_raw: &[String] = entry.memory.as_ref().unwrap(); + let memory: Vec = memory_raw + .iter() + .flat_map(|s| { + let c = s.parse(); + // ensure!(c.is_ok(), "No memory."); + let a: U256 = c.unwrap(); + let d: [u8; 32] = a.to_be_bytes(); + d + }) + .collect(); + let init_code = &memory[offset..offset + size]; + let contract_address = tx.from.create2_from_code(salt, init_code); + let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((code_hash, next_ctx_available)); + + next_ctx_available += 1; + prev_jump = None; + } + "JUMP" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 1; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [counter, ..] = evm_stack[..] else { + unreachable!() + }; + let jump_target = counter.to::(); + + prev_jump = Some(jump_target); + } + "JUMPI" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 2; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [pc, condition, ..] = evm_stack[..] else { + unreachable!() + }; + let jump_target = pc.to::(); + let jump_condition = condition.is_zero().not(); + + prev_jump = if jump_condition { + Some(jump_target) + } else { + None + }; + } + "JUMPDEST" => { + let jumped_here = if let Some(jmp_target) = prev_jump { + jmp_target == entry.pc + } else { + false + }; + let jumpdest_offset = entry.pc as usize; + if jumped_here { + jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); + } + // else: we do not care about JUMPDESTs reached through fall-through. + prev_jump = None; + } + "EXTCODECOPY" | "EXTCODESIZE" => { + next_ctx_available += 1; + prev_jump = None; + } + // "RETURN" | "REVERT" | "STOP" => { + // ensure!(call_stack.is_empty().not(), "Call stack was empty at {op}."); + // do_pop = true; + // prev_jump = None; + // } + // "SELFDESTRUCT" => { + // do_pop = true; + // prev_jump = None; + // } + _ => { + prev_jump = None; + } + } + } + Ok(jumpdest_table) +} + +pub mod structlogprime { + use core::option::Option::None; + use std::collections::BTreeMap; + + use alloy::rpc::types::trace::geth::DefaultFrame; + use alloy_primitives::{Bytes, B256, U256}; + use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; + use serde_json::Value; + + /// Geth Default struct log trace frame + /// + /// + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub(crate) struct DefaultFramePrime { + /// Whether the transaction failed + pub failed: bool, + /// How much gas was used. + pub gas: u64, + /// Output of the transaction + #[serde(serialize_with = "alloy_serde::serialize_hex_string_no_prefix")] + pub return_value: Bytes, + /// Recorded traces of the transaction + pub struct_logs: Vec, + } + + /// Represents a struct log entry in a trace + /// + /// + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] + pub(crate) struct StructLogPrime { + /// Program counter + pub pc: u64, + /// Opcode to be executed + pub op: String, + /// Remaining gas + pub gas: u64, + /// Cost for executing op + #[serde(rename = "gasCost")] + pub gas_cost: u64, + /// Current call depth + pub depth: u64, + /// Error message if any + #[serde(default, skip)] + pub error: Option, + /// EVM stack + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stack: Option>, + /// Last call's return data. Enabled via enableReturnData + #[serde( + default, + rename = "returnData", + skip_serializing_if = "Option::is_none" + )] + pub return_data: Option, + /// ref + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option>, + /// Size of memory. + #[serde(default, rename = "memSize", skip_serializing_if = "Option::is_none")] + pub memory_size: Option, + /// Storage slots of current contract read from and written to. Only + /// emitted for SLOAD and SSTORE. Disabled via disableStorage + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_string_storage_map_opt" + )] + pub storage: Option>, + /// Refund counter + #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] + pub refund_counter: Option, + } + + /// Serializes a storage map as a list of key-value pairs _without_ + /// 0x-prefix + pub(crate) fn serialize_string_storage_map_opt( + storage: &Option>, + s: S, + ) -> Result { + match storage { + None => s.serialize_none(), + Some(storage) => { + let mut m = s.serialize_map(Some(storage.len()))?; + for (key, val) in storage.iter() { + let key = format!("{:?}", key); + let val = format!("{:?}", val); + // skip the 0x prefix + m.serialize_entry(&key.as_str()[2..], &val.as_str()[2..])?; + } + m.end() + } + } + } + + impl TryInto for DefaultFramePrime { + fn try_into(self) -> Result { + let a = serde_json::to_string(&self)?; + let b: DefaultFramePrime = serde_json::from_str(&a)?; + let c = serde_json::to_string(&b)?; + let d: DefaultFrame = serde_json::from_str(&c)?; + Ok(d) + } + + type Error = anyhow::Error; + } + + pub fn try_reserialize(structlog_object: Value) -> anyhow::Result { + let a = serde_json::to_string(&structlog_object)?; + let b: DefaultFramePrime = serde_json::from_str(&a)?; + let d: DefaultFrame = b.try_into()?; + Ok(d) + } +} diff --git a/zero_bin/rpc/src/lib.rs b/zero_bin/rpc/src/lib.rs index 87a581a14..d54d1b88d 100644 --- a/zero_bin/rpc/src/lib.rs +++ b/zero_bin/rpc/src/lib.rs @@ -21,6 +21,7 @@ use trace_decoder::{BlockLevelData, OtherBlockData}; use tracing::warn; pub mod jerigon; +pub mod jumpdest; pub mod native; pub mod retry; diff --git a/zero_bin/rpc/src/main.rs b/zero_bin/rpc/src/main.rs index 12594877a..e9ddd0122 100644 --- a/zero_bin/rpc/src/main.rs +++ b/zero_bin/rpc/src/main.rs @@ -217,6 +217,9 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::Registry::default() .with( tracing_subscriber::fmt::layer() + // With the default configuration trace information is written + // to stdout, but we already use to write our payload (the witness). + .with_writer(std::io::stderr) .with_ansi(false) .compact() .with_filter(EnvFilter::from_default_env()), diff --git a/zero_bin/rpc/src/native/mod.rs b/zero_bin/rpc/src/native/mod.rs index 2e9527274..f128e6d6c 100644 --- a/zero_bin/rpc/src/native/mod.rs +++ b/zero_bin/rpc/src/native/mod.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::ops::Deref; use std::sync::Arc; @@ -15,7 +14,8 @@ use zero_bin_common::provider::CachedProvider; mod state; mod txn; -type CodeDb = BTreeSet>; +pub use txn::process_transaction; +pub use txn::process_transactions; /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( @@ -39,7 +39,7 @@ where } /// Processes the block with the given block number and returns the block trace. -async fn process_block_trace( +pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, ) -> anyhow::Result diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 5e3be656a..10a110e83 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -9,25 +9,27 @@ use alloy::{ Provider, }, rpc::types::{ - eth::Transaction, - eth::{AccessList, Block}, + eth::{AccessList, Block, Transaction}, trace::geth::{ - AccountState, DiffMode, GethDebugBuiltInTracerType, GethTrace, PreStateConfig, - PreStateFrame, PreStateMode, + AccountState, DiffMode, GethDebugBuiltInTracerType, GethDebugTracerType, + GethDebugTracingOptions, GethTrace, PreStateConfig, PreStateFrame, PreStateMode, + StructLog, }, - trace::geth::{GethDebugTracerType, GethDebugTracingOptions}, }, transports::Transport, }; use anyhow::Context as _; +use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use super::CodeDb; -use crate::Compat; +use crate::{ + jumpdest::{self, structlogprime::try_reserialize}, + Compat, +}; /// Processes the transactions in the given block and updates the code db. -pub(super) async fn process_transactions( +pub async fn process_transactions( block: &Block, provider: &ProviderT, ) -> anyhow::Result<(CodeDb, Vec)> @@ -55,7 +57,7 @@ where /// Processes the transaction with the given transaction hash and updates the /// accounts state. -async fn process_transaction( +pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, ) -> anyhow::Result<(CodeDb, TxnInfo)> @@ -63,17 +65,12 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace) = fetch_tx_data(provider, &tx.hash).await?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + fetch_tx_data(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let tx_meta = TxnMeta { - byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), - new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), - gas_used: tx_receipt.gas_used as u64, - }; - let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), @@ -85,7 +82,30 @@ where // Handle case when transaction failed and a contract creation was reverted if !tx_status && tx_receipt.contract_address.is_some() { tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); - } + }; + + let struct_logs_opt: Option> = match structlog_trace { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + }; + + let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) + .map(Some) + .unwrap_or_default() + }); + + // if jumpdest_table.is_some() { eprintln!("======================> 1")}; + + let tx_meta = TxnMeta { + byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), + new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), + gas_used: tx_receipt.gas_used as u64, + jumpdest_table, + }; Ok(( code_db, @@ -103,7 +123,12 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, -) -> anyhow::Result<(::ReceiptResponse, GethTrace, GethTrace), anyhow::Error> +) -> anyhow::Result<( + ::ReceiptResponse, + GethTrace, + GethTrace, + GethTrace, +)> where ProviderT: Provider, TransportT: Transport + Clone, @@ -111,14 +136,21 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); + let structlog_trace_fut = + provider.debug_trace_transaction(*tx_hash, jumpdest::structlog_tracing_options()); - let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut + )?; Ok(( tx_receipt.context("Transaction receipt not found.")?, pre_trace, diff_trace, + structlog_trace, )) } diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index 815a7048d..f3d128d46 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -17,24 +17,25 @@ else num_procs=$(nproc) fi -# Force the working directory to always be the `tools/` directory. +# Force the working directory to always be the `tools/` directory. TOOLS_DIR=$(dirname $(realpath "$0")) PROOF_OUTPUT_DIR="${TOOLS_DIR}/proofs" -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" -echo "Block batch size: $BLOCK_BATCH_SIZE" +BATCH_SIZE=${BATCH_SIZE:=1} +echo "Batch size: $BATCH_SIZE" OUTPUT_LOG="${TOOLS_DIR}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" TEST_OUT_PATH="${TOOLS_DIR}/test.out" + # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs -export TOKIO_WORKER_THREADS=$num_procs +#export TOKIO_WORKER_THREADS=$num_procs -export RUST_MIN_STACK=33554432 -export RUST_BACKTRACE=full -export RUST_LOG=info +#export RUST_MIN_STACK=33554432 +#export RUST_BACKTRACE=full +#export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' @@ -95,7 +96,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --quiet --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $TEST_OUT_PATH + cargo run --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" rm $TEST_OUT_PATH @@ -108,10 +109,9 @@ fi cargo build --release --jobs "$num_procs" - start_time=$(date +%s%N) -"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ - --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG +"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE \ + --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $OUTPUT_LOG end_time=$(date +%s%N) set +o pipefail From 459148215f9e3b92d8a041ae92a1ec4eceecf296 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 20:26:10 +0200 Subject: [PATCH 002/107] feedback + cleanups --- zero_bin/rpc/src/jumpdest.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 7e9bde277..2a35cc877 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -105,11 +105,11 @@ pub(crate) fn generate_jumpdest_table( ensure!(curr_depth <= next_ctx_available, "Structlog is malformed."); - // ensure!(call_stack.is_empty().not(), "Call stack was empty."); while curr_depth < call_stack.len() { call_stack.pop(); } + ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); trace!("TX: {:?}", tx.hash); @@ -179,6 +179,7 @@ pub(crate) fn generate_jumpdest_table( }; let contract_address = tx.from.create(tx.nonce); + ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((code_hash, next_ctx_available)); @@ -202,26 +203,30 @@ pub(crate) fn generate_jumpdest_table( let offset: usize = offset.to(); ensure!(*size < U256::from(usize::MAX)); let size: usize = size.to(); + let memory_size = entry.memory.as_ref().unwrap().len(); let salt: [u8; 32] = salt.to_be_bytes(); ensure!( - size == 0 - || (entry.memory.is_some() && entry.memory.as_ref().unwrap().len() >= size), - "No or insufficient memory available for {op}." + entry.memory.is_some() && size <= memory_size, + "No or insufficient memory available for {op}. Contract size is {size} while memory size is {memory_size}." ); let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory: Vec = memory_raw + let memory_parsed: Vec> = memory_raw .iter() - .flat_map(|s| { + .map(|s| { let c = s.parse(); - // ensure!(c.is_ok(), "No memory."); + ensure!(c.is_ok(), "Parsing memory failed."); let a: U256 = c.unwrap(); let d: [u8; 32] = a.to_be_bytes(); - d + Ok(d) }) .collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + let memory: Vec = mem_res?.concat(); + let init_code = &memory[offset..offset + size]; let contract_address = tx.from.create2_from_code(salt, init_code); + ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((code_hash, next_ctx_available)); @@ -284,15 +289,6 @@ pub(crate) fn generate_jumpdest_table( next_ctx_available += 1; prev_jump = None; } - // "RETURN" | "REVERT" | "STOP" => { - // ensure!(call_stack.is_empty().not(), "Call stack was empty at {op}."); - // do_pop = true; - // prev_jump = None; - // } - // "SELFDESTRUCT" => { - // do_pop = true; - // prev_jump = None; - // } _ => { prev_jump = None; } From 91c29454e9509ad13eb5e5235e84eae967575132 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 21:35:55 +0200 Subject: [PATCH 003/107] cleanups --- test_jerigon.sh | 9 +++++---- zero_bin/rpc/src/jerigon.rs | 2 ++ zero_bin/rpc/src/native/txn.rs | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index 49f88ed12..fe335869b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -60,17 +60,18 @@ do done # TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -TESTNETBLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -TESTNETBLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` +BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` SHUF=`shuf -e $TESTNETBLOCKS` echo $SHUF +echo "Testing: $BLOCKS" -for BLOCK in $TESTNETBLOCKS; do +for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` - WITNESS="$witnesses/$BLOCK.jerigon.$GITHASH.witness.json" + WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Checking block $BLOCK" diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 3a9f7f866..2fefd90ec 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -19,6 +19,7 @@ use serde_json::json; use trace_decoder::{ BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, }; +use tracing::debug; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; @@ -177,6 +178,7 @@ where let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) + .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) .map(Some) .unwrap_or_default() }); diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 10a110e83..f0be69ebf 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -22,6 +22,7 @@ use anyhow::Context as _; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; +use tracing::debug; use crate::{ jumpdest::{self, structlogprime::try_reserialize}, @@ -94,12 +95,11 @@ where let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) + .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) .map(Some) .unwrap_or_default() }); - // if jumpdest_table.is_some() { eprintln!("======================> 1")}; - let tx_meta = TxnMeta { byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), From b58c5d67cfd4f0e7948e6a2419318ce5e0a40f25 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 23:08:47 +0200 Subject: [PATCH 004/107] fix overflow --- zero_bin/rpc/src/jumpdest.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 2a35cc877..b4f048f28 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -199,9 +199,9 @@ pub(crate) fn generate_jumpdest_table( let [_value, offset, size, salt, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*offset < U256::from(usize::MAX)); + ensure!(*offset <= U256::from(usize::MAX)); let offset: usize = offset.to(); - ensure!(*size < U256::from(usize::MAX)); + ensure!(*size <= U256::from(usize::MAX)); let size: usize = size.to(); let memory_size = entry.memory.as_ref().unwrap().len(); let salt: [u8; 32] = salt.to_be_bytes(); @@ -246,7 +246,8 @@ pub(crate) fn generate_jumpdest_table( let [counter, ..] = evm_stack[..] else { unreachable!() }; - let jump_target = counter.to::(); + ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + let jump_target: u64 = counter.to(); prev_jump = Some(jump_target); } @@ -260,10 +261,11 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [pc, condition, ..] = evm_stack[..] else { + let [counter, condition, ..] = evm_stack[..] else { unreachable!() }; - let jump_target = pc.to::(); + ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { From 037fb5710c5fd66888f1fe1f4a00960c5eddca03 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 10:58:36 +0200 Subject: [PATCH 005/107] fmt --- zero_bin/rpc/src/jumpdest.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index b4f048f28..7ea971e13 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -246,7 +246,10 @@ pub(crate) fn generate_jumpdest_table( let [counter, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + ensure!( + *counter <= U256::from(u64::MAX), + "Operand for {op} caused overflow." + ); let jump_target: u64 = counter.to(); prev_jump = Some(jump_target); @@ -264,7 +267,10 @@ pub(crate) fn generate_jumpdest_table( let [counter, condition, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + ensure!( + *counter <= U256::from(u64::MAX), + "Operand for {op} caused overflow." + ); let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); From 85ee8c274b44a7506c7e21732c88993c76e88fe3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 14:44:37 +0200 Subject: [PATCH 006/107] fix testscripts --- test_jerigon.sh | 18 ++++++++++-------- test_native.sh | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index fe335869b..deac74b2b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euxo pipefail +set -uxo pipefail #export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= @@ -15,7 +15,7 @@ export TOKIO_WORKER_THREADS=4 export RUST_BACKTRACE=full export RUST_LOG=info export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=33554432 +export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." @@ -60,16 +60,18 @@ do done # TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +# BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +#BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` -BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` SHUF=`shuf -e $TESTNETBLOCKS` echo $SHUF -echo "Testing: $BLOCKS" +#echo "Testing: $BLOCKS" +printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt +printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in $BLOCKS; do +for BLOCK in {1..256}; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" @@ -77,13 +79,13 @@ for BLOCK in $BLOCKS; do echo "Checking block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? - if [ -n $EXITCODE ] + if [ $EXITCODE -eq 0 ] then RESULT="success" else RESULT="failure" fi - printf "%10i %s %s\n" $BLOCK $GITHASH $RESULT | tee -a result.txt + printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt done exit 0 diff --git a/test_native.sh b/test_native.sh index 913f27f13..fcb85fba5 100755 --- a/test_native.sh +++ b/test_native.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euxo pipefail +set -uxo pipefail export RPC= if [ -z $RPC ]; then @@ -49,10 +49,10 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" until [ -f $WITNESS -a -s $WITNESS ]; do echo "Fetching block $BLOCK" - cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --release --verbose --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS EXITCODE=$? - if [ -n $EXITCODE -a -f $WITNESS -a -s $WITNESS ] + if [ $EXITCODE -eq 0 -a -f $WITNESS -a -s $WITNESS ] then printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt break @@ -71,7 +71,7 @@ for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? - if [ -n $EXITCODE ] + if [ $EXITCODE -eq 0 ] then RESULT="success" else From 1243768b0b026305d326394903bed23c5ba8b546 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 14:45:24 +0200 Subject: [PATCH 007/107] refactor --- zero_bin/rpc/src/jerigon.rs | 34 ++++++----------- zero_bin/rpc/src/jumpdest.rs | 68 +++++++++++++++++++++++++++++----- zero_bin/rpc/src/native/txn.rs | 32 +++++++--------- 3 files changed, 84 insertions(+), 50 deletions(-) diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 2fefd90ec..3c36f7454 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -1,11 +1,9 @@ use std::collections::BTreeMap; use std::ops::Deref as _; -use alloy::providers::ext::DebugApi; -use alloy::rpc::types::trace::geth::StructLog; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, trace::geth::GethTrace, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, Block, BlockTransactionsKind, Transaction}, transports::Transport, }; use alloy_primitives::Address; @@ -23,8 +21,7 @@ use tracing::debug; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; -use crate::jumpdest; -use crate::jumpdest::structlogprime::try_reserialize; +use crate::jumpdest::{self, get_normalized_structlog}; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] @@ -159,28 +156,21 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let structlog_trace = provider - .debug_trace_transaction(tx.hash, jumpdest::structlog_tracing_options()) - .await?; - - let struct_logs_opt: Option> = match structlog_trace { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - }; - let tx_traces = tx_trace .iter() .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) .collect(); - let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { - jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) - .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) - .map(Some) - .unwrap_or_default() + let structlog_opt = get_normalized_structlog(provider, &tx.hash).await?; + + let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { + jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( + |error| { + debug!("JumpDestTable generation failed with reason: {}", error); + None + }, + Some, + ) }); Ok(jumpdest_table) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 7ea971e13..97531eee9 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -7,25 +7,31 @@ use std::sync::OnceLock; use __compat_primitive_types::H256; use alloy::primitives::Address; use alloy::primitives::U160; +use alloy::providers::ext::DebugApi; +use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; use alloy::rpc::types::trace::geth::StructLog; use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; +use alloy::transports::RpcError; +use alloy::transports::Transport; +use alloy::transports::TransportErrorKind; +use alloy_primitives::B256; use alloy_primitives::U256; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use structlogprime::normalize_structlog; use trace_decoder::TxnTrace; use tracing::trace; -/// Tracing options for the `debug_traceTransaction` call to get structlog. -/// Used for filling JUMPDEST table. -pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { +/// Pass `true` for the components needed. +fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { GethDebugTracingOptions { config: GethDefaultTracingOptions { - disable_stack: Some(false), + disable_stack: Some(!stack), // needed for CREATE2 - disable_memory: Some(false), - disable_storage: Some(true), + disable_memory: Some(!memory), + disable_storage: Some(!storage), ..GethDefaultTracingOptions::default() }, tracer: None, @@ -33,6 +39,38 @@ pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { } } +fn trace_contains_create2(structlog: Vec) -> bool { + structlog.iter().any(|entry| entry.op == "CREATE2") +} + +// Gets the lightest possible structlog for transcation `tx_hash`. +pub(crate) async fn get_normalized_structlog( + provider: &ProviderT, + tx_hash: &B256, +) -> Result>, RpcError> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let light_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) + .await?; + + let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; + + let need_memory = structlogs_opt.is_some_and(trace_contains_create2); + + let structlog = provider + .debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ) + .await?; + + let ret = normalize_structlog(structlog).await; + Ok(ret) +} + /// Provides a way to check in constant time if an address points to a /// precompile. fn precompiles() -> &'static HashSet
{ @@ -309,7 +347,7 @@ pub mod structlogprime { use core::option::Option::None; use std::collections::BTreeMap; - use alloy::rpc::types::trace::geth::DefaultFrame; + use alloy::rpc::types::trace::geth::{DefaultFrame, GethTrace, StructLog}; use alloy_primitives::{Bytes, B256, U256}; use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; use serde_json::Value; @@ -412,10 +450,22 @@ pub mod structlogprime { type Error = anyhow::Error; } - pub fn try_reserialize(structlog_object: Value) -> anyhow::Result { - let a = serde_json::to_string(&structlog_object)?; + pub fn try_reserialize(structlog_object: &Value) -> anyhow::Result { + let a = serde_json::to_string(structlog_object)?; let b: DefaultFramePrime = serde_json::from_str(&a)?; let d: DefaultFrame = b.try_into()?; Ok(d) } + + pub(crate) async fn normalize_structlog( + unnormalized_structlog: GethTrace, + ) -> Option> { + match unnormalized_structlog { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(&structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + } + } } diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index f0be69ebf..80d2f367b 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -18,14 +18,14 @@ use alloy::{ }, transports::Transport, }; -use anyhow::Context as _; +use anyhow::{Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::debug; use crate::{ - jumpdest::{self, structlogprime::try_reserialize}, + jumpdest::{self, get_normalized_structlog}, Compat, }; @@ -66,7 +66,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + let (tx_receipt, pre_trace, diff_trace, structlog_opt) = fetch_tx_data(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); @@ -85,19 +85,14 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let struct_logs_opt: Option> = match structlog_trace { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - }; - - let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { - jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) - .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) - .map(Some) - .unwrap_or_default() + let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( + |error| { + debug!("JumpDestTable generation failed with reason: {}", error); + None + }, + Some, + ) }); let tx_meta = TxnMeta { @@ -127,7 +122,7 @@ async fn fetch_tx_data( ::ReceiptResponse, GethTrace, GethTrace, - GethTrace, + Option>, )> where ProviderT: Provider, @@ -136,8 +131,7 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - let structlog_trace_fut = - provider.debug_trace_transaction(*tx_hash, jumpdest::structlog_tracing_options()); + let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( tx_receipt_fut, From e7244c6bf648343800572a10024d91050cfd5cd7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 22:29:01 +0200 Subject: [PATCH 008/107] for testing --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 10 ++++++++-- test_jerigon.sh | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index d0cb867fa..19b1db7c8 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; use keccak_hash::H256; -use log::Level; +use log::{trace, Level}; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -167,7 +167,13 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( let ctx_proofs = (*jumpdest_table_rpc) .iter() .flat_map(|(code_addr, ctx_jumpdests)| { - prove_context_jumpdests(&code_map[code_addr], ctx_jumpdests) + let code = if code_map.contains_key(code_addr) { + &code_map[code_addr] + } else { + &vec![] + }; + trace!("code: {:?} <============", &code); + prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); JumpDestTableProcessed::new(ctx_proofs) diff --git a/test_jerigon.sh b/test_jerigon.sh index deac74b2b..0bd16b311 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -41,6 +41,13 @@ TESTNETBLOCKS=" KNOWNFAILED=" +2 +15 +28 +35 +37 +43 +65 28 444 " @@ -71,7 +78,7 @@ echo $SHUF printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in {1..256}; do +for BLOCK in {66..688}; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" From 16e9c261ed14dcb9ec666863635f61deb6e1716d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 17 Sep 2024 22:24:29 +0200 Subject: [PATCH 009/107] extract initcode --- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 7 +++- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- evm_arithmetization/src/generation/mod.rs | 4 +- .../src/generation/prover_input.rs | 21 +++++++--- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- test_jerigon.sh | 27 +++++++++++-- trace_decoder/src/core.rs | 39 +++++++++++++++---- 15 files changed, 90 insertions(+), 30 deletions(-) diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 57b902c63..2b35b2a1c 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,7 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 19b1db7c8..7bc5e1dcc 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -172,7 +172,12 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( } else { &vec![] }; - trace!("code: {:?} <============", &code); + trace!( + "code: {:?}, code_addr: {:?} <============", + &code, + &code_addr + ); + trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index 1cc5588e1..fbae6647d 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,7 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; @@ -372,7 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 08fe57b1a..673b516b0 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 87d9ed1f7..e5ec7bde3 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -111,7 +111,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_tables: Vec>, + pub jumpdest_tables: Option, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -165,7 +165,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_tables: Vec>, + pub jumpdest_tables: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index ae49366b3..dc59bda2c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,6 +7,7 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; +use log::trace; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -769,15 +770,18 @@ impl GenerationState { dbg!(&self.inputs.jumpdest_tables); eprintln!("Generating JUMPDEST tables"); // w for witness - let txn_idx = self.next_txn_index - 1; - let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref(); - let rpc = rpcw.map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + // let txn_idx = self.next_txn_index - 1; + // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code + let rpcw = &self.inputs.jumpdest_tables; + let rpc = rpcw + .as_ref() + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { - if rpcw.is_some() && rpcw.unwrap() != &simw.clone() { + if rpcw.is_some() && rpcw.clone().unwrap() != simw.clone() { println!("SIMW {}", simw.clone()); - println!("RPCW {}", rpcw.unwrap()); - assert_eq!(simw.clone(), *rpcw.unwrap()); + println!("RPCW {}", rpcw.clone().unwrap()); + assert_eq!(simw.clone(), rpcw.clone().unwrap()); } } self.jumpdest_table = rpc; @@ -796,6 +800,11 @@ impl GenerationState { self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); + trace!( + "ctx: {ctx}, code_hash: {:?} code: {:?}", + keccak(code.clone()), + code + ); for offset in jumpdest_table.clone() { jdtw.insert(keccak(code.clone()), ctx, offset); } diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 51ac662df..12a84fbe7 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index f13f541b4..475d03a8e 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,7 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 6386dbe69..8d30f85d1 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,7 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index d9fe6f234..951ebbb28 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,7 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 27d68af7b..221113b26 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,7 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 4109ae09c..de8e096cf 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,7 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 3dfe13e43..2aa776e54 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,7 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/test_jerigon.sh b/test_jerigon.sh index 0bd16b311..b1021b03b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -48,10 +48,31 @@ KNOWNFAILED=" 37 43 65 + 28 444 -" +43 +460 +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +566 +662 +664 +665 +667 +670 +72 +77 +" # 470..663 from Robin @@ -78,11 +99,11 @@ echo $SHUF printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in {66..688}; do +for BLOCK in $KNOWNFAILED; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" - cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Checking block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 243bacf29..2cfb934ed 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -1,4 +1,4 @@ -use core::option::Option::None; +use core::{convert::Into as _, option::Option::None}; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -6,6 +6,11 @@ use std::{ }; use alloy::primitives::address; +use alloy::{ + consensus::{Transaction, TxEnvelope}, + primitives::TxKind, + rlp::Decodable as _, +}; use alloy_compat::Compat as _; use anyhow::{anyhow, bail, ensure, Context as _}; use ethereum_types::{Address, U256}; @@ -97,7 +102,7 @@ pub fn entrypoint( running_gas_used += gas_used; running_gas_used.into() }, - signed_txns: byte_code.into_iter().map(Into::into).collect(), + signed_txns: byte_code.clone().into_iter().map(Into::into).collect(), withdrawals, ger_data: None, tries: TrieInputs { @@ -109,14 +114,34 @@ pub fn entrypoint( trie_roots_after: after, checkpoint_state_trie_root, checkpoint_consolidated_hash, - contract_code: contract_code - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(), + contract_code: { + let cc = contract_code.into_iter(); + + let initcodes = + byte_code + .iter() + .filter_map(|nonempty_txn_raw| -> Option> { + let tx_envelope = + TxEnvelope::decode(&mut &nonempty_txn_raw[..]).unwrap(); + match tx_envelope.to() { + TxKind::Create => Some(tx_envelope.input().to_vec()), + TxKind::Call(_address) => None, + } + }); + + cc.chain(initcodes) + .map(|it| (keccak_hash::keccak(&it), it)) + .collect() + }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_tables, + jumpdest_tables: { + jumpdest_tables + .into_iter() + .collect::>>() + .map(|vj| JumpDestTableWitness::merge(vj.iter()).0) + }, }, ) .collect()) From f3871d96717b601852102384f013cae6f683964e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 17 Sep 2024 22:47:49 +0200 Subject: [PATCH 010/107] improve test script --- test_jerigon.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index b1021b03b..53309228c 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -uxo pipefail +set -uo pipefail #export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= @@ -20,7 +20,7 @@ export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." -TESTNETBLOCKS=" +CIBLOCKS=" 1 2 3 @@ -81,30 +81,28 @@ do ROBIN+=" $i" done -# Pick random blocks -for i in {1..10} -do - RANDOMBLOCKS+=" $((1 + $RANDOM % 688))" -done - -# TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -# BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -#BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` - -SHUF=`shuf -e $TESTNETBLOCKS` -echo $SHUF +TIP=688 +NUMRANDOMBLOCKS=10 +RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` +# CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" +#BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" +#BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` +BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt -printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt +echo "---------------------------" | tee -a witnesses/jerigon_results.txt -for BLOCK in $KNOWNFAILED; do +for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Checking block $BLOCK" + echo "Testing blocks:" + echo $BLOCKS + echo "Now testing block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] From 4fd6b8b7028c6a9d769b20f612533499018609ff Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 10:35:20 +0200 Subject: [PATCH 011/107] fix stack issue --- zero_bin/rpc/src/jumpdest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 97531eee9..67c5e68a2 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -212,7 +212,7 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [_value, _offset, _size, _salt, ..] = evm_stack[..] else { + let [_value, _offset, _size, ..] = evm_stack[..] else { unreachable!() }; From 88eb73d7db4f49e5f440e3c7d8e875c5343d7cd8 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 13:23:07 +0200 Subject: [PATCH 012/107] random fixes --- Cargo.toml | 2 +- .../src/generation/prover_input.rs | 24 +++--- test_jerigon.sh | 77 +++++++++++++++++-- zero_bin/rpc/src/jumpdest.rs | 4 +- 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 84a438806..294681d05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ tiny-keccak = "2.0.2" tokio = { version = "1.38.0", features = ["full"] } toml = "0.8.14" tower = "0.4" -tracing = "0.1" +tracing = { version = "0.1", features = ["attributes"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } u4 = "0.1.0" uint = "0.9.5" diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index dc59bda2c..8751e2ded 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,24 +767,30 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - dbg!(&self.inputs.jumpdest_tables); eprintln!("Generating JUMPDEST tables"); + dbg!(&self.inputs.jumpdest_tables); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code let rpcw = &self.inputs.jumpdest_tables; - let rpc = rpcw + let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { - if rpcw.is_some() && rpcw.clone().unwrap() != simw.clone() { - println!("SIMW {}", simw.clone()); - println!("RPCW {}", rpcw.clone().unwrap()); - assert_eq!(simw.clone(), rpcw.clone().unwrap()); - } + let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); + + let (sim, simw): (Option, Option) = + sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + + if let (Some(rw), Some(sw)) = (rpcw, simw) + && rw != &sw + { + trace!("SIMW {}", sw); + trace!("RPCW {}", rw); + assert_eq!(rw, &sw); } - self.jumpdest_table = rpc; + + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/test_jerigon.sh b/test_jerigon.sh index 53309228c..2b1f5c2d5 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -40,7 +40,71 @@ CIBLOCKS=" " -KNOWNFAILED=" +STILLFAIL=" +37 +75 +15 +35 +43 +72 +77 +184 +460 +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +566 +662 +664 +665 +667 +670 +477 +478 +444 +" + +JUMPI=" +662 +664 +665 +667 +670 +" + +CONTAINSKEY=" +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +72 +" + +CREATE2=" +43 +566 +77 +" + +DECODING=" +477 +478 +" + +USEDTOFAIL=" 2 15 28 @@ -50,7 +114,6 @@ KNOWNFAILED=" 65 28 -444 43 460 @@ -88,7 +151,8 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` # CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` -BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +#BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +BLOCKS="$DECODING" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" @@ -99,10 +163,9 @@ for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" - timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Testing blocks:" - echo $BLOCKS - echo "Now testing block $BLOCK" + timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Testing blocks: $BLOCKS." + echo "Now testing block $BLOCK .." zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 67c5e68a2..e31e90bf5 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -82,13 +82,13 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. +#[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], tx_traces: &BTreeMap, ) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); - ensure!(struct_log.is_empty().not(), "Structlog is empty."); let mut jumpdest_table = JumpDestTableWitness::default(); @@ -123,7 +123,7 @@ pub(crate) fn generate_jumpdest_table( keccak(init) }; - // `None` encodes that previous `entry`` was not a JUMP or JUMPI with true + // `None` encodes that previous `entry` was not a JUMP or JUMPI with true // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with // true condition and target `jump_target`. let mut prev_jump = None; From 39cd26c77d5f395fd3d1cf1e4c06e62b69ee2916 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 16:37:42 +0200 Subject: [PATCH 013/107] fix CREATE2 --- .../src/generation/prover_input.rs | 18 +++-- test_jerigon.sh | 2 + zero_bin/rpc/src/jerigon.rs | 13 ++-- zero_bin/rpc/src/jumpdest.rs | 66 ++++++++++++------- zero_bin/rpc/src/native/txn.rs | 2 +- zero_bin/tools/prove_stdio.sh | 4 +- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 8751e2ded..1919f98d2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,7 +7,7 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; -use log::trace; +use log::{info, trace}; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -767,7 +767,7 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - eprintln!("Generating JUMPDEST tables"); + info!("Generating JUMPDEST tables"); dbg!(&self.inputs.jumpdest_tables); // w for witness // let txn_idx = self.next_txn_index - 1; @@ -775,20 +775,18 @@ impl GenerationState { let rpcw = &self.inputs.jumpdest_tables; let rpc: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| set_jumpdest_analysis_inputs_rpc(&jdt, &self.inputs.contract_code)); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) - && rw != &sw - { - trace!("SIMW {}", sw); - trace!("RPCW {}", rw); - assert_eq!(rw, &sw); - } + if let (Some(rw), Some(sw)) = (rpcw, simw) && rw != &sw { + info!("SIMW {}", sw); + info!("RPCW {}", rw); + assert_eq!(rw, &sw); + } self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/test_jerigon.sh b/test_jerigon.sh index 2b1f5c2d5..5362d6156 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -163,9 +163,11 @@ for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." + export RUST_LOG=info zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 3c36f7454..0c3900ab8 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -3,7 +3,7 @@ use std::ops::Deref as _; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, transports::Transport, }; use alloy_primitives::Address; @@ -161,15 +161,20 @@ where .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) .collect(); - let structlog_opt = get_normalized_structlog(provider, &tx.hash).await?; + let structlog_opt: Option> = get_normalized_structlog(provider, &tx.hash) + .await + .ok() + .flatten(); let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!("JumpDestTable generation failed with reason: {}", error); + debug!("{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error); None }, - Some, + |jdt|{ + debug!("{:#?}: JumpDestTable generation succeceeded with result: {}",tx.hash, jdt); + Some(jdt)}, ) }); diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index e31e90bf5..321acd9fa 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -1,3 +1,5 @@ +use core::default::Default; +use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; @@ -21,8 +23,17 @@ use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; use structlogprime::normalize_structlog; +use tokio::time::timeout; use trace_decoder::TxnTrace; -use tracing::trace; +use tracing::{instrument, trace}; + +/// The maximum time we are willing to wait for a structlog before failing over +/// to simulating the JumpDest analysis. +const TIMEOUT_LIMIT: Duration = Duration::from_secs(10); + +/// Structure of Etheruem memory +type Word = [u8; 32]; +const WORDSIZE: usize = std::mem::size_of::(); /// Pass `true` for the components needed. fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { @@ -52,6 +63,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { + // Optimization: It may be a better default to pull the stack immediately. let light_structlog_trace = provider .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) .await?; @@ -59,16 +71,19 @@ where let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; let need_memory = structlogs_opt.is_some_and(trace_contains_create2); + trace!("Need structlog with memory: {need_memory}"); - let structlog = provider - .debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ) - .await?; + let structlog = provider.debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ); - let ret = normalize_structlog(structlog).await; - Ok(ret) + match timeout(TIMEOUT_LIMIT, structlog).await { + Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( + ellapsed_error, + )))), + Ok(structlog_res) => Ok(normalize_structlog(structlog_res?).await), + } } /// Provides a way to check in constant time if an address points to a @@ -119,7 +134,7 @@ pub(crate) fn generate_jumpdest_table( ); callee_addr_to_code_hash[&to_address] } else { - let init = tx.input.clone(); + let init = &tx.input; keccak(init) }; @@ -240,33 +255,36 @@ pub(crate) fn generate_jumpdest_table( ensure!(*offset <= U256::from(usize::MAX)); let offset: usize = offset.to(); ensure!(*size <= U256::from(usize::MAX)); + let size: usize = size.to(); - let memory_size = entry.memory.as_ref().unwrap().len(); - let salt: [u8; 32] = salt.to_be_bytes(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; + let salt: Word = salt.to_be_bytes(); ensure!( - entry.memory.is_some() && size <= memory_size, - "No or insufficient memory available for {op}. Contract size is {size} while memory size is {memory_size}." + entry.memory.is_some() && offset + size <= memory_size, + "Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size ); let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory_parsed: Vec> = memory_raw + let memory_parsed: Vec> = memory_raw .iter() .map(|s| { - let c = s.parse(); + // let c = s.parse(); + let c = U256::from_str_radix(s, 16); ensure!(c.is_ok(), "Parsing memory failed."); let a: U256 = c.unwrap(); - let d: [u8; 32] = a.to_be_bytes(); + let d: Word = a.to_be_bytes(); Ok(d) }) .collect(); - let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); let memory: Vec = mem_res?.concat(); let init_code = &memory[offset..offset + size]; - let contract_address = tx.from.create2_from_code(salt, init_code); - ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - let code_hash = callee_addr_to_code_hash[&contract_address]; - call_stack.push((code_hash, next_ctx_available)); + let init_code_hash = keccak(init_code); + // let contract_address = tx.from.create2_from_code(salt, init_code); + // ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); + // let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; prev_jump = None; @@ -286,7 +304,9 @@ pub(crate) fn generate_jumpdest_table( }; ensure!( *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow." + "Operand for {op} caused overflow: counter: {} is larger than u64::MAX {}", + *counter, + u64::MAX ); let jump_target: u64 = counter.to(); diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 80d2f367b..63e61d7e2 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -137,7 +137,7 @@ where tx_receipt_fut, pre_trace_fut, diff_trace_fut, - structlog_trace_fut + structlog_trace_fut, )?; Ok(( diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index f3d128d46..e00e79e8d 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -35,7 +35,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_MIN_STACK=33554432 #export RUST_BACKTRACE=full -#export RUST_LOG=info +#export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' @@ -102,7 +102,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then rm $TEST_OUT_PATH exit else - echo "Failed to create proof witnesses. See \"zk_evm/tools/test.out\" for more details." + echo "Failed to create proof witnesses. See \"zk_evm/zero_bin/tools/test.out\" for more details." exit 1 fi fi From 8a964b8bd2442c27b1f59c3dd61e3e502f5a80d2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 16:42:04 +0200 Subject: [PATCH 014/107] fmt, clippy --- .../src/generation/prover_input.rs | 14 ++++++++------ test_jerigon.sh | 2 +- zero_bin/rpc/src/jerigon.rs | 15 +++++++++++---- zero_bin/rpc/src/jumpdest.rs | 4 ++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 1919f98d2..77aaec99d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -775,18 +775,20 @@ impl GenerationState { let rpcw = &self.inputs.jumpdest_tables; let rpc: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(&jdt, &self.inputs.contract_code)); + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) && rw != &sw { - info!("SIMW {}", sw); - info!("RPCW {}", rw); - assert_eq!(rw, &sw); - } + if let (Some(rw), Some(sw)) = (rpcw, simw) + && rw != &sw + { + info!("SIMW {}", sw); + info!("RPCW {}", rw); + assert_eq!(rw, &sw); + } self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/test_jerigon.sh b/test_jerigon.sh index 5362d6156..fd7800872 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -152,7 +152,7 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` #BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -BLOCKS="$DECODING" +BLOCKS="$DECODING $CREATE2" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 0c3900ab8..82feee7a0 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -169,12 +169,19 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!("{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error); + debug!( + "{:#?}: JumpDestTable generation failed with reason: {}", + tx.hash, error + ); None }, - |jdt|{ - debug!("{:#?}: JumpDestTable generation succeceeded with result: {}",tx.hash, jdt); - Some(jdt)}, + |jdt| { + debug!( + "{:#?}: JumpDestTable generation succeceeded with result: {}", + tx.hash, jdt + ); + Some(jdt) + }, ) }); diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 321acd9fa..8e3ffe8f4 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -249,7 +249,7 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [_value, offset, size, salt, ..] = evm_stack[..] else { + let [_value, offset, size, _salt, ..] = evm_stack[..] else { unreachable!() }; ensure!(*offset <= U256::from(usize::MAX)); @@ -258,7 +258,7 @@ pub(crate) fn generate_jumpdest_table( let size: usize = size.to(); let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - let salt: Word = salt.to_be_bytes(); + // let salt: Word = salt.to_be_bytes(); ensure!( entry.memory.is_some() && offset + size <= memory_size, From 32e68bfa33c8ae3c0361ba85c9e61039225a32ae Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 13:46:54 +0200 Subject: [PATCH 015/107] investigate 15,35 --- .../src/generation/prover_input.rs | 5 ++- test_jerigon.sh | 26 ++++++++++-- trace_decoder/src/core.rs | 1 + zero_bin/rpc/src/jerigon.rs | 2 +- zero_bin/rpc/src/jumpdest.rs | 40 ++++++++++--------- zero_bin/tools/prove_stdio.sh | 2 +- 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 77aaec99d..c2b1d2169 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -790,7 +790,10 @@ impl GenerationState { assert_eq!(rw, &sw); } - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + info!("SIMW {:#?}", sim); + info!("RPCW {:#?}", rpc); + + self.jumpdest_table = sim;// = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/test_jerigon.sh b/test_jerigon.sh index fd7800872..a1a1c85ca 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -11,9 +11,9 @@ fi mkdir -p witnesses export RAYON_NUM_THREADS=4 -export TOKIO_WORKER_THREADS=4 +export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=info +export RUST_LOG=rpc=trace,evm_arithmetization::generation::prover_input=trace export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' export RUST_MIN_STACK=67108864 @@ -137,6 +137,23 @@ USEDTOFAIL=" 77 " +ROUND2=" +15 +35 +566 +664 +665 +667 +670 +" +#444 + +TESTED=" +4 +5 +28 +65 +" # 470..663 from Robin for i in {470..663} @@ -152,7 +169,8 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` #BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -BLOCKS="$DECODING $CREATE2" +#BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI" +BLOCKS="$ROUND2" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" @@ -164,7 +182,7 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 2cfb934ed..d227edfb8 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -76,6 +76,7 @@ pub fn entrypoint( withdrawals, )?; + dbg!(&batches.first().unwrap().jumpdest_tables); let mut running_gas_used = 0; Ok(batches .into_iter() diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 82feee7a0..b3742f27f 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -177,7 +177,7 @@ where }, |jdt| { debug!( - "{:#?}: JumpDestTable generation succeceeded with result: {}", + "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); Some(jdt) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 8e3ffe8f4..4861aeeca 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -1,4 +1,5 @@ use core::default::Default; +use core::option::Option::None; use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; @@ -97,7 +98,7 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -#[instrument] +// #[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], @@ -107,6 +108,7 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); + // This does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces .iter() .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) @@ -126,16 +128,16 @@ pub(crate) fn generate_jumpdest_table( } ); - let entrypoint_code_hash: H256 = if let Some(to_address) = tx.to { - // Guard against transactions to a non-contract address. - ensure!( - callee_addr_to_code_hash.contains_key(&to_address), - format!("Callee addr {} is not at contract address", to_address) - ); - callee_addr_to_code_hash[&to_address] - } else { - let init = &tx.input; - keccak(init) + let entrypoint_code_hash: H256 = match tx.to { + Some(to_address) if precompiles().contains(&to_address) => return Ok(jumpdest_table), + Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { + return Ok(jumpdest_table) + } + Some(to_address) => callee_addr_to_code_hash[&to_address], + None => { + let init = &tx.input; + keccak(init) + } }; // `None` encodes that previous `entry` was not a JUMP or JUMPI with true @@ -165,14 +167,14 @@ pub(crate) fn generate_jumpdest_table( ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); - trace!("TX: {:?}", tx.hash); - trace!("STEP: {:?}", step); - trace!("STEPS: {:?}", struct_log.len()); - trace!("OPCODE: {}", entry.op.as_str()); - trace!("CODE: {:?}", code_hash); - trace!("CTX: {:?}", ctx); - trace!("CURR_DEPTH: {:?}", curr_depth); - trace!("{:#?}\n", entry); + // trace!("TX: {:?}", tx.hash); + // trace!("STEP: {:?}", step); + // trace!("STEPS: {:?}", struct_log.len()); + // trace!("OPCODE: {}", entry.op.as_str()); + // trace!("CODE: {:?}", code_hash); + // trace!("CTX: {:?}", ctx); + // trace!("CURR_DEPTH: {:?}", curr_depth); + // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index e00e79e8d..88bc359fc 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -96,7 +96,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH + cargo run --quiet --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" rm $TEST_OUT_PATH From 184878dda760dd48cbf400d23407e69cacd4fbe0 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 21:08:52 +0200 Subject: [PATCH 016/107] fix scripts --- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 6 +++--- scripts/test_native.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index cffb39a34..80515c9e1 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index d81fd3067..240148594 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -138,8 +138,6 @@ USEDTOFAIL=" " ROUND2=" -15 -35 664 665 667 @@ -153,6 +151,8 @@ NOWSUCCESS=" 28 65 566 +15 +35 " # 470..663 from Robin @@ -186,7 +186,7 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - scripts/prove_stdio.sh $WITNESS test_only + prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 088205b22..d2a7aac5f 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -69,7 +69,7 @@ echo "Testing prepared witnesses.." for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" - zero_bin/tools/prove_stdio.sh $WITNESS test_only + prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then From c000b5a55d7bccdeed29da60559533114279aa77 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 22:46:57 +0200 Subject: [PATCH 017/107] remove redtape for JUMP/I --- .../src/cpu/kernel/tests/mod.rs | 2 +- .../src/generation/prover_input.rs | 6 +- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 5 +- zero/src/rpc/jerigon.rs | 11 ++-- zero/src/rpc/jumpdest.rs | 58 ++++++++++--------- zero/src/rpc/native/txn.rs | 2 +- 7 files changed, 45 insertions(+), 41 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index aae20ae78..f4fb947e6 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -26,9 +26,9 @@ mod signed_syscalls; mod transaction_parsing; mod transient_storage; -use ::core::iter::Iterator as _; use std::{ops::Range, str::FromStr}; +use ::core::iter::Iterator as _; use anyhow::Result; use ethereum_types::U256; use plonky2::hash::hash_types::RichField; diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e2ea4b179..c561504df 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -790,10 +790,10 @@ impl GenerationState { assert_eq!(rw, &sw); } - info!("SIMW {:#?}", sim); - info!("RPCW {:#?}", rpc); + info!("SIM {:#?}", sim); + info!("RPC {:#?}", rpc); - self.jumpdest_table = sim;// = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 80515c9e1..dc59806b9 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 240148594..baee31ea7 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -13,8 +13,7 @@ mkdir -p witnesses export RAYON_NUM_THREADS=4 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=rpc=trace,evm_arithmetization::generation::prover_input=trace -export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` @@ -142,10 +141,10 @@ ROUND2=" 665 667 670 -444 " NOWSUCCESS=" +444 4 5 28 diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 9a9946a23..ab8cf8d37 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -16,9 +16,12 @@ use serde_json::json; use trace_decoder::{ BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, }; -use tracing::debug; +use tracing::info; -use super::{fetch_other_block_data, jumpdest::{self, get_normalized_structlog}}; +use super::{ + fetch_other_block_data, + jumpdest::{self, get_normalized_structlog}, +}; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; /// Transaction traces retrieved from Erigon zeroTracer. @@ -167,14 +170,14 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!( + info!( "{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error ); None }, |jdt| { - debug!( + info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 4861aeeca..533a9229b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -143,7 +143,7 @@ pub(crate) fn generate_jumpdest_table( // `None` encodes that previous `entry` was not a JUMP or JUMPI with true // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with // true condition and target `jump_target`. - let mut prev_jump = None; + let mut prev_jump: Option = None; // Call depth of the previous `entry`. We initialize to 0 as this compares // smaller to 1. @@ -167,14 +167,14 @@ pub(crate) fn generate_jumpdest_table( ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); - // trace!("TX: {:?}", tx.hash); - // trace!("STEP: {:?}", step); - // trace!("STEPS: {:?}", struct_log.len()); - // trace!("OPCODE: {}", entry.op.as_str()); - // trace!("CODE: {:?}", code_hash); - // trace!("CTX: {:?}", ctx); - // trace!("CURR_DEPTH: {:?}", curr_depth); - // trace!("{:#?}\n", entry); + trace!("TX: {:?}", tx.hash); + trace!("STEP: {:?}", step); + trace!("STEPS: {:?}", struct_log.len()); + trace!("OPCODE: {}", entry.op.as_str()); + trace!("CODE: {:?}", code_hash); + trace!("CTX: {:?}", ctx); + trace!("CURR_DEPTH: {:?}", curr_depth); + trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { @@ -226,7 +226,7 @@ pub(crate) fn generate_jumpdest_table( let operands = 3; ensure!( evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); let [_value, _offset, _size, ..] = evm_stack[..] else { @@ -301,18 +301,18 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [counter, ..] = evm_stack[..] else { + let [jump_target, ..] = evm_stack[..] else { unreachable!() }; - ensure!( - *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow: counter: {} is larger than u64::MAX {}", - *counter, - u64::MAX - ); - let jump_target: u64 = counter.to(); - - prev_jump = Some(jump_target); + // ensure!( + // *counter <= U256::from(u64::MAX), + // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX + // {}", *counter, + // u64::MAX + // ); + // let jump_target: u64 = counter.to(); + + prev_jump = Some(*jump_target); } "JUMPI" => { ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); @@ -324,25 +324,27 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [counter, condition, ..] = evm_stack[..] else { + let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; - ensure!( - *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow." - ); - let jump_target: u64 = counter.to(); + // ensure!( + // *counter <= U256::from(u64::MAX), + // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX + // {}", *counter, + // u64::MAX + // ); + // let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { - Some(jump_target) + Some(*jump_target) } else { None }; } "JUMPDEST" => { let jumped_here = if let Some(jmp_target) = prev_jump { - jmp_target == entry.pc + jmp_target == U256::from(entry.pc) } else { false }; diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index fdfdf23f5..1305f9010 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,6 +1,5 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use crate::rpc::Compat; use __compat_primitive_types::{H256, U256}; use alloy::{ primitives::{keccak256, Address, B256}, @@ -26,6 +25,7 @@ use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::debug; use crate::rpc::jumpdest::{self, get_normalized_structlog}; +use crate::rpc::Compat; /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( From ec817013e670ae78b7fedc95dd28f27de5ca7e81 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 00:12:35 +0200 Subject: [PATCH 018/107] misc --- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 8 ++++---- zero/src/rpc/jumpdest.rs | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index dc59806b9..f525d8b4c 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' +#export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index baee31ea7..fe2eba2fa 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -13,8 +13,8 @@ mkdir -p witnesses export RAYON_NUM_THREADS=4 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=67108864 +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." @@ -138,9 +138,9 @@ USEDTOFAIL=" ROUND2=" 664 -665 667 670 +665 " NOWSUCCESS=" @@ -185,7 +185,7 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - prove_stdio.sh $WITNESS test_only + ./prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 533a9229b..d6dc4579d 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -26,7 +26,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::TxnTrace; -use tracing::{instrument, trace}; +use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. @@ -98,7 +98,6 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -// #[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], From bff471ed894b97df98171abdafbf2f33b8ee835a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 11:49:25 +0200 Subject: [PATCH 019/107] fix ci --- scripts/prove_stdio.sh | 11 +++++------ scripts/test_jerigon.sh | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index f525d8b4c..8c5e60fe9 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -30,17 +30,16 @@ TEST_OUT_PATH="${REPO_ROOT}/test.out" BLOCK_BATCH_SIZE=${BLOCK_BATCH_SIZE:=1} - # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs -#export TOKIO_WORKER_THREADS=$num_procs +export TOKIO_WORKER_THREADS=$num_procs -#export RUST_MIN_STACK=33554432 -#export RUST_BACKTRACE=full -#export RUST_LOG=trace +export RUST_MIN_STACK=33554432 +export RUST_BACKTRACE=full +export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -#export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index fe2eba2fa..5bb8e7293 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -2,7 +2,6 @@ set -uo pipefail -#export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= if [ -z $RPC ]; then # You must set an RPC endpoint From ca9620dc26c01de7a77540323317b30e36fa180f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 16:35:39 +0200 Subject: [PATCH 020/107] minimize diff --- .gitignore | 1 - .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 2 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- .../src/cpu/kernel/tests/mod.rs | 1 - .../src/generation/jumpdest.rs | 16 +------ evm_arithmetization/src/generation/mod.rs | 6 +-- .../src/generation/prover_input.rs | 16 +++---- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- scripts/prove_stdio.sh | 9 ++-- scripts/test_jerigon.sh | 11 +---- trace_decoder/src/core.rs | 18 ++++---- trace_decoder/src/interface.rs | 2 +- trace_decoder/src/jumpdest.rs | 1 - trace_decoder/src/lib.rs | 1 - zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jumpdest.rs | 43 +++++++------------ zero/src/rpc/native/mod.rs | 3 +- zero/src/rpc/native/txn.rs | 15 +++++-- 26 files changed, 71 insertions(+), 98 deletions(-) delete mode 100644 trace_decoder/src/jumpdest.rs diff --git a/.gitignore b/.gitignore index d262be31e..9f5ee7a64 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ # Misc ###### /**/*.ignoreme -**/output.log /**/*.ipynb /**/*.log /**/*.out diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 2b35b2a1c..106608935 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,7 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 85d43771f..9c2f7bfac 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -118,9 +118,9 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, + pub(crate) jumpdest_table: Option, pub(crate) accounts: BTreeMap, pub(crate) storage: BTreeMap<(U256, U256), usize>, - pub(crate) jumpdest_table: Option, pub(crate) next_txn_index: usize, } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index fbae6647d..4a6dc9447 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,7 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; @@ -372,7 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 673b516b0..716e238d0 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index f4fb947e6..2b983d099 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -28,7 +28,6 @@ mod transient_storage; use std::{ops::Range, str::FromStr}; -use ::core::iter::Iterator as _; use anyhow::Result; use ethereum_types::U256; use plonky2::hash::hash_types::RichField; diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index fc2a8e1a6..9a3201cce 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -51,18 +51,11 @@ impl JumpDestTableWitness { /// Creates the required `ctx` keys and `code_hash`. Idempotent. pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { (*self).entry(code_hash).or_default().insert(ctx, offset); - - // TODO(einar) remove before publishing PR. - assert!(self.0.contains_key(&code_hash)); - assert!(self.0[&code_hash].0.contains_key(&ctx)); - assert!(self.0[&code_hash].0[&ctx].contains(&offset)); } pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { let mut curr_max_ctx = prev_max_ctx; - // TODO: Opportunity for optimization: Simulate to generate only missing - // JUMPDEST tables. for (code_hash, ctx_tbl) in (*other).iter() { for (ctx, jumpdests) in ctx_tbl.0.iter() { let batch_ctx = prev_max_ctx + ctx; @@ -70,12 +63,7 @@ impl JumpDestTableWitness { for offset in jumpdests { self.insert(*code_hash, batch_ctx, *offset); - - assert!(self.0.contains_key(code_hash)); - assert!(self.0[code_hash].0.contains_key(&batch_ctx)); - assert!(self.0[code_hash].0[&batch_ctx].contains(offset)); } - // dbg!(&self); } } @@ -90,7 +78,7 @@ impl JumpDestTableWitness { impl Display for JumpDestTableWitness { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "=== JumpDestTableWitness ===")?; + writeln!(f, "\n=== JumpDestTableWitness ===")?; for (code, ctxtbls) in &self.0 { write!(f, "codehash: {:#x}\n{}", code, ctxtbls)?; @@ -115,7 +103,7 @@ impl Display for ContextJumpDests { impl Display for JumpDestTableProcessed { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "=== JumpDestTableProcessed ===")?; + writeln!(f, "\n=== JumpDestTableProcessed ===")?; let v = sorted(self.0.clone()); for (ctx, code) in v { diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index e5ec7bde3..0a775eed5 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -111,7 +111,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_tables: Option, + pub jumpdest_table: Option, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -165,7 +165,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_tables: Option, + pub jumpdest_table: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -240,7 +240,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), - jumpdest_tables: self.jumpdest_tables.clone(), + jumpdest_table: self.jumpdest_table.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index c561504df..e30ea3b6e 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -43,8 +43,7 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; -/// A set of contract code as a byte arrays. From this a mapping: hash -> -/// contract can be built. +/// A set to hold contract code as a byte vectors. pub type CodeDb = BTreeSet>; /// Prover input function represented as a scoped function name. @@ -768,11 +767,11 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_tables); + dbg!(&self.inputs.jumpdest_table); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code - let rpcw = &self.inputs.jumpdest_tables; + let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); @@ -809,13 +808,10 @@ impl GenerationState { self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); - trace!( - "ctx: {ctx}, code_hash: {:?} code: {:?}", - keccak(code.clone()), - code - ); + let code_hash = keccak(code.clone()); + trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); for offset in jumpdest_table.clone() { - jdtw.insert(keccak(code.clone()), ctx, offset); + jdtw.insert(code_hash, ctx, offset); } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 12a84fbe7..f3af1539e 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 475d03a8e..d251ed6fc 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,7 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 8d30f85d1..576eb1e57 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,7 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 951ebbb28..9de1e5d63 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,7 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 221113b26..f3e6996c1 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,7 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index de8e096cf..933f3fd2c 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,7 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 2aa776e54..6f4686ef6 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,7 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 8c5e60fe9..2b804f98a 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -21,6 +21,9 @@ fi REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +echo "Block batch size: $BLOCK_BATCH_SIZE" + BATCH_SIZE=${BATCH_SIZE:=1} echo "Batch size: $BATCH_SIZE" @@ -28,15 +31,13 @@ OUTPUT_LOG="${REPO_ROOT}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" TEST_OUT_PATH="${REPO_ROOT}/test.out" -BLOCK_BATCH_SIZE=${BLOCK_BATCH_SIZE:=1} - # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=trace +export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' @@ -103,7 +104,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then rm $TEST_OUT_PATH exit else - echo "Failed to create proof witnesses. See \"zk_evm/zero_bin/tools/test.out\" for more details." + echo "Failed to create proof witnesses. See \"zk_evm/test.out\" for more details." exit 1 fi fi diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 5bb8e7293..6d1125216 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -163,15 +163,10 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -# CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" -#BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" -#BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` -#BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -#BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI" -BLOCKS="$ROUND2" +BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` -#echo "Testing: $BLOCKS" +echo "Testing: $BLOCKS" printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt echo "---------------------------" | tee -a witnesses/jerigon_results.txt @@ -194,5 +189,3 @@ for BLOCK in $BLOCKS; do fi printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt done - -exit 0 diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 2b1a0e49c..cd410e58b 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -77,7 +77,6 @@ pub fn entrypoint( withdrawals, )?; - dbg!(&batches.first().unwrap().jumpdest_tables); let mut running_gas_used = 0; Ok(batches .into_iter() @@ -117,32 +116,35 @@ pub fn entrypoint( checkpoint_state_trie_root, checkpoint_consolidated_hash, contract_code: { - let cc = contract_code.into_iter(); - let initcodes = byte_code .iter() - .filter_map(|nonempty_txn_raw| -> Option> { + .filter_map(|nonempty_txn_bytes| -> Option> { let tx_envelope = - TxEnvelope::decode(&mut &nonempty_txn_raw[..]).unwrap(); + TxEnvelope::decode(&mut &nonempty_txn_bytes[..]).unwrap(); match tx_envelope.to() { TxKind::Create => Some(tx_envelope.input().to_vec()), TxKind::Call(_address) => None, } }); - cc.chain(initcodes) + contract_code + .into_iter() + .chain(initcodes) .map(|it| (keccak_hash::keccak(&it), it)) .collect() }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_tables: { + jumpdest_table: { + // Note that this causes any batch containing just a single `None` to collapse + // into a `None`, which causing failover to simulating jumpdest analysis for the + // whole batch. There is an optimization opportunity here. jumpdest_tables .into_iter() .collect::>>() - .map(|vj| JumpDestTableWitness::merge(vj.iter()).0) + .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0) }, }, ) diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 5c72d6b3f..81cc87bbe 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -170,7 +170,7 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } -// Question: Why has this has been removed upstream. Proably unused. +// Review: Re-adding this. It has been removed upstream. impl ContractCodeUsage { /// Get code hash from a read or write operation of contract code. pub fn get_code_hash(&self) -> H256 { diff --git a/trace_decoder/src/jumpdest.rs b/trace_decoder/src/jumpdest.rs deleted file mode 100644 index 8b1378917..000000000 --- a/trace_decoder/src/jumpdest.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index aaab848d4..8c54eea8e 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -60,7 +60,6 @@ use plonky2::field::goldilocks_field::GoldilocksField; mod type1; // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 // add backend/prod support for type 2 -mod jumpdest; #[cfg(test)] #[allow(dead_code)] mod type2; diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index d9f25901a..b9751196c 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -207,7 +207,7 @@ async fn main() -> anyhow::Result<()> { .with( tracing_subscriber::fmt::layer() // With the default configuration trace information is written - // to stdout, but we already use to write our payload (the witness). + // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) .with_ansi(false) .compact() diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d6dc4579d..a48d4759a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -65,11 +65,12 @@ where TransportT: Transport + Clone, { // Optimization: It may be a better default to pull the stack immediately. - let light_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) + let stackonly_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) .await?; - let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; + let structlogs_opt: Option> = + normalize_structlog(stackonly_structlog_trace).await; let need_memory = structlogs_opt.is_some_and(trace_contains_create2); trace!("Need structlog with memory: {need_memory}"); @@ -107,7 +108,7 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); - // This does not contain `initcodes`. + // This map does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces .iter() .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) @@ -192,22 +193,24 @@ pub(crate) fn generate_jumpdest_table( }; let callee_address = { - // Clear the upper half of the operand. let callee_raw = *address; - // let (callee_raw, _overflow) = callee_raw.overflowing_shl(128); - // let (callee_raw, _overflow) = callee_raw.overflowing_shr(128); - ensure!(callee_raw <= U256::from(U160::MAX)); let lower_20_bytes = U160::from(callee_raw); Address::from(lower_20_bytes) }; + if callee_addr_to_code_hash.contains_key(&callee_address) { + let next_code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((next_code_hash, next_ctx_available)); + }; + if precompiles().contains(&callee_address) { trace!("Called precompile at address {}.", &callee_address); - } else if callee_addr_to_code_hash.contains_key(&callee_address) { - let code_hash = callee_addr_to_code_hash[&callee_address]; - call_stack.push((code_hash, next_ctx_available)); - } else { + }; + + if callee_addr_to_code_hash.contains_key(&callee_address).not() + && precompiles().contains(&callee_address).not() + { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun trace!( @@ -303,13 +306,6 @@ pub(crate) fn generate_jumpdest_table( let [jump_target, ..] = evm_stack[..] else { unreachable!() }; - // ensure!( - // *counter <= U256::from(u64::MAX), - // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX - // {}", *counter, - // u64::MAX - // ); - // let jump_target: u64 = counter.to(); prev_jump = Some(*jump_target); } @@ -326,13 +322,6 @@ pub(crate) fn generate_jumpdest_table( let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; - // ensure!( - // *counter <= U256::from(u64::MAX), - // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX - // {}", *counter, - // u64::MAX - // ); - // let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { @@ -347,7 +336,7 @@ pub(crate) fn generate_jumpdest_table( } else { false }; - let jumpdest_offset = entry.pc as usize; + let jumpdest_offset = TryInto::::try_into(entry.pc)?; if jumped_here { jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 11766b148..5563cad5b 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -15,8 +15,7 @@ use crate::provider::CachedProvider; mod state; mod txn; -pub use txn::process_transaction; -pub use txn::process_transactions; +pub use txn::{process_transaction, process_transactions}; /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 1305f9010..34ed820df 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -22,7 +22,7 @@ use anyhow::{Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::debug; +use tracing::info; use crate::rpc::jumpdest::{self, get_normalized_structlog}; use crate::rpc::Compat; @@ -86,10 +86,19 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( |error| { - debug!("JumpDestTable generation failed with reason: {}", error); + info!( + "{:#?}: JumpDestTable generation failed with reason: {}", + tx.hash, error + ); None }, - Some, + |jdt| { + info!( + "{:#?}: JumpDestTable generation succeeded with result: {}", + tx.hash, jdt + ); + Some(jdt) + }, ) }); From 4c97c0f8537aca0b279933286b13100890ac8a57 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 17:14:19 +0200 Subject: [PATCH 021/107] include whole function in timeout --- zero/src/rpc/jumpdest.rs | 48 +++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index a48d4759a..f44aa7a43 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -51,7 +51,7 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } -fn trace_contains_create2(structlog: Vec) -> bool { +fn trace_contains_create2(structlog: &[StructLog]) -> bool { structlog.iter().any(|entry| entry.op == "CREATE2") } @@ -64,27 +64,39 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - // Optimization: It may be a better default to pull the stack immediately. - let stackonly_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) - .await?; + let inner = async { + // Optimization: It may be a better default to pull the stack immediately. + let stackonly_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) + .await?; - let structlogs_opt: Option> = - normalize_structlog(stackonly_structlog_trace).await; + let stackonly_structlog_opt: Option> = + normalize_structlog(&stackonly_structlog_trace).await; - let need_memory = structlogs_opt.is_some_and(trace_contains_create2); - trace!("Need structlog with memory: {need_memory}"); + let need_memory = stackonly_structlog_opt + .as_deref() + .is_some_and(trace_contains_create2); + trace!("Need structlog with memory: {need_memory}"); - let structlog = provider.debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ); + if need_memory.not() { + return Ok(stackonly_structlog_opt); + }; + + let memory_structlog_fut = provider.debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ); + + let memory_structlog = normalize_structlog(&memory_structlog_fut.await?).await; + + Ok::>, RpcError>(memory_structlog) + }; - match timeout(TIMEOUT_LIMIT, structlog).await { + match timeout(TIMEOUT_LIMIT, inner).await { Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( ellapsed_error, )))), - Ok(structlog_res) => Ok(normalize_structlog(structlog_res?).await), + Ok(structlog_res) => Ok(structlog_res?), } } @@ -470,11 +482,11 @@ pub mod structlogprime { } pub(crate) async fn normalize_structlog( - unnormalized_structlog: GethTrace, + unnormalized_structlog: &GethTrace, ) -> Option> { match unnormalized_structlog { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(&structlog_js_object) + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs.clone()), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) .ok() .map(|s| s.struct_logs), _ => None, From 8bce013d66407b8b097274328dbd0998493d9977 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 18:54:58 +0200 Subject: [PATCH 022/107] avoid ensure macro --- zero/src/rpc/jumpdest.rs | 181 +++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 82 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index f44aa7a43..d916ba4b5 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -176,7 +176,10 @@ pub(crate) fn generate_jumpdest_table( call_stack.pop(); } - ensure!(call_stack.is_empty().not(), "Call stack was empty."); + ensure!( + call_stack.is_empty().not(), + "Call stack was unexpectedly empty." + ); let (code_hash, ctx) = call_stack.last().unwrap(); trace!("TX: {:?}", tx.hash); @@ -190,26 +193,32 @@ pub(crate) fn generate_jumpdest_table( match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 2; // actually 6 or 7. - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + let operands_used = 2; // actually 6 or 7. + + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); + // maybe increment ctx here + continue; + } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 let [_gas, address, ..] = evm_stack[..] else { unreachable!() }; - let callee_address = { - let callee_raw = *address; - ensure!(callee_raw <= U256::from(U160::MAX)); - let lower_20_bytes = U160::from(callee_raw); - Address::from(lower_20_bytes) + if *address > U256::from(U160::MAX) { + trace!( + "{op}: Callee address {} was larger than possible {}.", + *address, + U256::from(U160::MAX) + ); + continue; }; + let lower_20_bytes = U160::from(*address); + let callee_address = Address::from(lower_20_bytes); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; @@ -231,69 +240,63 @@ pub(crate) fn generate_jumpdest_table( ); } next_ctx_available += 1; - prev_jump = None; } - "CREATE" => { + "CREATE" | "CREATE2" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 3; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); - let [_value, _offset, _size, ..] = evm_stack[..] else { - unreachable!() - }; + let operands_used = 3; - let contract_address = tx.from.create(tx.nonce); - ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - let code_hash = callee_addr_to_code_hash[&contract_address]; - call_stack.push((code_hash, next_ctx_available)); + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + }; - next_ctx_available += 1; - prev_jump = None; - } - "CREATE2" => { - ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); - // We reverse the stack, so the order matches our assembly code. - let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 4; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); - let [_value, offset, size, _salt, ..] = evm_stack[..] else { + let [_value, offset, size, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*offset <= U256::from(usize::MAX)); + if *offset > U256::from(usize::MAX) { + trace!( + "{op}: Offset {offset} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; let offset: usize = offset.to(); - ensure!(*size <= U256::from(usize::MAX)); + if *size > U256::from(usize::MAX) { + trace!( + "{op}: Size {size} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; let size: usize = size.to(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - // let salt: Word = salt.to_be_bytes(); - ensure!( - entry.memory.is_some() && offset + size <= memory_size, - "Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size - ); + if entry.memory.is_none() || offset + size > memory_size { + trace!("Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size); + continue; + } let memory_raw: &[String] = entry.memory.as_ref().unwrap(); let memory_parsed: Vec> = memory_raw .iter() - .map(|s| { - // let c = s.parse(); - let c = U256::from_str_radix(s, 16); - ensure!(c.is_ok(), "Parsing memory failed."); - let a: U256 = c.unwrap(); - let d: Word = a.to_be_bytes(); - Ok(d) + .map(|mem_line| { + let mem_line_parsed = U256::from_str_radix(mem_line, 16)?; + Ok(mem_line_parsed.to_be_bytes()) }) .collect(); let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); - let memory: Vec = mem_res?.concat(); + if mem_res.is_err() { + trace!( + "{op}: Parsing memory failed with error: {}", + mem_res.unwrap_err() + ); + continue; + } + let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; let init_code_hash = keccak(init_code); @@ -303,18 +306,17 @@ pub(crate) fn generate_jumpdest_table( call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; - prev_jump = None; } "JUMP" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 1; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + if evm_stack.len() < operands { + trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + } let [jump_target, ..] = evm_stack[..] else { unreachable!() }; @@ -322,42 +324,55 @@ pub(crate) fn generate_jumpdest_table( prev_jump = Some(*jump_target); } "JUMPI" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 2; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + if evm_stack.len() < operands { + trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len()); + continue; + }; + let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; let jump_condition = condition.is_zero().not(); - prev_jump = if jump_condition { - Some(*jump_target) - } else { - None - }; + if jump_condition { + prev_jump = Some(*jump_target) + } } "JUMPDEST" => { - let jumped_here = if let Some(jmp_target) = prev_jump { - jmp_target == U256::from(entry.pc) - } else { - false - }; - let jumpdest_offset = TryInto::::try_into(entry.pc)?; - if jumped_here { - jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); + let mut jumped_here = false; + + if let Some(jmp_target) = prev_jump { + jumped_here = jmp_target == U256::from(entry.pc); } - // else: we do not care about JUMPDESTs reached through fall-through. prev_jump = None; + + if jumped_here.not() { + trace!( + "{op}: JUMPDESTs at offset {} was reached through fall-through.", + entry.pc + ); + continue; + } + + let jumpdest_offset = TryInto::::try_into(entry.pc); + if jumpdest_offset.is_err() { + trace!( + "{op}: Could not cast offset {} to usize {}.", + entry.pc, + usize::MAX + ); + continue; + } + jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { - next_ctx_available += 1; prev_jump = None; + next_ctx_available += 1; } _ => { prev_jump = None; @@ -367,6 +382,8 @@ pub(crate) fn generate_jumpdest_table( Ok(jumpdest_table) } +/// This module exists as a workaround for parsing `StrucLog`. The `error` +/// field is a string in Alloy but an object in Erigon. pub mod structlogprime { use core::option::Option::None; use std::collections::BTreeMap; From b0ebc2c2fe0999940ed52f105a4df0c43d150232 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 19:25:33 +0200 Subject: [PATCH 023/107] fix CREATE --- zero/src/rpc/jumpdest.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d916ba4b5..8ef51bbfa 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -52,7 +52,9 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } fn trace_contains_create2(structlog: &[StructLog]) -> bool { - structlog.iter().any(|entry| entry.op == "CREATE2") + structlog + .iter() + .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") } // Gets the lightest possible structlog for transcation `tx_hash`. From 62c7053d36a16a67287a70fb370773cb66a091e2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 11:27:07 +0200 Subject: [PATCH 024/107] small adjustments --- Cargo.lock | 1 - Cargo.toml | 2 + evm_arithmetization/Cargo.toml | 1 - .../src/generation/prover_input.rs | 2 +- scripts/test_native.sh | 51 ++++++++++++------- zero/src/rpc/jumpdest.rs | 6 +-- 6 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f929a38c1..0ed81ea1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2068,7 +2068,6 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plonky2_util", - "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", diff --git a/Cargo.toml b/Cargo.toml index 386ecff5e..cac073af2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" +enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -89,6 +90,7 @@ serde = "1.0.203" serde-big-array = "0.5.1" serde_json = "1.0.118" serde_path_to_error = "0.1.16" +serde_with = "3.8.1" sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "1.0.61" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index e8e832c2d..ffdb8d2f5 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,7 +15,6 @@ homepage.workspace = true keywords.workspace = true [dependencies] -__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e30ea3b6e..b502b68af 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,7 +767,7 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_table); + // dbg!(&self.inputs.jumpdest_table); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code diff --git a/scripts/test_native.sh b/scripts/test_native.sh index d2a7aac5f..943a00ca3 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -10,22 +10,25 @@ fi mkdir -p witnesses # Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=4 -export TOKIO_WORKER_THREADS=4 +export RAYON_NUM_THREADS=1 +export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=info -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=33554432 +#export RUST_LOG=info +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=33554432 -MAINNETBLOCKS=" +CANCUNBLOCKS=" 20548415 20240058 19665756 20634472 19807080 20634403 +" + +PRECANCUN=" 19096840 19240700 " @@ -33,17 +36,18 @@ MAINNETBLOCKS=" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 -NUMRANDOMBLOCKS=10 +NUMRANDOMBLOCKS=100 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -#BLOCKS="$MAINNETBLOCKS $RANDOMBLOCKS" -BLOCKS="$MAINNETBLOCKS" +BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +#BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" echo "Downloading witnesses.." +echo "------------------------"| tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" @@ -56,20 +60,15 @@ for BLOCK in $BLOCKS; do then printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt break + else + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure | tee -a witnesses/native_results.txt fi - - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure| tee -a witnesses/native_results.txt done echo "Witness for block $BLOCK ($WITNESS) prepared." -done - -echo "Finished downloading witnesses." -echo "Testing prepared witnesses.." -for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" - prove_stdio.sh $WITNESS test_only + ./prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then @@ -80,4 +79,20 @@ for WITNESS in witnesses/*.native.$GITHASH.witness.json; do printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt done -echo "Finished testing witnesses." +#echo "Finished downloading witnesses." +#echo "Testing prepared witnesses.." +# +#for WITNESS in witnesses/*.native.$GITHASH.witness.json; do +# echo "Testing $WITNESS" +# ./prove_stdio.sh $WITNESS test_only +# EXITCODE=$? +# if [ $EXITCODE -eq 0 ] +# then +# RESULT="success" +# else +# RESULT="failure" +# fi +# printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt +#done +# +#echo "Finished testing witnesses." diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 8ef51bbfa..38894679a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -30,7 +30,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(10); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(2*60); /// Structure of Etheruem memory type Word = [u8; 32]; @@ -51,7 +51,7 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } -fn trace_contains_create2(structlog: &[StructLog]) -> bool { +fn trace_contains_create(structlog: &[StructLog]) -> bool { structlog .iter() .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") @@ -77,7 +77,7 @@ where let need_memory = stackonly_structlog_opt .as_deref() - .is_some_and(trace_contains_create2); + .is_some_and(trace_contains_create); trace!("Need structlog with memory: {need_memory}"); if need_memory.not() { From 74b86fd916f6ab41fbec9cbe4e69dc16448be3de Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 13:39:22 +0200 Subject: [PATCH 025/107] fmt --- zero/src/rpc/jumpdest.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 38894679a..d9ba8cd4c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -30,7 +30,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(2*60); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(2 * 60); /// Structure of Etheruem memory type Word = [u8; 32]; @@ -51,13 +51,15 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } +/// Predicate that determines whether a `StructLog` that includes memory is +/// required. fn trace_contains_create(structlog: &[StructLog]) -> bool { structlog .iter() .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") } -// Gets the lightest possible structlog for transcation `tx_hash`. +/// Gets the lightest possible structlog for transcation `tx_hash`. pub(crate) async fn get_normalized_structlog( provider: &ProviderT, tx_hash: &B256, From c8be8888fb190cdcfa5abcaff04f0bfbf575684b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 14:29:46 +0200 Subject: [PATCH 026/107] feedback --- scripts/test_native.sh | 2 +- zero/src/prover/cli.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 943a00ca3..56de59a49 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -uxo pipefail +set -uo pipefail export RPC= if [ -z $RPC ]; then diff --git a/zero/src/prover/cli.rs b/zero/src/prover/cli.rs index 0dfa383b6..a6cdaebf9 100644 --- a/zero/src/prover/cli.rs +++ b/zero/src/prover/cli.rs @@ -18,7 +18,7 @@ pub struct CliProverConfig { #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 19)] max_cpu_len_log: usize, /// Number of transactions in a batch to process at once. - #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 1)] + #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 10)] batch_size: usize, /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] From d00439fc64fa2c9eb88aa58228ea007bf4e03e53 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 12:20:12 +0200 Subject: [PATCH 027/107] feedback --- .gitignore | 1 - zero/src/rpc/jumpdest.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9f5ee7a64..0016ff3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,3 @@ ################# /proofs /**/*.zkproof -/witnesses diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d9ba8cd4c..558804bc7 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -386,7 +386,7 @@ pub(crate) fn generate_jumpdest_table( Ok(jumpdest_table) } -/// This module exists as a workaround for parsing `StrucLog`. The `error` +/// This module exists as a workaround for parsing `StructLog`. The `error` /// field is a string in Alloy but an object in Erigon. pub mod structlogprime { use core::option::Option::None; From 6876c07e9b1f07b7afb2e673288834fc093e6355 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 16:31:51 +0200 Subject: [PATCH 028/107] Add JumpdestSrc parameter --- trace_decoder/src/core.rs | 4 ++-- zero/src/bin/rpc.rs | 4 ++++ zero/src/rpc/mod.rs | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index cd410e58b..78c9d9b96 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -6,10 +6,9 @@ use std::{ mem, }; -use alloy::primitives::address; use alloy::{ consensus::{Transaction, TxEnvelope}, - primitives::TxKind, + primitives::{address, TxKind}, rlp::Decodable as _, }; use alloy_compat::Compat as _; @@ -138,6 +137,7 @@ pub fn entrypoint( block_hashes: b_hashes.clone(), burn_addr, jumpdest_table: { + // TODO See the issue Simulate to get jumpdests on a per-transaction basis #653. // Note that this causes any batch containing just a single `None` to collapse // into a `None`, which causing failover to simulating jumpdest analysis for the // whole batch. There is an optimization opportunity here. diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index b9751196c..115a7619e 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -15,6 +15,7 @@ use zero::block_interval::BlockIntervalStream; use zero::prover::BlockProverInput; use zero::provider::CachedProvider; use zero::rpc; +use zero::rpc::JumpdestSrc; use self::rpc::{retry::build_http_retry_provider, RpcType}; @@ -34,6 +35,9 @@ struct RpcToolConfig { /// The RPC Tracer Type. #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, + /// The source of jumpdest tables. + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] backoff: u64, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index e3c218eb6..da1ac474b 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -39,6 +39,14 @@ pub enum RpcType { Native, } +/// The Jumpdest source type. +#[derive(ValueEnum, Clone, Debug, Copy)] +pub enum JumpdestSrc { + Simulation, + Zero, + Jerigon, +} + /// Obtain the prover input for one block pub async fn block_prover_input( cached_provider: Arc>, From 60efef9383b45954c9a735e5a9ae669fa89d43e7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 17:36:09 +0200 Subject: [PATCH 029/107] Refactor --- zero/src/bin/leader.rs | 2 ++ zero/src/bin/leader/cli.rs | 5 ++++- zero/src/bin/leader/client.rs | 4 +++- zero/src/bin/rpc.rs | 4 ++++ zero/src/rpc/jerigon.rs | 36 ++++++++++++++++++++--------------- zero/src/rpc/mod.rs | 17 +++++++++++++++-- zero/src/rpc/native/mod.rs | 3 +++ 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 728f6bb0e..775439fde 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -72,6 +72,7 @@ async fn main() -> Result<()> { Command::Rpc { rpc_url, rpc_type, + jumpdest_src, block_interval, checkpoint_block_number, previous_proof, @@ -91,6 +92,7 @@ async fn main() -> Result<()> { backoff, max_retries, block_time, + jumpdest_src, }, block_interval, LeaderConfig { diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 3569efc41..78ad0fbdb 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -4,7 +4,7 @@ use alloy::transports::http::reqwest::Url; use clap::{Parser, Subcommand, ValueHint}; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; -use zero::rpc::RpcType; +use zero::rpc::{JumpdestSrc, RpcType}; /// zero-bin leader config #[derive(Parser)] @@ -43,6 +43,9 @@ pub(crate) enum Command { // The node RPC type (jerigon / native). #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, + /// The source of jumpdest tables. + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] block_interval: String, diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index 6825b6683..136fb0845 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -10,7 +10,7 @@ use tracing::info; use zero::block_interval::{BlockInterval, BlockIntervalStream}; use zero::pre_checks::check_previous_proof_and_checkpoint; use zero::prover::{self, BlockProverInput, ProverConfig}; -use zero::rpc; +use zero::rpc::{self, JumpdestSrc}; use zero::rpc::{retry::build_http_retry_provider, RpcType}; #[derive(Debug)] @@ -20,6 +20,7 @@ pub struct RpcParams { pub backoff: u64, pub max_retries: u32, pub block_time: u64, + pub jumpdest_src: JumpdestSrc, } #[derive(Debug)] @@ -87,6 +88,7 @@ pub(crate) async fn client_main( block_id, leader_config.checkpoint_block_number, rpc_params.rpc_type, + rpc_params.jumpdest_src, ) .await?; block_tx diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 115a7619e..3c5c3ed88 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -25,6 +25,7 @@ struct FetchParams { pub end_block: u64, pub checkpoint_block_number: Option, pub rpc_type: RpcType, + pub jumpdest_src: JumpdestSrc, } #[derive(Args, Clone, Debug)] @@ -105,6 +106,7 @@ where block_id, checkpoint_block_number, params.rpc_type, + params.jumpdest_src, ) .await?; @@ -133,6 +135,7 @@ impl Cli { end_block, checkpoint_block_number, rpc_type: self.config.rpc_type, + jumpdest_src: self.config.jumpdest_src, }; let block_prover_inputs = @@ -159,6 +162,7 @@ impl Cli { end_block: block_number, checkpoint_block_number: None, rpc_type: self.config.rpc_type, + jumpdest_src: self.config.jumpdest_src, }; let block_prover_inputs = diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index ab8cf8d37..15ad0edf1 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,6 +1,8 @@ +use core::iter::Iterator; use std::collections::BTreeMap; use std::ops::Deref as _; +use __compat_primitive_types::H160; use alloy::{ providers::Provider, rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, @@ -21,6 +23,7 @@ use tracing::info; use super::{ fetch_other_block_data, jumpdest::{self, get_normalized_structlog}, + JumpdestSrc, }; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; @@ -36,6 +39,7 @@ pub async fn block_prover_input( cached_provider: std::sync::Arc>, target_block_id: BlockId, checkpoint_block_number: u64, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, @@ -65,17 +69,18 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let tx_traces: Vec<&BTreeMap<__compat_primitive_types::H160, TxnTrace>> = tx_results - .iter() - .map(|TxnInfo { traces, meta: _ }| traces) - .collect::>(); - - let jdts: Vec> = process_transactions( - &block, - cached_provider.get_provider().await?.deref(), - &tx_traces, - ) - .await?; + let jdts: Vec> = match jumpdest_src { + JumpdestSrc::Simulation => vec![None; tx_results.len()], + JumpdestSrc::Zero => { + process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + tx_results.iter().map(|TxnInfo { traces, meta: _ }| traces), // &tx_traces, + ) + .await? + } + JumpdestSrc::Jerigon => todo!(), + }; // weave in the JDTs let txn_info = tx_results @@ -124,14 +129,15 @@ where } /// Processes the transactions in the given block and updates the code db. -pub async fn process_transactions( +pub async fn process_transactions<'i, I, ProviderT, TransportT>( block: &Block, provider: &ProviderT, - tx_traces: &[&BTreeMap<__compat_primitive_types::H160, TxnTrace>], + tx_traces: I, ) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, + I: Iterator>, { let futures = block .transactions @@ -139,7 +145,7 @@ where .context("No transactions in block")? .iter() .zip(tx_traces) - .map(|(tx, &tx_trace)| process_transaction(provider, tx, tx_trace)) + .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) .collect::>(); let vec_of_res = futures.collect::>().await; @@ -151,7 +157,7 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, - tx_trace: &BTreeMap<__compat_primitive_types::H160, TxnTrace>, + tx_trace: &BTreeMap, ) -> anyhow::Result> where ProviderT: Provider, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index da1ac474b..061cd9743 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -53,6 +53,7 @@ pub async fn block_prover_input( block_id: BlockId, checkpoint_block_number: u64, rpc_type: RpcType, + jumpdest_src: JumpdestSrc, ) -> Result where ProviderT: Provider, @@ -60,10 +61,22 @@ where { match rpc_type { RpcType::Jerigon => { - jerigon::block_prover_input(cached_provider, block_id, checkpoint_block_number).await + jerigon::block_prover_input( + cached_provider, + block_id, + checkpoint_block_number, + jumpdest_src, + ) + .await } RpcType::Native => { - native::block_prover_input(cached_provider, block_id, checkpoint_block_number).await + native::block_prover_input( + cached_provider, + block_id, + checkpoint_block_number, + jumpdest_src, + ) + .await } } } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 5563cad5b..6dcf07af3 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -17,11 +17,14 @@ mod txn; pub use txn::{process_transaction, process_transactions}; +use super::JumpdestSrc; + /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( provider: Arc>, block_number: BlockId, checkpoint_block_number: u64, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, From b07752dd60a45b626c0fc168661ba0cee76640e9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 18:12:43 +0200 Subject: [PATCH 030/107] Add jmp src to native --- zero/src/rpc/native/mod.rs | 11 ++++++++--- zero/src/rpc/native/txn.rs | 39 +++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 6dcf07af3..76dd302ae 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -31,7 +31,7 @@ where TransportT: Transport + Clone, { let (block_trace, other_data) = try_join!( - process_block_trace(provider.clone(), block_number), + process_block_trace(provider.clone(), block_number, jumpdest_src), crate::rpc::fetch_other_block_data(provider.clone(), block_number, checkpoint_block_number) )?; @@ -45,6 +45,7 @@ where pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, @@ -54,8 +55,12 @@ where .get_block(block_number, BlockTransactionsKind::Full) .await?; - let (code_db, txn_info) = - txn::process_transactions(&block, cached_provider.get_provider().await?.deref()).await?; + let (code_db, txn_info) = txn::process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + jumpdest_src, + ) + .await?; let trie_pre_images = state::process_state_witness(cached_provider, block, &txn_info).await?; Ok(BlockTrace { diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 34ed820df..582b7ee8f 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,3 +1,4 @@ +use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use __compat_primitive_types::{H256, U256}; @@ -24,13 +25,17 @@ use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::info; -use crate::rpc::jumpdest::{self, get_normalized_structlog}; use crate::rpc::Compat; +use crate::rpc::{ + jumpdest::{self, get_normalized_structlog}, + JumpdestSrc, +}; /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( block: &Block, provider: &ProviderT, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<(CodeDb, Vec)> where ProviderT: Provider, @@ -41,7 +46,7 @@ where .as_transactions() .context("No transactions in block")? .iter() - .map(|tx| process_transaction(provider, tx)) + .map(|tx| process_transaction(provider, tx, jumpdest_src)) .collect::>() .try_fold( (BTreeSet::new(), Vec::new()), @@ -59,13 +64,14 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<(CodeDb, TxnInfo)> where ProviderT: Provider, TransportT: Transport + Clone, { let (tx_receipt, pre_trace, diff_trace, structlog_opt) = - fetch_tx_data(provider, &tx.hash).await?; + fetch_tx_data(provider, &tx.hash, jumpdest_src).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); @@ -125,6 +131,7 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<( ::ReceiptResponse, GethTrace, @@ -138,14 +145,24 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); - - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( - tx_receipt_fut, - pre_trace_fut, - diff_trace_fut, - structlog_trace_fut, - )?; + + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { + JumpdestSrc::Zero => { + let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); + futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut, + )? + } + JumpdestSrc::Simulation => { + let (tx_receipt, pre_trace, diff_trace) = + futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + (tx_receipt, pre_trace, diff_trace, None) + } + JumpdestSrc::Jerigon => todo!(), + }; Ok(( tx_receipt.context("Transaction receipt not found.")?, From 66ea81151d7b12d4dfedb846eb3d2e0db560ef16 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 20:06:22 +0200 Subject: [PATCH 031/107] Feedback --- trace_decoder/src/core.rs | 9 +++++---- trace_decoder/src/lib.rs | 1 + zero/src/rpc/jumpdest.rs | 14 ++++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 78c9d9b96..4890a68b4 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -34,6 +34,11 @@ use crate::{ SeparateTriePreImages, TxnInfo, TxnMeta, TxnTrace, }; +/// Addresses of precompiled Ethereum contracts. +pub const PRECOMPILE_ADDRESSES: Range = + address!("0000000000000000000000000000000000000001") + ..address!("000000000000000000000000000000000000000a"); + /// TODO(0xaatif): document this after https://github.com/0xPolygonZero/zk_evm/issues/275 pub fn entrypoint( trace: BlockTrace, @@ -496,10 +501,6 @@ fn middle( state_mask.insert(TrieKey::from_address(addr)); } else { // Simple state access - const PRECOMPILE_ADDRESSES: Range = - address!("0000000000000000000000000000000000000001") - ..address!("000000000000000000000000000000000000000a"); - if receipt.status || !PRECOMPILE_ADDRESSES.contains(&addr.compat()) { // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/pull/613 // masking like this SHOULD be a space-saving optimization, diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index 8c54eea8e..587707883 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -67,6 +67,7 @@ mod typed_mpt; mod wire; pub use core::entrypoint; +pub use core::PRECOMPILE_ADDRESSES; mod core; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 558804bc7..fc42ebb11 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -26,6 +26,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::TxnTrace; +use trace_decoder::PRECOMPILE_ADDRESSES; use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over @@ -145,7 +146,9 @@ pub(crate) fn generate_jumpdest_table( ); let entrypoint_code_hash: H256 = match tx.to { - Some(to_address) if precompiles().contains(&to_address) => return Ok(jumpdest_table), + Some(to_address) if PRECOMPILE_ADDRESSES.contains(&to_address) => { + return Ok(jumpdest_table) + } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { return Ok(jumpdest_table) } @@ -201,11 +204,13 @@ pub(crate) fn generate_jumpdest_table( ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands_used = 2; // actually 6 or 7. + // These opcodes expect 6 or 7 operands on the stack, but for jumpdest-table + // generation we only use 2, and failures will be handled in + // next iteration by popping the stack accordingly. + let operands_used = 2; if evm_stack.len() < operands_used { trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); - // maybe increment ctx here continue; } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 @@ -304,9 +309,6 @@ pub(crate) fn generate_jumpdest_table( let init_code = &memory[offset..offset + size]; let init_code_hash = keccak(init_code); - // let contract_address = tx.from.create2_from_code(salt, init_code); - // ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - // let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; From f230b84423df8f268d4d56a98bc2e4042bf7649d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 20:16:24 +0200 Subject: [PATCH 032/107] fixup! Feedback --- zero/src/rpc/jumpdest.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index fc42ebb11..238ac7b7a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -3,9 +3,7 @@ use core::option::Option::None; use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; -use std::collections::HashSet; use std::ops::Not as _; -use std::sync::OnceLock; use __compat_primitive_types::H256; use alloy::primitives::Address; @@ -105,15 +103,6 @@ where } } -/// Provides a way to check in constant time if an address points to a -/// precompile. -fn precompiles() -> &'static HashSet
{ - static PRECOMPILES: OnceLock> = OnceLock::new(); - PRECOMPILES.get_or_init(|| { - HashSet::
::from_iter((1..=0xa).map(|x| Address::from(U160::from(x)))) - }) -} - /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. pub(crate) fn generate_jumpdest_table( @@ -234,12 +223,12 @@ pub(crate) fn generate_jumpdest_table( call_stack.push((next_code_hash, next_ctx_available)); }; - if precompiles().contains(&callee_address) { + if PRECOMPILE_ADDRESSES.contains(&callee_address) { trace!("Called precompile at address {}.", &callee_address); }; if callee_addr_to_code_hash.contains_key(&callee_address).not() - && precompiles().contains(&callee_address).not() + && PRECOMPILE_ADDRESSES.contains(&callee_address).not() { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun From 90722a30578b7e3aac2d5b107fae46ebb79e6f3c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 25 Sep 2024 23:29:58 +0200 Subject: [PATCH 033/107] feedback --- zero/src/rpc/jerigon.rs | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 15ad0edf1..5fcf7f584 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -79,36 +79,21 @@ where ) .await? } - JumpdestSrc::Jerigon => todo!(), + JumpdestSrc::Jerigon => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), }; + let mut code_db = CodeDb::default(); // weave in the JDTs let txn_info = tx_results .into_iter() .zip(jdts) - .map( - |( - TxnInfo { - traces, - meta: - TxnMeta { - byte_code, - new_receipt_trie_node_byte, - gas_used, - jumpdest_table: _, - }, - }, - jdt, - )| TxnInfo { - traces, - meta: TxnMeta { - byte_code, - new_receipt_trie_node_byte, - gas_used, - jumpdest_table: jdt, - }, - }, - ) + .map(|(mut tx_info, jdt)| { + tx_info.meta.jumpdest_table = jdt.map(|(j, c)| { + code_db.extend(c); + j + }); + tx_info + }) .collect(); let other_data = @@ -148,8 +133,11 @@ where .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) .collect::>(); - let vec_of_res = futures.collect::>().await; - vec_of_res.into_iter().collect::, _>>() + futures + .collect::>() + .await + .into_iter() + .collect::, _>>() } /// Processes the transaction with the given transaction hash and updates the From a0e0879d74624adb1c0bab36e62c2d78bc34da0e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 25 Sep 2024 23:33:07 +0200 Subject: [PATCH 034/107] fix missing code for CREATE --- .../src/cpu/kernel/interpreter.rs | 5 +- .../src/generation/prover_input.rs | 11 +-- scripts/test_jerigon.sh | 86 ++++++++++++++++++- trace_decoder/src/core.rs | 22 ++++- zero/src/rpc/jerigon.rs | 25 +++--- zero/src/rpc/jumpdest.rs | 11 ++- zero/src/rpc/native/txn.rs | 18 ++-- 7 files changed, 143 insertions(+), 35 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 9c2f7bfac..5ab8fc659 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -173,9 +173,10 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( &vec![] }; trace!( - "code: {:?}, code_addr: {:?} <============", + "code: {:?}, code_addr: {:?}, {:?} <============", &code, - &code_addr + &code_addr, + code_map.contains_key(code_addr), ); trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index b502b68af..b7d4a6788 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,23 +767,20 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - // dbg!(&self.inputs.jumpdest_table); - // w for witness - // let txn_idx = self.next_txn_index - 1; - // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code + dbg!(&self.inputs.jumpdest_table); + dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + info!("Generating JUMPDEST tables part2"); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) - && rw != &sw - { + if let (Some(rw), Some(sw)) = (rpcw, simw) { info!("SIMW {}", sw); info!("RPCW {}", rw); assert_eq!(rw, &sw); diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 6d1125216..5426b5b01 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -153,6 +153,89 @@ NOWSUCCESS=" 35 " +ROUND3=" +125 +127 +131 +132 +136 +141 +142 +143 +145 +149 +150 +151 +153 +154 +186 +187 +188 +190 +193 +195 +197 +199 +201 +214 +220 +221 +222 +223 +226 +228 +229 +230 +231 +232 +234 +242 +256 +257 +258 +262 +264 +267 +268 +282 +284 +285 +287 +292 +294 +295 +301 +303 +304 +321 +325 +333 +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +528 +529 +530 +531 +532 +533 +534 +566 +570 +664 +77 +548 +" + + # 470..663 from Robin for i in {470..663} do @@ -163,7 +246,8 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS" +#$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" +BLOCKS="$ROUND3" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 4890a68b4..c6e530a75 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -132,11 +132,24 @@ pub fn entrypoint( } }); - contract_code + let initmap: HashMap<_, _> = initcodes .into_iter() - .chain(initcodes) .map(|it| (keccak_hash::keccak(&it), it)) - .collect() + .collect(); + + let contractmap: HashMap<_, _> = contract_code + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(); + + log::trace!("Initmap {:?}", initmap); + log::trace!("Contractmap {:?}", contractmap); + //log::trace!("DECODER: {:#?}", res); + + let mut c = code.inner.clone(); + c.extend(contractmap); + c.extend(initmap); + c }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -643,9 +656,10 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. +#[derive(Clone)] struct Hash2Code { /// Key must always be [`hash`] of value. - inner: HashMap>, + pub inner: HashMap>, } impl Hash2Code { diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 5fcf7f584..0b79cfd70 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -10,14 +10,12 @@ use alloy::{ }; use alloy_primitives::Address; use anyhow::Context as _; -use evm_arithmetization::jumpdest::JumpDestTableWitness; +use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::FuturesOrdered; use futures::StreamExt as _; use serde::Deserialize; use serde_json::json; -use trace_decoder::{ - BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, -}; +use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnTrace}; use tracing::info; use super::{ @@ -69,7 +67,7 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let jdts: Vec> = match jumpdest_src { + let jdts: Vec> = match jumpdest_src { JumpdestSrc::Simulation => vec![None; tx_results.len()], JumpdestSrc::Zero => { process_transactions( @@ -107,7 +105,7 @@ where .context("invalid hex returned from call to eth_getWitness")?, }), txn_info, - code_db: Default::default(), + code_db, }, other_data, }) @@ -118,7 +116,7 @@ pub async fn process_transactions<'i, I, ProviderT, TransportT>( block: &Block, provider: &ProviderT, tx_traces: I, -) -> anyhow::Result>> +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, @@ -146,7 +144,7 @@ pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, tx_trace: &BTreeMap, -) -> anyhow::Result> +) -> anyhow::Result> where ProviderT: Provider, TransportT: Transport + Clone, @@ -161,7 +159,7 @@ where .ok() .flatten(); - let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { + let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { info!( @@ -170,15 +168,18 @@ where ); None }, - |jdt| { + |(jdt, code_db)| { info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); - Some(jdt) + Some((jdt, code_db)) }, ) }); - Ok(jumpdest_table) + // TODO + // let jumpdest_table = jc.map(|(j, c)| j); + + Ok(jc) } diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 238ac7b7a..ac7173cbf 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -20,6 +20,7 @@ use alloy_primitives::B256; use alloy_primitives::U256; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; +use evm_arithmetization::CodeDb; use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; @@ -109,10 +110,11 @@ pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], tx_traces: &BTreeMap, -) -> anyhow::Result { +) -> anyhow::Result<(JumpDestTableWitness, CodeDb)> { trace!("Generating JUMPDEST table for tx: {}", tx.hash); let mut jumpdest_table = JumpDestTableWitness::default(); + let mut code_db = CodeDb::default(); // This map does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces @@ -136,10 +138,10 @@ pub(crate) fn generate_jumpdest_table( let entrypoint_code_hash: H256 = match tx.to { Some(to_address) if PRECOMPILE_ADDRESSES.contains(&to_address) => { - return Ok(jumpdest_table) + return Ok((jumpdest_table, code_db)) } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { - return Ok(jumpdest_table) + return Ok((jumpdest_table, code_db)) } Some(to_address) => callee_addr_to_code_hash[&to_address], None => { @@ -297,6 +299,7 @@ pub(crate) fn generate_jumpdest_table( let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; + code_db.insert(init_code.to_vec()); let init_code_hash = keccak(init_code); call_stack.push((init_code_hash, next_ctx_available)); @@ -374,7 +377,7 @@ pub(crate) fn generate_jumpdest_table( } } } - Ok(jumpdest_table) + Ok((jumpdest_table, code_db)) } /// This module exists as a workaround for parsing `StructLog`. The `error` diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 582b7ee8f..c8dab028c 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -76,11 +76,14 @@ where let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { + let (mut code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), GethTrace::PreStateTracer(PreStateFrame::Diff(diff)), - ) => process_tx_traces(access_list, read, diff).await?, + ) => { + info!("{:?} {:?} {:?}", tx.hash, read, diff); + process_tx_traces(access_list, read, diff).await? + } _ => unreachable!(), }; @@ -89,7 +92,7 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( |error| { info!( @@ -98,16 +101,21 @@ where ); None }, - |jdt| { + |(jdt, code_db)| { info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); - Some(jdt) + Some((jdt, code_db)) }, ) }); + let jumpdest_table = jc.map(|(j, c)| { + code_db.extend(c); + j + }); + let tx_meta = TxnMeta { byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), From 6bff4e4eac04fec5b881bd75bf9d7921011c0353 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 26 Sep 2024 17:24:44 +0200 Subject: [PATCH 035/107] fix --- scripts/prove_stdio.sh | 4 +-- scripts/test_jerigon.sh | 19 ++++++----- scripts/test_native.sh | 66 ++++++++++++++------------------------- trace_decoder/src/core.rs | 25 +++++++++------ 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 2b804f98a..25db16e19 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -35,9 +35,9 @@ TEST_OUT_PATH="${REPO_ROOT}/test.out" export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs -export RUST_MIN_STACK=33554432 +#export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=info +#export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 5426b5b01..e30300787 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -9,9 +9,11 @@ if [ -z $RPC ]; then fi mkdir -p witnesses -export RAYON_NUM_THREADS=4 +# Must match the values in prove_stdio.sh or build is dirty. +export RAYON_NUM_THREADS=1 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full +#export RUST_LOG=info #export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' #export RUST_MIN_STACK=67108864 @@ -251,8 +253,8 @@ BLOCKS="$ROUND3" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" -printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt -echo "---------------------------" | tee -a witnesses/jerigon_results.txt +printf "githash block verdict duration\n" | tee -a witnesses/jerigon_results.txt +echo "------------------------------------" | tee -a witnesses/jerigon_results.txt for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` @@ -263,13 +265,16 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - ./prove_stdio.sh $WITNESS test_only + SECONDS=0 + timeout 10m ./prove_stdio.sh $WITNESS test_only EXITCODE=$? + DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + echo $DURATION if [ $EXITCODE -eq 0 ] then - RESULT="success" + VERDICT="success" else - RESULT="failure" + VERDICT="failure" fi - printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt + printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/jerigon_results.txt done diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 56de59a49..63da02115 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -33,6 +33,11 @@ PRECANCUN=" 19240700 " +#It's visible with block 20727641 +ROUND1=`echo {20727640..20727650}` + + + CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 @@ -42,57 +47,34 @@ RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +BLOCKS="$ROUND1" #BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" -echo "Downloading witnesses.." -echo "------------------------"| tee -a witnesses/native_results.txt +echo "Testing: $BLOCKS" +printf "githash block verdict duration\n" | tee -a witnesses/native_results.txt +echo "------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do + GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" - until [ -f $WITNESS -a -s $WITNESS ]; do - echo "Fetching block $BLOCK" - cargo run --release --verbose --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - EXITCODE=$? - - if [ $EXITCODE -eq 0 -a -f $WITNESS -a -s $WITNESS ] - then - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt - break - else - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure | tee -a witnesses/native_results.txt - fi - done - - echo "Witness for block $BLOCK ($WITNESS) prepared." - - echo "Testing $WITNESS" - ./prove_stdio.sh $WITNESS test_only + echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Testing blocks: $BLOCKS." + echo "Now testing block $BLOCK .." + export RUST_LOG=info + SECONDS=0 + timeout 30m ./prove_stdio.sh $WITNESS test_only EXITCODE=$? + DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + echo $DURATION if [ $EXITCODE -eq 0 ] then - RESULT="success" + VERDICT="success" else - RESULT="failure" + VERDICT="failure" fi - printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt + printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/native_results.txt done - -#echo "Finished downloading witnesses." -#echo "Testing prepared witnesses.." -# -#for WITNESS in witnesses/*.native.$GITHASH.witness.json; do -# echo "Testing $WITNESS" -# ./prove_stdio.sh $WITNESS test_only -# EXITCODE=$? -# if [ $EXITCODE -eq 0 ] -# then -# RESULT="success" -# else -# RESULT="failure" -# fi -# printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt -#done -# -#echo "Finished testing witnesses." diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index c6e530a75..6a32046d3 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -53,7 +53,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - code.extend(code_db); + code.extend(code_db.clone()); let OtherBlockData { b_data: @@ -132,24 +132,30 @@ pub fn entrypoint( } }); + // TODO convert to Hash2Code let initmap: HashMap<_, _> = initcodes .into_iter() .map(|it| (keccak_hash::keccak(&it), it)) .collect(); + log::trace!("Initmap {:?}", initmap); let contractmap: HashMap<_, _> = contract_code .into_iter() .map(|it| (keccak_hash::keccak(&it), it)) .collect(); - - log::trace!("Initmap {:?}", initmap); log::trace!("Contractmap {:?}", contractmap); - //log::trace!("DECODER: {:#?}", res); - let mut c = code.inner.clone(); - c.extend(contractmap); - c.extend(initmap); - c + let codemap: HashMap<_, _> = code_db + .clone() + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(); + log::trace!("Codemap {:?}", codemap); + + let mut res = codemap; + res.extend(contractmap); + res.extend(initmap); + res }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -656,10 +662,9 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. -#[derive(Clone)] struct Hash2Code { /// Key must always be [`hash`] of value. - pub inner: HashMap>, + inner: HashMap>, } impl Hash2Code { From c26f4752921db7c7f0e14255e1299490a52de4d7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 26 Sep 2024 20:19:56 +0200 Subject: [PATCH 036/107] fix arguments --- evm_arithmetization/src/generation/prover_input.rs | 2 +- scripts/test_jerigon.sh | 2 +- zero/src/bin/leader/cli.rs | 2 +- zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jerigon.rs | 7 ++++--- zero/src/rpc/mod.rs | 7 ++++--- zero/src/rpc/native/txn.rs | 6 +++--- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 43511394a..9015ff4a3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -812,7 +812,7 @@ impl GenerationState { if let (Some(rw), Some(sw)) = (rpcw, simw) { info!("SIMW {}", sw); info!("RPCW {}", rw); - assert_eq!(rw, &sw); + // assert_eq!(rw, &sw); } info!("SIM {:#?}", sim); diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index e30300787..685939540 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -261,7 +261,7 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 78ad0fbdb..d89a349ea 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,7 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index a55073342..fe29247ee 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,7 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 0b79cfd70..73280fc99 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -68,8 +68,8 @@ where .await?; let jdts: Vec> = match jumpdest_src { - JumpdestSrc::Simulation => vec![None; tx_results.len()], - JumpdestSrc::Zero => { + JumpdestSrc::ProverSimulation => vec![None; tx_results.len()], + JumpdestSrc::ClientFetchedStructlogs => { process_transactions( &block, cached_provider.get_provider().await?.deref(), @@ -77,7 +77,8 @@ where ) .await? } - JumpdestSrc::Jerigon => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), + JumpdestSrc::ServerFetchedStructlogs => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), + JumpdestSrc::Serverside => todo!(), }; let mut code_db = CodeDb::default(); diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index e55a1ac89..4f4908eb0 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -44,9 +44,10 @@ pub enum RpcType { /// The Jumpdest source type. #[derive(ValueEnum, Clone, Debug, Copy)] pub enum JumpdestSrc { - Simulation, - Zero, - Jerigon, + ProverSimulation, + ClientFetchedStructlogs, + ServerFetchedStructlogs, // later + Serverside, // later } /// Obtain the prover input for one block diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index c8dab028c..96d30f554 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -155,7 +155,7 @@ where let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { - JumpdestSrc::Zero => { + JumpdestSrc::ClientFetchedStructlogs => { let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); futures::try_join!( tx_receipt_fut, @@ -164,12 +164,12 @@ where structlog_trace_fut, )? } - JumpdestSrc::Simulation => { + JumpdestSrc::ProverSimulation => { let (tx_receipt, pre_trace, diff_trace) = futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; (tx_receipt, pre_trace, diff_trace, None) } - JumpdestSrc::Jerigon => todo!(), + _ => todo!(), }; Ok(( From f3674090ed2e1d482c027580b96bbaf1dd7c5dd5 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 17:39:59 +0200 Subject: [PATCH 037/107] feedback --- trace_decoder/src/interface.rs | 10 ---------- zero/src/rpc/jumpdest.rs | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index e57ac7c81..1005b2afa 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -166,16 +166,6 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } -// Review: Re-adding this. It has been removed upstream. -impl ContractCodeUsage { - /// Get code hash from a read or write operation of contract code. - pub fn get_code_hash(&self) -> H256 { - match self { - ContractCodeUsage::Read(hash) => *hash, - ContractCodeUsage::Write(bytes) => keccak(bytes), - } - } -} /// Other data that is needed for proof gen. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index b9bf3c148..e4ef4c788 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -27,6 +27,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::is_precompile; +use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::trace; @@ -53,6 +54,14 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } +/// Get code hash from a read or write operation of contract code. +fn get_code_hash(usage: &ContractCodeUsage) -> H256 { + match usage { + ContractCodeUsage::Read(hash) => *hash, + ContractCodeUsage::Write(bytes) => keccak(bytes), + } +} + /// Predicate that determines whether a `StructLog` that includes memory is /// required. fn trace_contains_create(structlog: &[StructLog]) -> bool { @@ -124,7 +133,7 @@ pub(crate) fn generate_jumpdest_table( .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) .filter(|(_callee_addr, code_usage)| code_usage.is_some()) .map(|(callee_addr, code_usage)| { - (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + (*callee_addr, get_code_hash(code_usage.as_ref().unwrap())) }) .collect(); @@ -204,6 +213,10 @@ pub(crate) fn generate_jumpdest_table( if evm_stack.len() < operands_used { trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); + // Note for future debugging: There may exist edge cases, where the call + // context has been incremented before the call op fails. This should be + // accounted for before this and the following `continue`. The details are + // defined in `sys_calls.asm`. continue; } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 @@ -217,6 +230,7 @@ pub(crate) fn generate_jumpdest_table( *address, U256::from(U160::MAX) ); + // Se note above. continue; }; let lower_20_bytes = U160::from(*address); @@ -368,6 +382,7 @@ pub(crate) fn generate_jumpdest_table( ); continue; } + assert!(jumpdest_offset.unwrap() < 24576); jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { From 464dbc0d369ab448860a49c55a64676a492e1dcd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 19:18:00 +0200 Subject: [PATCH 038/107] fix --- scripts/prove_stdio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 25db16e19..e94b5d5a4 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -37,7 +37,7 @@ export TOKIO_WORKER_THREADS=$num_procs #export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -#export RUST_LOG=trace +export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' From 2783ccdd1dcc0124cd09c89b538c7e4731db0b90 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 23:57:10 +0200 Subject: [PATCH 039/107] debugging 460 --- .../src/generation/prover_input.rs | 24 +++++---- scripts/prove_stdio.sh | 17 +++--- scripts/test_jerigon.sh | 53 +++++++++++++++++-- scripts/test_native.sh | 4 +- trace_decoder/src/interface.rs | 2 +- zero/src/rpc/jumpdest.rs | 2 +- 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 9015ff4a3..5ae94e1b2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -796,27 +796,29 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_table); - dbg!(&self.inputs.txn_hashes); + // dbg!(&self.inputs.jumpdest_table); + // dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables part2"); + info!("Generating JUMPDEST tables: Running SIM"); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - let (sim, simw): (Option, Option) = + let (sim, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) { - info!("SIMW {}", sw); - info!("RPCW {}", rw); - // assert_eq!(rw, &sw); - } + // if let (Some(rw), Some(sw)) = (rpcw, simw) { + // info!("SIMW {}", sw); + // info!("RPCW {}", rw); + // // assert_eq!(rw, &sw); + // } - info!("SIM {:#?}", sim); - info!("RPC {:#?}", rpc); + // info!("SIMW {:#?}", &simw); + // info!("RPCW {:#?}", rpcw); + assert_eq!(rpcw, simw); + assert_eq!(rpc, sim); self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index e94b5d5a4..077ce9261 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -21,15 +21,15 @@ fi REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-1}" echo "Block batch size: $BLOCK_BATCH_SIZE" -BATCH_SIZE=${BATCH_SIZE:=1} +BATCH_SIZE=${BATCH_SIZE:-1} echo "Batch size: $BATCH_SIZE" OUTPUT_LOG="${REPO_ROOT}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" -TEST_OUT_PATH="${REPO_ROOT}/test.out" +TEST_OUT_PATH="${REPO_ROOT}/$3.test.out" # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs @@ -98,14 +98,17 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH + cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" - rm $TEST_OUT_PATH + #rm $TEST_OUT_PATH exit - else - echo "Failed to create proof witnesses. See \"zk_evm/test.out\" for more details." + elif grep -q 'Proving task finished with error' $TEST_OUT_PATH; then + echo -e "\n\nFailed to create proof witnesses. See \"zk_evm/test.out\" for more details." exit 1 + else + echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See \"zk_evm/test.out\" for more details." + exit 2 fi fi diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 685939540..d1a6d272f 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -237,6 +237,51 @@ ROUND3=" 548 " +ROUND4=" +136 +186 +268 +282 +301 +304 +321 +333 +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +528 +529 +530 +531 +532 +533 +534 +570 +664 +" + +ROUND5=" +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +664 +" # 470..663 from Robin for i in {470..663} @@ -249,11 +294,11 @@ NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND3" +BLOCKS="$ROUND5" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" -printf "githash block verdict duration\n" | tee -a witnesses/jerigon_results.txt +printf "\ngithash block verdict duration\n" | tee -a witnesses/jerigon_results.txt echo "------------------------------------" | tee -a witnesses/jerigon_results.txt for BLOCK in $BLOCKS; do @@ -261,12 +306,12 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace + SECONDS=0 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - SECONDS=0 - timeout 10m ./prove_stdio.sh $WITNESS test_only + timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 63da02115..5e8bfeff8 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -10,8 +10,8 @@ fi mkdir -p witnesses # Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=1 -export TOKIO_WORKER_THREADS=1 +#export RAYON_NUM_THREADS=1 +#export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full #export RUST_LOG=info #export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 1005b2afa..35248f9a3 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -8,7 +8,7 @@ use ethereum_types::{Address, U256}; use evm_arithmetization::jumpdest::JumpDestTableWitness; use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; use evm_arithmetization::ConsolidatedHash; -use keccak_hash::{keccak, H256}; +use keccak_hash::H256; use mpt_trie::partial_trie::HashedPartialTrie; use serde::{Deserialize, Serialize}; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index e4ef4c788..8e1c7670f 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -33,7 +33,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(2 * 60); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(10 * 60); /// Structure of Etheruem memory type Word = [u8; 32]; From abee81258d89d14551a674d107e9a8ca257c2f07 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 13:42:08 +0200 Subject: [PATCH 040/107] debugging 460 --- .../src/cpu/kernel/interpreter.rs | 6 ++- .../src/generation/prover_input.rs | 39 ++++++++++++------- trace_decoder/src/core.rs | 4 +- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 054c89c27..985ed2dba 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -98,7 +98,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - let jdtw = interpreter + let (a, jdtw) = interpreter .generation_state .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); @@ -106,7 +106,9 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) + // (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) + interpreter.generation_state.jumpdest_table = Some(a.clone()); + Some((a, jdtw)) } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 5ae94e1b2..83213783c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -359,9 +359,12 @@ impl GenerationState { } /// Returns the next used jump address. + /// todo fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // get_code from self.memory + if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } @@ -798,11 +801,14 @@ impl GenerationState { info!("Generating JUMPDEST tables"); // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); - let rpcw = &self.inputs.jumpdest_table; - let rpc: Option = rpcw - .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables: Running SIM"); + // let rpcw = &self.inputs.jumpdest_table.clone(); + // let rpc: Option = rpcw + // .as_ref() + // .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, + // &self.inputs.contract_code)); info!("Generating JUMPDEST tables: + // Running SIM"); + + self.inputs.jumpdest_table = None; let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); @@ -815,12 +821,15 @@ impl GenerationState { // // assert_eq!(rw, &sw); // } - // info!("SIMW {:#?}", &simw); + info!("SIMW {:#?}", &simw); // info!("RPCW {:#?}", rpcw); - assert_eq!(rpcw, simw); - assert_eq!(rpc, sim); + info!("SIMP {:#?}", &sim); + // info!("RPCP {:#?}", &rpc); + // assert_eq!(rpcw, simw); + // assert_eq!(rpc, sim); - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + // self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = sim; Ok(()) } @@ -829,12 +838,12 @@ impl GenerationState { /// compute their respective proofs, by calling /// `get_proofs_and_jumpdests` pub(crate) fn set_jumpdest_analysis_inputs( - &mut self, + &self, jumpdest_table: HashMap>, - ) -> JumpDestTableWitness { + ) -> (JumpDestTableProcessed, JumpDestTableWitness) { let mut jdtw = JumpDestTableWitness::default(); - self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( - jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { + let jdtp = JumpDestTableProcessed::new(HashMap::from_iter(jumpdest_table.into_iter().map( + |(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); let code_hash = keccak(code.clone()); trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); @@ -847,9 +856,9 @@ impl GenerationState { } else { (ctx, vec![]) } - }), + }, ))); - jdtw + (jdtp, jdtw) } pub(crate) fn get_current_code(&self) -> Result, ProgramError> { diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 0baac3f47..ded46a83b 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -65,7 +65,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - code.extend(code_db.clone()); + // code.extend(code_db.clone()); let OtherBlockData { b_data: @@ -837,7 +837,7 @@ impl Hash2Code { pub fn get(&mut self, hash: H256) -> anyhow::Result> { match self.inner.get(&hash) { Some(code) => Ok(code.clone()), - None => bail!("no code for hash {}", hash), + None => bail!("no code for hash {:#x}", hash), } } pub fn insert(&mut self, code: Vec) { From 8bfccddcac3405888318d52aae5c3654621a2509 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 13:44:51 +0200 Subject: [PATCH 041/107] dbg --- scripts/test_jerigon.sh | 2 +- scripts/test_native.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index d1a6d272f..ed7076e30 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -2,7 +2,7 @@ set -uo pipefail -export RPC= +RPC=${RPC_JERIGON} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 5e8bfeff8..7f092f232 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -2,7 +2,7 @@ set -uo pipefail -export RPC= +RPC=${RPC_NATIVE} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 From f9c2f76f5bb2345c12e87d575bff4e27925368ee Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 14:57:32 +0200 Subject: [PATCH 042/107] bugfix --- trace_decoder/src/core.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index ded46a83b..fca50ac73 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -447,7 +447,7 @@ fn middle( storage_read, storage_written, code_usage, - self_destructed: _, + self_destructed, }, ) in traces .into_iter() @@ -544,6 +544,11 @@ fn middle( state_mask.insert(TrieKey::from_address(addr)); } } + + if self_destructed { + storage_tries.remove(&keccak_hash::keccak(addr)); + state_mask.extend(state_trie.reporting_remove(addr)?) + } } jumpdest_tables.push(jumpdest_table); From 313d78d3ed9febbc5a847250b7ee47150d062650 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 15:12:21 +0200 Subject: [PATCH 043/107] dbg --- .../src/generation/prover_input.rs | 23 +++++++++---------- scripts/test_jerigon.sh | 4 ++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 83213783c..fb8f69b7d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -801,12 +801,11 @@ impl GenerationState { info!("Generating JUMPDEST tables"); // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); - // let rpcw = &self.inputs.jumpdest_table.clone(); - // let rpc: Option = rpcw - // .as_ref() - // .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, - // &self.inputs.contract_code)); info!("Generating JUMPDEST tables: - // Running SIM"); + let rpcw = &self.inputs.jumpdest_table.clone(); + let rpc: Option = rpcw + .as_ref() + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; @@ -822,14 +821,14 @@ impl GenerationState { // } info!("SIMW {:#?}", &simw); - // info!("RPCW {:#?}", rpcw); + info!("RPCW {:#?}", rpcw); info!("SIMP {:#?}", &sim); - // info!("RPCP {:#?}", &rpc); - // assert_eq!(rpcw, simw); - // assert_eq!(rpc, sim); + info!("RPCP {:#?}", &rpc); + assert_eq!(rpcw, simw); + assert_eq!(rpc, sim); - // self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; - self.jumpdest_table = sim; + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + // self.jumpdest_table = sim; Ok(()) } diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index ed7076e30..781ac92ce 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -283,6 +283,10 @@ ROUND5=" 664 " +ROUND6=" +664 +" + # 470..663 from Robin for i in {470..663} do From 09079e61ba4303b0593c60fbb97763774dc57e84 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 16:53:46 +0200 Subject: [PATCH 044/107] fix --- trace_decoder/src/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index fca50ac73..4b3303ddd 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -65,7 +65,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - // code.extend(code_db.clone()); + code.extend(code_db.clone()); let OtherBlockData { b_data: From 7c84a63a6d4318fa360c7c04d4b024c4d7bf83b3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 17:58:11 +0200 Subject: [PATCH 045/107] batching working --- .../src/generation/prover_input.rs | 17 +++++++++-------- scripts/test_jerigon.sh | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index fb8f69b7d..425df5397 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -802,18 +802,19 @@ impl GenerationState { // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table.clone(); - let rpc: Option = rpcw + let rpcp: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; - let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - let (sim, ref simw): (Option, Option) = + let (simp, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + info!("Generating JUMPDEST tables: finished"); + // if let (Some(rw), Some(sw)) = (rpcw, simw) { // info!("SIMW {}", sw); // info!("RPCW {}", rw); @@ -822,12 +823,12 @@ impl GenerationState { info!("SIMW {:#?}", &simw); info!("RPCW {:#?}", rpcw); - info!("SIMP {:#?}", &sim); - info!("RPCP {:#?}", &rpc); - assert_eq!(rpcw, simw); - assert_eq!(rpc, sim); + info!("SIMP {:#?}", &simp); + info!("RPCP {:#?}", &rpcp); + // assert_eq!(simw, rpcw); + // assert_eq!(simp, rpcp); - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; // self.jumpdest_table = sim; Ok(()) diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 781ac92ce..8e02977c8 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -298,7 +298,7 @@ NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND5" +BLOCKS="$ROUND6" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" From 4202ece130bcb221adb1152aed29d9fc599d65f4 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 22:04:26 +0200 Subject: [PATCH 046/107] cleanups --- .../src/generation/prover_input.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 425df5397..f3b5ec54a 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -799,9 +799,7 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - // dbg!(&self.inputs.jumpdest_table); - // dbg!(&self.inputs.txn_hashes); - let rpcw = &self.inputs.jumpdest_table.clone(); + let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); @@ -815,21 +813,12 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - // if let (Some(rw), Some(sw)) = (rpcw, simw) { - // info!("SIMW {}", sw); - // info!("RPCW {}", rw); - // // assert_eq!(rw, &sw); - // } - info!("SIMW {:#?}", &simw); - info!("RPCW {:#?}", rpcw); + info!("RPCW {:#?}", &rpcw); info!("SIMP {:#?}", &simp); info!("RPCP {:#?}", &rpcp); - // assert_eq!(simw, rpcw); - // assert_eq!(simp, rpcp); self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - // self.jumpdest_table = sim; Ok(()) } From e1124d3c631aae1123370c3a7c2878b079bfbc5c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 14:32:13 +0200 Subject: [PATCH 047/107] feedback docs --- .../src/generation/jumpdest.rs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 9a3201cce..1966c265d 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -1,3 +1,27 @@ +//! EVM opcode 0x5B or 91 is [`JUMPDEST`] which encodes a a valid offset, that +//! opcodes `JUMP` and `JUMPI` can jump to. Jumps to non-[`JUMPDEST`] +//! instructions are invalid. During an execution a [`JUMPDEST`] may be visited +//! zero or more times. Offsets are measured in bytes with respect to the +//! beginning of some contract code, which is uniquely identified by its +//! `CodeHash`. Every time control flow is switches to another contract through +//! a `CALL`-like instruction a new call context, `Context`, is created. Thus, +//! the tripple (`CodeHash`,`Context`, `Offset`) uniquely identifies an visited +//! [`JUMPDEST`] offset of an execution. +//! +//! Since an operation like e.g. `PUSH 0x5B` does not encode a valid +//! [`JUMPDEST`] in its second byte, and `PUSH32 +//! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not +//! encode valid [`JUMPDESTS`] in bytes 1-32, some deligence must be exercised +//! when proving validity of jump operations. +//! +//! This module concerns itself with data structures for collecting these +//! offsets for [`JUMPDEST`] that was visited during an execution and are not +//! recording duplicity. The proofs that each of these offsets are not rendered +//! invalid by any of the previous 32 bytes `PUSH1`-`PUSH32` is computed later +//! in [`prove_context_jumpdests`] on basis of these collections. +//! +//! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b + use std::cmp::max; use std::{ collections::{BTreeSet, HashMap}, @@ -9,12 +33,13 @@ use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; -/// Each `CodeAddress` can be called one or more times, each time creating a new -/// `Context`. Each `Context` will contain one or more offsets of `JUMPDEST`. +/// Each `CodeAddress` can be called one or more times, +/// each time creating a new `Context`. +/// Each `Context` will contain one or more offsets of `JUMPDEST`. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub struct ContextJumpDests(pub HashMap>); -/// The result after proving a `JumpDestTableWitness`. +/// The result after proving a [`JumpDestTableWitness`]. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed(HashMap>); From 1f8d476c869ff0567d5bb1d6ce52e7c8fb63be48 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 15:20:27 +0200 Subject: [PATCH 048/107] feedback --- evm_arithmetization/src/generation/jumpdest.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 1966c265d..a7a104a15 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -11,7 +11,7 @@ //! Since an operation like e.g. `PUSH 0x5B` does not encode a valid //! [`JUMPDEST`] in its second byte, and `PUSH32 //! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not -//! encode valid [`JUMPDESTS`] in bytes 1-32, some deligence must be exercised +//! encode valid [`JUMPDESTS`] in bytes 1-32, some diligence must be exercised //! when proving validity of jump operations. //! //! This module concerns itself with data structures for collecting these @@ -64,10 +64,6 @@ impl JumpDestTableProcessed { } impl JumpDestTableWitness { - pub fn new(ctx_map: HashMap) -> Self { - Self(ctx_map) - } - pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { self.0.get(code_hash) } From 7319a15a3eb99493822d7a72fae09d1d6e1acce1 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 16:06:59 +0200 Subject: [PATCH 049/107] feedback filtermap --- evm_arithmetization/src/generation/jumpdest.rs | 4 ++-- zero/src/rpc/jumpdest.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index a7a104a15..f980d51ac 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -33,7 +33,7 @@ use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; -/// Each `CodeAddress` can be called one or more times, +/// Each `CodeHash` can be called one or more times, /// each time creating a new `Context`. /// Each `Context` will contain one or more offsets of `JUMPDEST`. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] @@ -43,7 +43,7 @@ pub struct ContextJumpDests(pub HashMap>); #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed(HashMap>); -/// Map `CodeAddress -> (Context -> [JumpDests])` +/// Map `CodeHash -> (Context -> [JumpDests])` #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub struct JumpDestTableWitness(HashMap); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 8e1c7670f..e4b8e5e29 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -127,16 +127,19 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); let mut code_db = CodeDb::default(); - // This map does not contain `initcodes`. + // This map does neither contain the `init` field of Contract Deployment + // transactions nor CREATE, CREATE2 payloads. let callee_addr_to_code_hash: HashMap = tx_traces .iter() - .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) - .filter(|(_callee_addr, code_usage)| code_usage.is_some()) - .map(|(callee_addr, code_usage)| { - (*callee_addr, get_code_hash(code_usage.as_ref().unwrap())) + .filter_map(|(callee_addr, trace)| { + trace + .code_usage + .as_ref() + .map(|code| (*callee_addr, get_code_hash(&code))) }) .collect(); + // REVIEW: will be removed before merge trace!( "Transaction: {} is a {}.", tx.hash, @@ -166,9 +169,6 @@ pub(crate) fn generate_jumpdest_table( // true condition and target `jump_target`. let mut prev_jump: Option = None; - // Call depth of the previous `entry`. We initialize to 0 as this compares - // smaller to 1. - //let mut prev_depth = 0; // The next available context. Starts at 1. Never decrements. let mut next_ctx_available = 1; // Immediately use context 1; From d4838e0c83191afbe34cfd050b1efd1f3fdc49f3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 16:43:28 +0200 Subject: [PATCH 050/107] review --- zero/src/rpc/jumpdest.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index e4b8e5e29..3997e545c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -191,6 +191,7 @@ pub(crate) fn generate_jumpdest_table( ); let (code_hash, ctx) = call_stack.last().unwrap(); + // REVIEW: will be removed before merge trace!("TX: {:?}", tx.hash); trace!("STEP: {:?}", step); trace!("STEPS: {:?}", struct_log.len()); From 27b5719457015015df0a1bf0be7cc51cde2d681f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 3 Oct 2024 12:26:57 +0200 Subject: [PATCH 051/107] fmt --- zero/src/rpc/jumpdest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 3997e545c..14b133705 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -135,7 +135,7 @@ pub(crate) fn generate_jumpdest_table( trace .code_usage .as_ref() - .map(|code| (*callee_addr, get_code_hash(&code))) + .map(|code| (*callee_addr, get_code_hash(code))) }) .collect(); From eaf3ed7175f3de8c196b709691b3eabc38654e89 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 3 Oct 2024 13:35:14 +0200 Subject: [PATCH 052/107] fix set_jumpdest_analysis_inputs_rpc --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 4 ++-- .../src/cpu/kernel/tests/core/jumpdest_analysis.rs | 3 ++- evm_arithmetization/src/generation/prover_input.rs | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 985ed2dba..9ff445fa2 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -100,7 +100,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let (a, jdtw) = interpreter .generation_state - .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); + .get_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", @@ -163,7 +163,7 @@ pub(crate) fn set_registers_and_run( /// /// - `jumpdest_table_rpc`: The raw table received from RPC. /// - `code_db`: The corresponding database of contract code used in the trace. -pub(crate) fn set_jumpdest_analysis_inputs_rpc( +pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, ) -> JumpDestTableProcessed { diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index 607f8cc69..d1b076118 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -17,7 +17,8 @@ use crate::witness::operation::CONTEXT_SCALING_FACTOR; impl Interpreter { pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { - let _ = self.generation_state.set_jumpdest_analysis_inputs(jumps); + let (jdtp, _jdtw) = self.generation_state.get_jumpdest_analysis_inputs(jumps); + self.generation_state.jumpdest_table = Some(jdtp); } pub(crate) fn get_jumpdest_bit(&self, offset: usize) -> U256 { diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index f3b5ec54a..e4206cb15 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -26,7 +26,7 @@ use crate::cpu::kernel::constants::cancun_constants::{ }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; use crate::cpu::kernel::interpreter::{ - set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, + get_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, }; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; @@ -798,11 +798,13 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call + // REVIEW: This will be rewritten to only run simulation when + // `self.inputs.jumpdest_table` is `None`. info!("Generating JUMPDEST tables"); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; @@ -826,7 +828,7 @@ impl GenerationState { /// Given a HashMap containing the contexts and the jumpdest addresses, /// compute their respective proofs, by calling /// `get_proofs_and_jumpdests` - pub(crate) fn set_jumpdest_analysis_inputs( + pub(crate) fn get_jumpdest_analysis_inputs( &self, jumpdest_table: HashMap>, ) -> (JumpDestTableProcessed, JumpDestTableWitness) { From 10b6a22cd4b9826ad7f8c77e02bc14872c15f112 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:03:41 +0200 Subject: [PATCH 053/107] discuss: deser in #427 (#681) * refactor: StructPogPrime * keep module doc, avoid name `vec` --- zero/src/rpc/jumpdest.rs | 192 ++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 116 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 14b133705..3f5057b27 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -13,6 +13,7 @@ use alloy::primitives::U160; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; +use alloy::rpc::types::trace::geth::GethTrace; use alloy::rpc::types::trace::geth::StructLog; use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; use alloy::transports::RpcError; @@ -24,7 +25,6 @@ use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use evm_arithmetization::CodeDb; use keccak_hash::keccak; -use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; @@ -86,7 +86,7 @@ where .await?; let stackonly_structlog_opt: Option> = - normalize_structlog(&stackonly_structlog_trace).await; + trace2structlog(stackonly_structlog_trace).unwrap_or_default(); let need_memory = stackonly_structlog_opt .as_deref() @@ -102,7 +102,7 @@ where structlog_tracing_options(true, need_memory, false), ); - let memory_structlog = normalize_structlog(&memory_structlog_fut.await?).await; + let memory_structlog = trace2structlog(memory_structlog_fut.await?).unwrap_or_default(); Ok::>, RpcError>(memory_structlog) }; @@ -398,131 +398,91 @@ pub(crate) fn generate_jumpdest_table( Ok((jumpdest_table, code_db)) } +fn trace2structlog(trace: GethTrace) -> Result>, serde_json::Error> { + match trace { + GethTrace::Default(it) => Ok(Some(it.struct_logs)), + GethTrace::JS(it) => Ok(Some(compat::deserialize(it)?.struct_logs)), + _ => Ok(None), + } +} /// This module exists as a workaround for parsing `StructLog`. The `error` -/// field is a string in Alloy but an object in Erigon. -pub mod structlogprime { - use core::option::Option::None; - use std::collections::BTreeMap; +/// field is a string in Geth and Alloy but an object in Erigon. A PR[^1] has +/// been merged to fix this upstream and should eventually render this +/// unnecessary. [^1]: `https://github.com/erigontech/erigon/pull/12089` +mod compat { + use std::{collections::BTreeMap, fmt, iter}; - use alloy::rpc::types::trace::geth::{DefaultFrame, GethTrace, StructLog}; + use alloy::rpc::types::trace::geth::{DefaultFrame, StructLog}; use alloy_primitives::{Bytes, B256, U256}; - use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; - use serde_json::Value; - - /// Geth Default struct log trace frame - /// - /// - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub(crate) struct DefaultFramePrime { - /// Whether the transaction failed - pub failed: bool, - /// How much gas was used. - pub gas: u64, - /// Output of the transaction - #[serde(serialize_with = "alloy_serde::serialize_hex_string_no_prefix")] - pub return_value: Bytes, - /// Recorded traces of the transaction - pub struct_logs: Vec, - } + use serde::{de::SeqAccess, Deserialize, Deserializer}; - /// Represents a struct log entry in a trace - /// - /// - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] - pub(crate) struct StructLogPrime { - /// Program counter - pub pc: u64, - /// Opcode to be executed - pub op: String, - /// Remaining gas - pub gas: u64, - /// Cost for executing op - #[serde(rename = "gasCost")] - pub gas_cost: u64, - /// Current call depth - pub depth: u64, - /// Error message if any - #[serde(default, skip)] - pub error: Option, - /// EVM stack - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stack: Option>, - /// Last call's return data. Enabled via enableReturnData - #[serde( - default, - rename = "returnData", - skip_serializing_if = "Option::is_none" - )] - pub return_data: Option, - /// ref - #[serde(default, skip_serializing_if = "Option::is_none")] - pub memory: Option>, - /// Size of memory. - #[serde(default, rename = "memSize", skip_serializing_if = "Option::is_none")] - pub memory_size: Option, - /// Storage slots of current contract read from and written to. Only - /// emitted for SLOAD and SSTORE. Disabled via disableStorage - #[serde( - default, - skip_serializing_if = "Option::is_none", - serialize_with = "serialize_string_storage_map_opt" - )] - pub storage: Option>, - /// Refund counter - #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] - pub refund_counter: Option, + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + _DefaultFrame::deserialize(d) } - /// Serializes a storage map as a list of key-value pairs _without_ - /// 0x-prefix - pub(crate) fn serialize_string_storage_map_opt( - storage: &Option>, - s: S, - ) -> Result { - match storage { - None => s.serialize_none(), - Some(storage) => { - let mut m = s.serialize_map(Some(storage.len()))?; - for (key, val) in storage.iter() { - let key = format!("{:?}", key); - let val = format!("{:?}", val); - // skip the 0x prefix - m.serialize_entry(&key.as_str()[2..], &val.as_str()[2..])?; - } - m.end() - } + /// The `error` field is a `string` in `geth` etc. but an `object` in + /// `erigon`. + fn error<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum Error { + String(String), + #[allow(dead_code)] + Object(serde_json::Map), } + Ok(match Error::deserialize(d)? { + Error::String(it) => Some(it), + Error::Object(_) => None, + }) } - impl TryInto for DefaultFramePrime { - fn try_into(self) -> Result { - let a = serde_json::to_string(&self)?; - let b: DefaultFramePrime = serde_json::from_str(&a)?; - let c = serde_json::to_string(&b)?; - let d: DefaultFrame = serde_json::from_str(&c)?; - Ok(d) - } - - type Error = anyhow::Error; + #[derive(Deserialize)] + #[serde(remote = "DefaultFrame", rename_all = "camelCase")] + struct _DefaultFrame { + failed: bool, + gas: u64, + return_value: Bytes, + #[serde(deserialize_with = "vec_structlog")] + struct_logs: Vec, } - pub fn try_reserialize(structlog_object: &Value) -> anyhow::Result { - let a = serde_json::to_string(structlog_object)?; - let b: DefaultFramePrime = serde_json::from_str(&a)?; - let d: DefaultFrame = b.try_into()?; - Ok(d) + fn vec_structlog<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an array of `StructLog`") + } + fn visit_seq>(self, mut seq: A) -> Result { + #[derive(Deserialize)] + struct With(#[serde(with = "_StructLog")] StructLog); + let v = iter::from_fn(|| seq.next_element().transpose()) + .map(|it| it.map(|With(it)| it)) + .collect::>()?; + Ok(v) + } + } + + d.deserialize_seq(Visitor) } - pub(crate) async fn normalize_structlog( - unnormalized_structlog: &GethTrace, - ) -> Option> { - match unnormalized_structlog { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs.clone()), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - } + #[derive(Deserialize)] + #[serde(remote = "StructLog", rename_all = "camelCase")] + struct _StructLog { + pc: u64, + op: String, + gas: u64, + gas_cost: u64, + depth: u64, + #[serde(deserialize_with = "error")] + error: Option, + stack: Option>, + return_data: Option, + memory: Option>, + #[serde(rename = "memSize")] + memory_size: Option, + storage: Option>, + #[serde(rename = "refund")] + refund_counter: Option, } } From c11d17d80c7909a8559b2b5ccc4a85a1ad0d4ca7 Mon Sep 17 00:00:00 2001 From: Marko Atanasievski Date: Mon, 7 Oct 2024 20:31:25 +0200 Subject: [PATCH 054/107] feat: block structlog retrieval (#682) * feat: use block structlogs retrieval * fix: cleanup cli argumetns * fix: error handling * fix: error handling 2 * adjustments * avoid CREATE/2 * fixes * rename function * remove support for CREATE/2 --- zero/src/bin/leader/cli.rs | 2 +- zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jerigon.rs | 129 ++++++++---------------- zero/src/rpc/jumpdest.rs | 198 ++++++++++--------------------------- zero/src/rpc/mod.rs | 3 +- zero/src/rpc/native/txn.rs | 169 ++++++++++++++++++------------- 6 files changed, 199 insertions(+), 304 deletions(-) diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index d89a349ea..cab645fbd 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,7 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index fe29247ee..5dd548992 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,7 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 73280fc99..5138346a9 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,30 +1,25 @@ use core::iter::Iterator; -use std::collections::BTreeMap; use std::ops::Deref as _; -use __compat_primitive_types::H160; +use alloy::eips::BlockNumberOrTag; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, Block, BlockTransactionsKind}, transports::Transport, }; -use alloy_primitives::Address; use anyhow::Context as _; -use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; -use futures::stream::FuturesOrdered; -use futures::StreamExt as _; +use compat::Compat; +use evm_arithmetization::jumpdest::JumpDestTableWitness; use serde::Deserialize; use serde_json::json; -use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnTrace}; -use tracing::info; +use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; +use tracing::warn; -use super::{ - fetch_other_block_data, - jumpdest::{self, get_normalized_structlog}, - JumpdestSrc, -}; +use super::{fetch_other_block_data, JumpdestSrc}; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; +use crate::rpc::jumpdest::{generate_jumpdest_table, get_block_normalized_structlogs}; + /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] pub struct ZeroTxResult { @@ -67,30 +62,31 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let jdts: Vec> = match jumpdest_src { + let block_jumpdest_table_witnesses: Vec> = match jumpdest_src { JumpdestSrc::ProverSimulation => vec![None; tx_results.len()], JumpdestSrc::ClientFetchedStructlogs => { + // In case of the error with retrieving structlogs from the server, + // continue without interruption. Equivalent to `ProverSimulation` case. process_transactions( &block, cached_provider.get_provider().await?.deref(), - tx_results.iter().map(|TxnInfo { traces, meta: _ }| traces), // &tx_traces, + &tx_results, ) - .await? + .await + .unwrap_or_else(|e| { + warn!("failed to fetch server structlogs for block {target_block_id}: {e}"); + Vec::new() + }) } - JumpdestSrc::ServerFetchedStructlogs => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), JumpdestSrc::Serverside => todo!(), }; - let mut code_db = CodeDb::default(); // weave in the JDTs let txn_info = tx_results .into_iter() - .zip(jdts) - .map(|(mut tx_info, jdt)| { - tx_info.meta.jumpdest_table = jdt.map(|(j, c)| { - code_db.extend(c); - j - }); + .zip(block_jumpdest_table_witnesses) + .map(|(mut tx_info, jdtw)| { + tx_info.meta.jumpdest_table = jdtw; tx_info }) .collect(); @@ -106,81 +102,42 @@ where .context("invalid hex returned from call to eth_getWitness")?, }), txn_info, - code_db, + code_db: Default::default(), }, other_data, }) } -/// Processes the transactions in the given block and updates the code db. -pub async fn process_transactions<'i, I, ProviderT, TransportT>( +/// Processes the transactions in the given block, generating jumpdest tables +/// and updates the code database +pub async fn process_transactions<'i, ProviderT, TransportT>( block: &Block, provider: &ProviderT, - tx_traces: I, -) -> anyhow::Result>> + tx_results: &[TxnInfo], +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, - I: Iterator>, { - let futures = block + let block_structlogs = + get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) + .await?; + + let tx_traces = tx_results + .iter() + .map(|tx| tx.traces.iter().map(|(h, t)| (h.compat(), t))); + + let block_jumpdest_tables = block .transactions .as_transactions() - .context("No transactions in block")? + .context("no transactions in block")? .iter() + .zip(block_structlogs) .zip(tx_traces) - .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) - .collect::>(); - - futures - .collect::>() - .await - .into_iter() - .collect::, _>>() -} - -/// Processes the transaction with the given transaction hash and updates the -/// accounts state. -pub async fn process_transaction( - provider: &ProviderT, - tx: &Transaction, - tx_trace: &BTreeMap, -) -> anyhow::Result> -where - ProviderT: Provider, - TransportT: Transport + Clone, -{ - let tx_traces = tx_trace - .iter() - .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) - .collect(); - - let structlog_opt: Option> = get_normalized_structlog(provider, &tx.hash) - .await - .ok() - .flatten(); - - let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_log| { - jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( - |error| { - info!( - "{:#?}: JumpDestTable generation failed with reason: {}", - tx.hash, error - ); - None - }, - |(jdt, code_db)| { - info!( - "{:#?}: JumpDestTable generation succeeded with result: {}", - tx.hash, jdt - ); - Some((jdt, code_db)) - }, - ) - }); - - // TODO - // let jumpdest_table = jc.map(|(j, c)| j); + .map(|((tx, structlog), tx_trace)| { + structlog.and_then(|it| generate_jumpdest_table(tx, &it.1, tx_trace).ok()) + }) + .collect::>(); - Ok(jc) + Ok(block_jumpdest_tables) } diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 3f5057b27..976125877 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -1,43 +1,31 @@ use core::default::Default; use core::option::Option::None; use core::str::FromStr as _; -use core::time::Duration; -use std::collections::BTreeMap; use std::collections::HashMap; use std::ops::Not as _; use __compat_primitive_types::H160; use __compat_primitive_types::H256; -use alloy::primitives::Address; -use alloy::primitives::U160; +use alloy::eips::BlockNumberOrTag; +use alloy::primitives::{Address, U160}; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; -use alloy::rpc::types::trace::geth::GethTrace; -use alloy::rpc::types::trace::geth::StructLog; -use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; -use alloy::transports::RpcError; +use alloy::rpc::types::trace::geth::{ + GethDebugTracingOptions, GethDefaultTracingOptions, GethTrace, StructLog, TraceResult, +}; use alloy::transports::Transport; -use alloy::transports::TransportErrorKind; -use alloy_primitives::B256; -use alloy_primitives::U256; +use alloy_primitives::{TxHash, U256}; +use anyhow::bail; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; -use evm_arithmetization::CodeDb; use keccak_hash::keccak; -use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; -use tracing::trace; - -/// The maximum time we are willing to wait for a structlog before failing over -/// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(10 * 60); - -/// Structure of Etheruem memory -type Word = [u8; 32]; -const WORDSIZE: usize = std::mem::size_of::(); +use tracing::{trace, warn}; +#[derive(Debug, Clone)] +pub struct TxStructLogs(pub Option, pub Vec); /// Pass `true` for the components needed. fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { @@ -62,80 +50,52 @@ fn get_code_hash(usage: &ContractCodeUsage) -> H256 { } } -/// Predicate that determines whether a `StructLog` that includes memory is -/// required. -fn trace_contains_create(structlog: &[StructLog]) -> bool { - structlog - .iter() - .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") -} - -/// Gets the lightest possible structlog for transcation `tx_hash`. -pub(crate) async fn get_normalized_structlog( +pub(crate) async fn get_block_normalized_structlogs( provider: &ProviderT, - tx_hash: &B256, -) -> Result>, RpcError> + block: &BlockNumberOrTag, +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, { - let inner = async { - // Optimization: It may be a better default to pull the stack immediately. - let stackonly_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) - .await?; - - let stackonly_structlog_opt: Option> = - trace2structlog(stackonly_structlog_trace).unwrap_or_default(); - - let need_memory = stackonly_structlog_opt - .as_deref() - .is_some_and(trace_contains_create); - trace!("Need structlog with memory: {need_memory}"); - - if need_memory.not() { - return Ok(stackonly_structlog_opt); - }; - - let memory_structlog_fut = provider.debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ); - - let memory_structlog = trace2structlog(memory_structlog_fut.await?).unwrap_or_default(); - - Ok::>, RpcError>(memory_structlog) - }; + let block_stackonly_structlog_traces = provider + .debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)) + .await?; + + let block_normalized_stackonly_structlog_traces = block_stackonly_structlog_traces + .into_iter() + .map(|tx_trace_result| match tx_trace_result { + TraceResult::Success { + result, tx_hash, .. + } => Ok(trace_to_tx_structlog(tx_hash, result)), + TraceResult::Error { error, tx_hash } => Err(anyhow::anyhow!( + "error fetching structlog for tx: {tx_hash:?}. Error: {error:?}" + )), + }) + .collect::>, anyhow::Error>>()?; - match timeout(TIMEOUT_LIMIT, inner).await { - Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( - ellapsed_error, - )))), - Ok(structlog_res) => Ok(structlog_res?), - } + Ok(block_normalized_stackonly_structlog_traces) } /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -pub(crate) fn generate_jumpdest_table( +pub(crate) fn generate_jumpdest_table<'a>( tx: &Transaction, struct_log: &[StructLog], - tx_traces: &BTreeMap, -) -> anyhow::Result<(JumpDestTableWitness, CodeDb)> { + tx_traces: impl Iterator, +) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); let mut jumpdest_table = JumpDestTableWitness::default(); - let mut code_db = CodeDb::default(); // This map does neither contain the `init` field of Contract Deployment // transactions nor CREATE, CREATE2 payloads. let callee_addr_to_code_hash: HashMap = tx_traces - .iter() .filter_map(|(callee_addr, trace)| { trace .code_usage .as_ref() - .map(|code| (*callee_addr, get_code_hash(code))) + .map(|code| (callee_addr, get_code_hash(code))) }) .collect(); @@ -152,10 +112,10 @@ pub(crate) fn generate_jumpdest_table( let entrypoint_code_hash: H256 = match tx.to { Some(to_address) if is_precompile(H160::from_str(&to_address.to_string())?) => { - return Ok((jumpdest_table, code_db)) + return Ok(jumpdest_table) } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { - return Ok((jumpdest_table, code_db)) + return Ok(jumpdest_table) } Some(to_address) => callee_addr_to_code_hash[&to_address], None => { @@ -259,68 +219,10 @@ pub(crate) fn generate_jumpdest_table( next_ctx_available += 1; } "CREATE" | "CREATE2" => { - prev_jump = None; - ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); - // We reverse the stack, so the order matches our assembly code. - let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands_used = 3; - - if evm_stack.len() < operands_used { - trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - continue; - }; - - let [_value, offset, size, ..] = evm_stack[..] else { - unreachable!() - }; - if *offset > U256::from(usize::MAX) { - trace!( - "{op}: Offset {offset} was too large to fit in usize {}.", - usize::MAX - ); - continue; - }; - let offset: usize = offset.to(); - - if *size > U256::from(usize::MAX) { - trace!( - "{op}: Size {size} was too large to fit in usize {}.", - usize::MAX - ); - continue; - }; - let size: usize = size.to(); - - let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - - if entry.memory.is_none() || offset + size > memory_size { - trace!("Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size); - continue; - } - let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory_parsed: Vec> = memory_raw - .iter() - .map(|mem_line| { - let mem_line_parsed = U256::from_str_radix(mem_line, 16)?; - Ok(mem_line_parsed.to_be_bytes()) - }) - .collect(); - let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); - if mem_res.is_err() { - trace!( - "{op}: Parsing memory failed with error: {}", - mem_res.unwrap_err() - ); - continue; - } - let memory: Vec = mem_res.unwrap().concat(); - - let init_code = &memory[offset..offset + size]; - code_db.insert(init_code.to_vec()); - let init_code_hash = keccak(init_code); - call_stack.push((init_code_hash, next_ctx_available)); - - next_ctx_available += 1; + bail!(format!( + "{} requires memory, aborting JUMPDEST-table generation.", + tx.hash + )); } "JUMP" => { prev_jump = None; @@ -383,7 +285,7 @@ pub(crate) fn generate_jumpdest_table( ); continue; } - assert!(jumpdest_offset.unwrap() < 24576); + ensure!(jumpdest_offset.unwrap() < 24576); jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { @@ -395,16 +297,24 @@ pub(crate) fn generate_jumpdest_table( } } } - Ok((jumpdest_table, code_db)) + Ok(jumpdest_table) } -fn trace2structlog(trace: GethTrace) -> Result>, serde_json::Error> { +fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option { match trace { - GethTrace::Default(it) => Ok(Some(it.struct_logs)), - GethTrace::JS(it) => Ok(Some(compat::deserialize(it)?.struct_logs)), - _ => Ok(None), + GethTrace::Default(structlog_frame) => { + Some(TxStructLogs(tx_hash, structlog_frame.struct_logs)) + } + GethTrace::JS(it) => { + let default_frame = compat::deserialize(it) + .inspect_err(|e| warn!("failed to deserialize js default frame {e:?}")) + .ok()?; + Some(TxStructLogs(tx_hash, default_frame.struct_logs)) + } + _ => None, } } + /// This module exists as a workaround for parsing `StructLog`. The `error` /// field is a string in Geth and Alloy but an object in Erigon. A PR[^1] has /// been merged to fix this upstream and should eventually render this @@ -474,7 +384,7 @@ mod compat { gas: u64, gas_cost: u64, depth: u64, - #[serde(deserialize_with = "error")] + #[serde(default, deserialize_with = "error")] error: Option, stack: Option>, return_data: Option, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 4f4908eb0..4e8a0278e 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -46,8 +46,7 @@ pub enum RpcType { pub enum JumpdestSrc { ProverSimulation, ClientFetchedStructlogs, - ServerFetchedStructlogs, // later - Serverside, // later + Serverside, // later } /// Obtain the prover input for one block diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 96d30f554..ffd692892 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -2,6 +2,8 @@ use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use __compat_primitive_types::{H256, U256}; +use alloy::eips::BlockNumberOrTag; +use alloy::rpc::types::trace::geth::TraceResult; use alloy::{ primitives::{keccak256, Address, B256}, providers::{ @@ -19,17 +21,41 @@ use alloy::{ }, transports::Transport, }; -use anyhow::{Context as _, Ok}; +use anyhow::{bail, Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::info; +use tracing::{error, warn}; +use crate::rpc::jumpdest::get_block_normalized_structlogs; use crate::rpc::Compat; use crate::rpc::{ - jumpdest::{self, get_normalized_structlog}, + jumpdest::{self}, JumpdestSrc, }; +pub(crate) async fn get_block_prestate_traces( + provider: &ProviderT, + block: &BlockNumberOrTag, + tracing_options: GethDebugTracingOptions, +) -> anyhow::Result> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let block_prestate_traces = provider + .debug_trace_block_by_number(*block, tracing_options) + .await?; + + block_prestate_traces + .into_iter() + .map(|trace_result| match trace_result { + TraceResult::Success { result, .. } => Ok(result), + TraceResult::Error { error, .. } => { + bail!("error fetching block prestate traces: {:?}", error) + } + }) + .collect::, anyhow::Error>>() +} /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( @@ -41,12 +67,58 @@ where ProviderT: Provider, TransportT: Transport + Clone, { + // Get block prestate traces + let block_prestate_trace = get_block_prestate_traces( + provider, + &BlockNumberOrTag::from(block.header.number), + prestate_tracing_options(false), + ) + .await?; + + // Get block diff traces + let block_diff_trace = get_block_prestate_traces( + provider, + &BlockNumberOrTag::from(block.header.number), + prestate_tracing_options(true), + ) + .await?; + + let block_structlogs = match jumpdest_src { + JumpdestSrc::ProverSimulation => vec![None; block_prestate_trace.len()], + JumpdestSrc::ClientFetchedStructlogs => { + // In case of the error with retrieving structlogs from the server, + // continue without interruption. Equivalent to `ProverSimulation` case. + get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) + .await + .unwrap_or_else(|e| { + warn!( + "failed to fetch server structlogs for block {}: {e}", + block.header.number + ); + vec![None; block_prestate_trace.len()] + }) + .into_iter() + .map(|tx_struct_log| tx_struct_log.map(|it| it.1)) + .collect() + } + JumpdestSrc::Serverside => todo!(), + }; + block .transactions .as_transactions() .context("No transactions in block")? .iter() - .map(|tx| process_transaction(provider, tx, jumpdest_src)) + .zip( + block_prestate_trace.into_iter().zip( + block_diff_trace + .into_iter() + .zip(block_structlogs.into_iter()), + ), + ) + .map(|(tx, (pre_trace, (diff_trace, structlog)))| { + process_transaction(provider, tx, pre_trace, diff_trace, structlog) + }) .collect::>() .try_fold( (BTreeSet::new(), Vec::new()), @@ -64,26 +136,24 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, - jumpdest_src: JumpdestSrc, + pre_trace: GethTrace, + diff_trace: GethTrace, + structlog_opt: Option>, ) -> anyhow::Result<(CodeDb, TxnInfo)> where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace, structlog_opt) = - fetch_tx_data(provider, &tx.hash, jumpdest_src).await?; + let tx_receipt = fetch_tx_receipt(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let (mut code_db, mut tx_traces) = match (pre_trace, diff_trace) { + let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), GethTrace::PreStateTracer(PreStateFrame::Diff(diff)), - ) => { - info!("{:?} {:?} {:?}", tx.hash, read, diff); - process_tx_traces(access_list, read, diff).await? - } + ) => process_tx_traces(access_list, read, diff).await?, _ => unreachable!(), }; @@ -92,28 +162,19 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_logs| { - jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( - |error| { - info!( - "{:#?}: JumpDestTable generation failed with reason: {}", - tx.hash, error - ); - None - }, - |(jdt, code_db)| { - info!( - "{:#?}: JumpDestTable generation succeeded with result: {}", - tx.hash, jdt - ); - Some((jdt, code_db)) - }, - ) - }); - - let jumpdest_table = jc.map(|(j, c)| { - code_db.extend(c); - j + let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, tx_traces.iter().map(|(a, t)| (*a, t))) + .map_or_else( + |error| { + error!( + "{}: JumpDestTable generation failed with reason: {:?}", + tx.hash.to_string(), + error + ); + None + }, + Some, + ) }); let tx_meta = TxnMeta { @@ -136,48 +197,16 @@ where } /// Fetches the transaction data for the given transaction hash. -async fn fetch_tx_data( +async fn fetch_tx_receipt( provider: &ProviderT, tx_hash: &B256, - jumpdest_src: JumpdestSrc, -) -> anyhow::Result<( - ::ReceiptResponse, - GethTrace, - GethTrace, - Option>, -)> +) -> anyhow::Result<::ReceiptResponse> where ProviderT: Provider, TransportT: Transport + Clone, { - let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); - let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); - let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { - JumpdestSrc::ClientFetchedStructlogs => { - let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); - futures::try_join!( - tx_receipt_fut, - pre_trace_fut, - diff_trace_fut, - structlog_trace_fut, - )? - } - JumpdestSrc::ProverSimulation => { - let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; - (tx_receipt, pre_trace, diff_trace, None) - } - _ => todo!(), - }; - - Ok(( - tx_receipt.context("Transaction receipt not found.")?, - pre_trace, - diff_trace, - structlog_trace, - )) + let tx_receipt = provider.get_transaction_receipt(*tx_hash).await?; + Ok(tx_receipt.context("Transaction receipt not found.")?) } /// Parse the access list data into a hashmap. From 06b19134b18f7e27f3d96b555473172868eec769 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 02:12:51 +0200 Subject: [PATCH 055/107] better tracing --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 2 +- zero/src/rpc/jerigon.rs | 16 ++++++++++++++-- zero/src/rpc/jumpdest.rs | 27 +++++++++++++++++++-------- zero/src/rpc/native/txn.rs | 4 ++-- 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d72342620..ffc338cd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5160,6 +5160,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -5170,12 +5180,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7eca11df4..168811499 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,7 +98,7 @@ tokio = { version = "1.38.0", features = ["full"] } toml = "0.8.14" tower = "0.4" tracing = { version = "0.1", features = ["attributes"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } u4 = "0.1.0" uint = "0.9.5" url = "2.5.2" diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 5138346a9..0bbdefe60 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -13,7 +13,7 @@ use evm_arithmetization::jumpdest::JumpDestTableWitness; use serde::Deserialize; use serde_json::json; use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; -use tracing::warn; +use tracing::{debug, warn}; use super::{fetch_other_block_data, JumpdestSrc}; use crate::prover::BlockProverInput; @@ -135,7 +135,19 @@ where .zip(block_structlogs) .zip(tx_traces) .map(|((tx, structlog), tx_trace)| { - structlog.and_then(|it| generate_jumpdest_table(tx, &it.1, tx_trace).ok()) + structlog.and_then(|it| { + generate_jumpdest_table(tx, &it.1, tx_trace).map_or_else( + |error| { + debug!( + "{}: JumpDestTable generation failed with reason: {:?}", + tx.hash.to_string(), + error + ); + None + }, + Some, + ) + }) }) .collect::>(); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 976125877..6188d7ae8 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -152,14 +152,25 @@ pub(crate) fn generate_jumpdest_table<'a>( let (code_hash, ctx) = call_stack.last().unwrap(); // REVIEW: will be removed before merge - trace!("TX: {:?}", tx.hash); - trace!("STEP: {:?}", step); - trace!("STEPS: {:?}", struct_log.len()); - trace!("OPCODE: {}", entry.op.as_str()); - trace!("CODE: {:?}", code_hash); - trace!("CTX: {:?}", ctx); - trace!("CURR_DEPTH: {:?}", curr_depth); - trace!("{:#?}\n", entry); + trace!( + step, + curr_depth, + tx_hash = ?tx.hash, + ?code_hash, + ctx, + entry.pc, + op, + // + ?entry, + ); + // trace!("TX: {:?}", tx.hash); + // trace!("STEP: {:?}", step); + // trace!("STEPS: {:?}", struct_log.len()); + // trace!("OPCODE: {}", entry.op.as_str()); + // trace!("CODE: {:?}", code_hash); + // trace!("CTX: {:?}", ctx); + // trace!("CURR_DEPTH: {:?}", curr_depth); + // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index ffd692892..bf69ec18e 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -25,7 +25,7 @@ use anyhow::{bail, Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::{error, warn}; +use tracing::{debug, warn}; use crate::rpc::jumpdest::get_block_normalized_structlogs; use crate::rpc::Compat; @@ -166,7 +166,7 @@ where jumpdest::generate_jumpdest_table(tx, &struct_logs, tx_traces.iter().map(|(a, t)| (*a, t))) .map_or_else( |error| { - error!( + debug!( "{}: JumpDestTable generation failed with reason: {:?}", tx.hash.to_string(), error From 339f6af55b3ffeb85f899a1d45544cc8072fce98 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 02:14:05 +0200 Subject: [PATCH 056/107] bug fix --- .../src/generation/prover_input.rs | 20 ++++++++++--- scripts/test_jerigon.sh | 4 +-- scripts/test_native.sh | 26 +++++++++++++---- zero/src/rpc/jerigon.rs | 2 +- zero/src/rpc/jumpdest.rs | 28 +++++++++++-------- 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e4206cb15..296de9959 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -815,10 +815,22 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - info!("SIMW {:#?}", &simw); - info!("RPCW {:#?}", &rpcw); - info!("SIMP {:#?}", &simp); - info!("RPCP {:#?}", &rpcp); + if simw != &rpcw { + if let Some(s) = simw { + info!("SIMW {}", s); + } + if let Some(r) = rpcw.as_ref() { + info!("RPCW {}", r); + } + info!("SIMW == RPCW ? {}", simw == &rpcw); + info!("tx: {:?}", self.inputs.txn_hashes); + panic!(); + // info!("SIMP {:?}", &simp); + // info!("RPCP {:?}", &rpcp); + // info!("SIMP == RPCP ? {}", &simp == &rpcp); + } else { + info!("JUMPDEST tables similar"); + } self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 8e02977c8..8a515f05e 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -297,8 +297,8 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -#$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND6" +#BLOCKS="72 185" #$ROUND5 $CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3 $ROUND5 $ROUND4" +BLOCKS="$CIBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 7f092f232..eb65b5758 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -set -uo pipefail +set -uxo pipefail -RPC=${RPC_NATIVE} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 @@ -33,9 +32,23 @@ PRECANCUN=" 19240700 " + #It's visible with block 20727641 ROUND1=`echo {20727640..20727650}` +ROUND2=" +20727641 +20727643 +20727644 +20727645 +20727646 +20727647 +20727648 +20727649 +20727650 +" + + CANCUN=19426587 @@ -48,12 +61,12 @@ GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" -BLOCKS="$ROUND1" +BLOCKS="20727641" #BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "githash block verdict duration\n" | tee -a witnesses/native_results.txt +printf "\n\ngithash block verdict duration\n" | tee -a witnesses/native_results.txt echo "------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do @@ -61,12 +74,13 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + rg "jump" $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 - timeout 30m ./prove_stdio.sh $WITNESS test_only + timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 0bbdefe60..21480ab4a 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -75,7 +75,7 @@ where .await .unwrap_or_else(|e| { warn!("failed to fetch server structlogs for block {target_block_id}: {e}"); - Vec::new() + vec![None; tx_results.len()] }) } JumpdestSrc::Serverside => todo!(), diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 6188d7ae8..1462d120f 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -4,10 +4,11 @@ use core::str::FromStr as _; use std::collections::HashMap; use std::ops::Not as _; +use ::compat::Compat; use __compat_primitive_types::H160; use __compat_primitive_types::H256; use alloy::eips::BlockNumberOrTag; -use alloy::primitives::{Address, U160}; +use alloy::primitives::Address; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; @@ -196,16 +197,21 @@ pub(crate) fn generate_jumpdest_table<'a>( unreachable!() }; - if *address > U256::from(U160::MAX) { - trace!( - "{op}: Callee address {} was larger than possible {}.", - *address, - U256::from(U160::MAX) - ); - // Se note above. - continue; - }; - let lower_20_bytes = U160::from(*address); + // if *address > U256::from(U160::MAX) { + // trace!( + // "{op}: Callee address {} was larger than possible {}.", + // *address, + // U256::from(U160::MAX) + // ); + // // Se note above. + // continue; + // }; + let a: [u8; 32] = address.compat().into(); + // a <<= 96; + // a >>= 96; + // let aa: [u8; 32] = a.into(); + let aaa: H256 = a.into(); + let lower_20_bytes: [u8; 20] = H160::from(aaa).into(); let callee_address = Address::from(lower_20_bytes); if callee_addr_to_code_hash.contains_key(&callee_address) { From 61a6b6a32393d03744dd5fc66f2a63d1e50f7b4b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 02:14:58 +0200 Subject: [PATCH 057/107] json --- zero/src/bin/rpc.rs | 2 +- zero/src/tracing.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 5dd548992..0b49a3a0a 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -219,8 +219,8 @@ async fn main() -> anyhow::Result<()> { // With the default configuration trace information is written // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) + .json() .with_ansi(false) - .compact() .with_filter(EnvFilter::from_default_env()), ) .init(); diff --git a/zero/src/tracing.rs b/zero/src/tracing.rs index 6c3f9a8bc..aa404059d 100644 --- a/zero/src/tracing.rs +++ b/zero/src/tracing.rs @@ -5,7 +5,7 @@ pub fn init() { .with( tracing_subscriber::fmt::layer() .with_ansi(false) - .compact() + .json() .with_filter(EnvFilter::from_default_env()), ) .init(); From f8f0a85cbb5096d4685013823872f6f76136ce5a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 17:13:01 +0200 Subject: [PATCH 058/107] reinstantiate timeout --- zero/src/rpc/jumpdest.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 1462d120f..549506177 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -3,6 +3,7 @@ use core::option::Option::None; use core::str::FromStr as _; use std::collections::HashMap; use std::ops::Not as _; +use std::time::Duration; use ::compat::Compat; use __compat_primitive_types::H160; @@ -21,10 +22,15 @@ use anyhow::bail; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::{trace, warn}; + +/// The maximum time we are willing to wait for a block's structlogs before +/// failing over to simulating the JumpDest analysis. +const TIMEOUT_LIMIT: Duration = Duration::from_secs(60); #[derive(Debug, Clone)] pub struct TxStructLogs(pub Option, pub Vec); @@ -59,9 +65,17 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let block_stackonly_structlog_traces = provider - .debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)) - .await?; + let block_stackonly_structlog_traces_fut = + provider.debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)); + + let block_stackonly_structlog_traces = + match timeout(TIMEOUT_LIMIT, block_stackonly_structlog_traces_fut).await { + Ok(traces) => traces?, + Err(elapsed) => { + trace!(target: "fetching block structlogs timed out", ?elapsed); + bail!(elapsed); + } + }; let block_normalized_stackonly_structlog_traces = block_stackonly_structlog_traces .into_iter() From dbb65ea5156ae94344aa3671657fc6a3ebfcb9aa Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 18:01:30 +0200 Subject: [PATCH 059/107] ignore None --- evm_arithmetization/src/generation/prover_input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 296de9959..6e2e32c47 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -815,7 +815,7 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - if simw != &rpcw { + if rpcw.is_some() && simw != &rpcw { if let Some(s) = simw { info!("SIMW {}", s); } From d415d2231886f2c502e36654c512c1390c7747e8 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 23:23:49 +0200 Subject: [PATCH 060/107] feedback --- .../src/cpu/kernel/tests/core/jumpdest_analysis.rs | 10 +++++----- evm_arithmetization/src/generation/prover_input.rs | 4 ++-- evm_arithmetization/tests/global_exit_root.rs | 2 +- zero/src/bin/leader/cli.rs | 7 ++++++- zero/src/bin/rpc.rs | 8 ++++++-- zero/src/rpc/native/txn.rs | 2 +- zero/src/tracing.rs | 1 - 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index d1b076118..0bb07eaf5 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -128,14 +128,14 @@ fn test_jumpdest_analysis() -> Result<()> { // We need to manually pop the jumpdest_table and push its value on the top of // the stack - (*interpreter + interpreter .generation_state .jumpdest_table .as_mut() - .unwrap()) - .get_mut(&CONTEXT) - .unwrap() - .pop(); + .unwrap() + .get_mut(&CONTEXT) + .unwrap() + .pop(); interpreter .push(41.into()) .expect("The stack should not overflow"); diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 6e2e32c47..1e4090aa8 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -375,12 +375,12 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { Ok((next_jumpdest_address + 1).into()) } else { - (*jumpdest_table).remove(&context); + jumpdest_table.remove(&context); Ok(U256::zero()) } } diff --git a/evm_arithmetization/tests/global_exit_root.rs b/evm_arithmetization/tests/global_exit_root.rs index c1f656efe..481643517 100644 --- a/evm_arithmetization/tests/global_exit_root.rs +++ b/evm_arithmetization/tests/global_exit_root.rs @@ -113,7 +113,7 @@ fn test_global_exit_root() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - batch_jumpdest_table: None, + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index cab645fbd..dac8c0b41 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,12 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg( + short = 'j', + long, + default_value = "client-fetched-structlogs", + required = false + )] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 0b49a3a0a..49c6fa995 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,12 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg( + short = 'j', + long, + default_value = "client-fetched-structlogs", + required = false + )] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] @@ -219,7 +224,6 @@ async fn main() -> anyhow::Result<()> { // With the default configuration trace information is written // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) - .json() .with_ansi(false) .with_filter(EnvFilter::from_default_env()), ) diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index bf69ec18e..a7c2777c5 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -206,7 +206,7 @@ where TransportT: Transport + Clone, { let tx_receipt = provider.get_transaction_receipt(*tx_hash).await?; - Ok(tx_receipt.context("Transaction receipt not found.")?) + tx_receipt.context("Transaction receipt not found.") } /// Parse the access list data into a hashmap. diff --git a/zero/src/tracing.rs b/zero/src/tracing.rs index aa404059d..20d376d86 100644 --- a/zero/src/tracing.rs +++ b/zero/src/tracing.rs @@ -5,7 +5,6 @@ pub fn init() { .with( tracing_subscriber::fmt::layer() .with_ansi(false) - .json() .with_filter(EnvFilter::from_default_env()), ) .init(); From 54a7df8416d32c92b986b8fa0e0ab4029e5d89ef Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 23:54:23 +0200 Subject: [PATCH 061/107] feedback: rustdoc --- evm_arithmetization/src/generation/jumpdest.rs | 10 +++++----- zero/src/rpc/jumpdest.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index f980d51ac..6def65e77 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -11,14 +11,14 @@ //! Since an operation like e.g. `PUSH 0x5B` does not encode a valid //! [`JUMPDEST`] in its second byte, and `PUSH32 //! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not -//! encode valid [`JUMPDESTS`] in bytes 1-32, some diligence must be exercised -//! when proving validity of jump operations. +//! encode any valid [`JUMPDEST`] in bytes 1-32, some diligence must be +//! exercised when proving validity of jump operations. //! //! This module concerns itself with data structures for collecting these //! offsets for [`JUMPDEST`] that was visited during an execution and are not -//! recording duplicity. The proofs that each of these offsets are not rendered -//! invalid by any of the previous 32 bytes `PUSH1`-`PUSH32` is computed later -//! in [`prove_context_jumpdests`] on basis of these collections. +//! recording duplicity. The proofs, that each of these offsets are not rendered +//! invalid by `PUSH1`-`PUSH32` in any of the previous 32 bytes, are computed +//! later in `prove_context_jumpdests` on basis of these collections. //! //! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 549506177..0b8016f61 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -347,9 +347,9 @@ fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option Date: Thu, 10 Oct 2024 16:52:30 +0200 Subject: [PATCH 062/107] feedback: add user-specified timeout --- zero/src/bin/leader.rs | 2 ++ zero/src/bin/leader/cli.rs | 5 +++++ zero/src/bin/leader/client.rs | 3 +++ zero/src/bin/rpc.rs | 9 +++++++++ zero/src/parsing.rs | 13 ++++++++++++- zero/src/rpc/jerigon.rs | 13 ++++++++++--- zero/src/rpc/jumpdest.rs | 6 ++---- zero/src/rpc/mod.rs | 5 ++++- zero/src/rpc/native/mod.rs | 7 +++++-- zero/src/rpc/native/txn.rs | 34 +++++++++++++++++++++------------- 10 files changed, 73 insertions(+), 24 deletions(-) diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 775439fde..d1b7c70fd 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -79,6 +79,7 @@ async fn main() -> Result<()> { block_time, backoff, max_retries, + timeout, } => { let previous_proof = get_previous_proof(previous_proof)?; let block_interval = BlockInterval::new(&block_interval)?; @@ -93,6 +94,7 @@ async fn main() -> Result<()> { max_retries, block_time, jumpdest_src, + timeout, }, block_interval, LeaderConfig { diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index dac8c0b41..ede966493 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -1,7 +1,9 @@ use std::path::PathBuf; +use std::time::Duration; use alloy::transports::http::reqwest::Url; use clap::{Parser, Subcommand, ValueHint}; +use zero::parsing::parse_duration; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; use zero::rpc::{JumpdestSrc, RpcType}; @@ -70,6 +72,9 @@ pub(crate) enum Command { /// The maximum number of retries #[arg(long, default_value_t = 0)] max_retries: u32, + /// Timeout for fetching structlog traces + #[arg(long, default_value = "60", value_parser = parse_duration)] + timeout: Duration, }, /// Reads input from HTTP and writes output to a directory. Http { diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index bd01fcdcd..b999bb584 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use alloy::rpc::types::{BlockId, BlockNumberOrTag}; use alloy::transports::http::reqwest::Url; @@ -21,6 +22,7 @@ pub struct RpcParams { pub max_retries: u32, pub block_time: u64, pub jumpdest_src: JumpdestSrc, + pub timeout: Duration, } #[derive(Debug)] @@ -94,6 +96,7 @@ pub(crate) async fn client_main( leader_config.checkpoint_block_number, rpc_params.rpc_type, rpc_params.jumpdest_src, + rpc_params.timeout, ) .await?; block_tx diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 49c6fa995..e87e9e34d 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use alloy::primitives::B256; use alloy::providers::Provider; @@ -13,6 +14,7 @@ use tracing_subscriber::{prelude::*, EnvFilter}; use url::Url; use zero::block_interval::BlockInterval; use zero::block_interval::BlockIntervalStream; +use zero::parsing::parse_duration; use zero::prover::BlockProverInput; use zero::provider::CachedProvider; use zero::rpc; @@ -27,6 +29,7 @@ struct FetchParams { pub checkpoint_block_number: Option, pub rpc_type: RpcType, pub jumpdest_src: JumpdestSrc, + pub timeout: Duration, } #[derive(Args, Clone, Debug)] @@ -51,6 +54,9 @@ struct RpcToolConfig { /// The maximum number of retries. #[arg(long, default_value_t = 0)] max_retries: u32, + /// Timeout for fetching structlog traces + #[arg(long, default_value = "60", value_parser = parse_duration)] + timeout: Duration, } #[derive(Subcommand)] @@ -113,6 +119,7 @@ where checkpoint_block_number, params.rpc_type, params.jumpdest_src, + params.timeout, ) .await?; @@ -142,6 +149,7 @@ impl Cli { checkpoint_block_number, rpc_type: self.config.rpc_type, jumpdest_src: self.config.jumpdest_src, + timeout: self.config.timeout, }; let block_prover_inputs = @@ -169,6 +177,7 @@ impl Cli { checkpoint_block_number: None, rpc_type: self.config.rpc_type, jumpdest_src: self.config.jumpdest_src, + timeout: self.config.timeout, }; let block_prover_inputs = diff --git a/zero/src/parsing.rs b/zero/src/parsing.rs index d1452a464..d22634968 100644 --- a/zero/src/parsing.rs +++ b/zero/src/parsing.rs @@ -1,5 +1,11 @@ //! Parsing utilities. -use std::{fmt::Display, ops::Add, ops::Range, str::FromStr}; +use std::{ + fmt::Display, + num::ParseIntError, + ops::{Add, Range}, + str::FromStr, + time::Duration, +}; use thiserror::Error; @@ -79,6 +85,11 @@ where } } +pub fn parse_duration(arg: &str) -> Result { + let seconds = arg.parse()?; + Ok(Duration::from_secs(seconds)) +} + #[cfg(test)] mod test { use super::*; diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 21480ab4a..6c9f08e5e 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,5 +1,6 @@ use core::iter::Iterator; use std::ops::Deref as _; +use std::time::Duration; use alloy::eips::BlockNumberOrTag; use alloy::{ @@ -33,6 +34,7 @@ pub async fn block_prover_input( target_block_id: BlockId, checkpoint_block_number: u64, jumpdest_src: JumpdestSrc, + fetch_timeout: Duration, ) -> anyhow::Result where ProviderT: Provider, @@ -71,6 +73,7 @@ where &block, cached_provider.get_provider().await?.deref(), &tx_results, + &fetch_timeout, ) .await .unwrap_or_else(|e| { @@ -114,14 +117,18 @@ pub async fn process_transactions<'i, ProviderT, TransportT>( block: &Block, provider: &ProviderT, tx_results: &[TxnInfo], + fetch_timeout: &Duration, ) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, { - let block_structlogs = - get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) - .await?; + let block_structlogs = get_block_normalized_structlogs( + provider, + &BlockNumberOrTag::from(block.header.number), + fetch_timeout, + ) + .await?; let tx_traces = tx_results .iter() diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 0b8016f61..0645254a8 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -28,9 +28,6 @@ use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::{trace, warn}; -/// The maximum time we are willing to wait for a block's structlogs before -/// failing over to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(60); #[derive(Debug, Clone)] pub struct TxStructLogs(pub Option, pub Vec); @@ -60,6 +57,7 @@ fn get_code_hash(usage: &ContractCodeUsage) -> H256 { pub(crate) async fn get_block_normalized_structlogs( provider: &ProviderT, block: &BlockNumberOrTag, + fetch_timeout: &Duration, ) -> anyhow::Result>> where ProviderT: Provider, @@ -69,7 +67,7 @@ where provider.debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)); let block_stackonly_structlog_traces = - match timeout(TIMEOUT_LIMIT, block_stackonly_structlog_traces_fut).await { + match timeout(*fetch_timeout, block_stackonly_structlog_traces_fut).await { Ok(traces) => traces?, Err(elapsed) => { trace!(target: "fetching block structlogs timed out", ?elapsed); diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 4e8a0278e..1fa63089d 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -1,6 +1,6 @@ zk_evm_common::check_chain_features!(); -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use __compat_primitive_types::{H256, U256}; use alloy::{ @@ -56,6 +56,7 @@ pub async fn block_prover_input( checkpoint_block_number: u64, rpc_type: RpcType, jumpdest_src: JumpdestSrc, + fetch_timeout: Duration, ) -> Result where ProviderT: Provider, @@ -68,6 +69,7 @@ where block_id, checkpoint_block_number, jumpdest_src, + fetch_timeout, ) .await } @@ -77,6 +79,7 @@ where block_id, checkpoint_block_number, jumpdest_src, + fetch_timeout, ) .await } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 76dd302ae..5fa2ff0aa 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -1,5 +1,5 @@ -use std::ops::Deref; use std::sync::Arc; +use std::{ops::Deref, time::Duration}; use alloy::{ providers::Provider, @@ -25,13 +25,14 @@ pub async fn block_prover_input( block_number: BlockId, checkpoint_block_number: u64, jumpdest_src: JumpdestSrc, + fetch_timeout: Duration, ) -> anyhow::Result where ProviderT: Provider, TransportT: Transport + Clone, { let (block_trace, other_data) = try_join!( - process_block_trace(provider.clone(), block_number, jumpdest_src), + process_block_trace(provider.clone(), block_number, jumpdest_src, &fetch_timeout), crate::rpc::fetch_other_block_data(provider.clone(), block_number, checkpoint_block_number) )?; @@ -46,6 +47,7 @@ pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, jumpdest_src: JumpdestSrc, + fetch_timeout: &Duration, ) -> anyhow::Result where ProviderT: Provider, @@ -59,6 +61,7 @@ where &block, cached_provider.get_provider().await?.deref(), jumpdest_src, + fetch_timeout, ) .await?; let trie_pre_images = state::process_state_witness(cached_provider, block, &txn_info).await?; diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index a7c2777c5..c9655f543 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,5 +1,6 @@ use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::time::Duration; use __compat_primitive_types::{H256, U256}; use alloy::eips::BlockNumberOrTag; @@ -62,6 +63,7 @@ pub async fn process_transactions( block: &Block, provider: &ProviderT, jumpdest_src: JumpdestSrc, + fetch_timeout: &Duration, ) -> anyhow::Result<(CodeDb, Vec)> where ProviderT: Provider, @@ -88,20 +90,26 @@ where JumpdestSrc::ClientFetchedStructlogs => { // In case of the error with retrieving structlogs from the server, // continue without interruption. Equivalent to `ProverSimulation` case. - get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) - .await - .unwrap_or_else(|e| { - warn!( - "failed to fetch server structlogs for block {}: {e}", - block.header.number - ); - vec![None; block_prestate_trace.len()] - }) - .into_iter() - .map(|tx_struct_log| tx_struct_log.map(|it| it.1)) - .collect() + get_block_normalized_structlogs( + provider, + &BlockNumberOrTag::from(block.header.number), + fetch_timeout, + ) + .await + .unwrap_or_else(|e| { + warn!( + "failed to fetch server structlogs for block {}: {e}", + block.header.number + ); + vec![None; block_prestate_trace.len()] + }) + .into_iter() + .map(|tx_struct_log| tx_struct_log.map(|it| it.1)) + .collect() } - JumpdestSrc::Serverside => todo!(), + JumpdestSrc::Serverside => todo!( + "Not implemented. See https://github.com/0xPolygonZero/erigon/issues/20 for details." + ), }; block From 98b9c8e5c430e537099bd717b16f17a1597444bd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 11 Oct 2024 17:23:13 +0200 Subject: [PATCH 063/107] feedback --- Cargo.toml | 41 ++++++++-------- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 3 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- trace_decoder/src/typed_mpt.rs | 1 - zero/src/rpc/jumpdest.rs | 47 ++++++------------- zero/src/tracing.rs | 1 + 15 files changed, 47 insertions(+), 68 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bfe6ca64c..f3f368650 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,13 @@ [workspace] members = [ - "common", - "compat", - "evm_arithmetization", - "mpt_trie", - "proc_macro", - "smt_trie", - "trace_decoder", - "zero", + "common", + "compat", + "evm_arithmetization", + "mpt_trie", + "proc_macro", + "smt_trie", + "trace_decoder", + "zero", ] resolver = "2" @@ -22,18 +22,18 @@ categories = ["cryptography::cryptocurrencies"] [workspace.dependencies] __compat_primitive_types = { version = "0.12.2", package = "primitive-types" } alloy = { version = '0.3.0', default-features = false, features = [ - "consensus", - "reqwest", - "json-rpc", - "rlp", - "rpc", - "rpc-client", - "rpc-types-eth", - "rpc-types-trace", - "providers", - "transports", - "transport-http", - "rpc-types-debug", + "consensus", + "reqwest", + "json-rpc", + "rlp", + "rpc", + "rpc-client", + "rpc-types-eth", + "rpc-types-trace", + "providers", + "transports", + "transport-http", + "rpc-types-debug", ] } alloy-primitives = "0.8.0" alloy-serde = "0.3.0" @@ -52,7 +52,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 26ed8136e..38344629c 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -192,7 +192,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 267d393eb..31857ac61 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -60,7 +60,7 @@ pub(crate) struct Interpreter { pub(crate) halt_context: Option, /// Counts the number of appearances of each opcode. For debugging purposes. pub(crate) opcode_count: HashMap, - /// A table of contexts and their reached JUMPDESTs. + /// A table of call contexts and the JUMPDEST offsets that they jumped to. jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -105,7 +105,6 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - // (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) interpreter.generation_state.jumpdest_table = Some(a.clone()); Some((a, jdtw)) } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index a71f2d67d..71933ec3e 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -193,7 +193,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let initial_stack = vec![]; @@ -371,7 +371,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index d254a19b1..27374888c 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 5c955d482..87d959925 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index b86dcb59f..2cb9a538b 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -195,7 +195,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index ba139f6d2..f34d5c621 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -199,7 +199,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index afceefca3..871fa90f7 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -266,7 +266,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 6a13ace55..eaf56dea6 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -170,7 +170,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 56c8ba0ae..f40eadc67 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -162,7 +162,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index d2d461030..4b3656cb6 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -105,7 +105,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/trace_decoder/src/typed_mpt.rs b/trace_decoder/src/typed_mpt.rs index f9c1467ab..8baf3cf29 100644 --- a/trace_decoder/src/typed_mpt.rs +++ b/trace_decoder/src/typed_mpt.rs @@ -280,7 +280,6 @@ pub trait StateTrie { ) -> anyhow::Result>; fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()>; fn get_by_address(&self, address: Address) -> Option; - #[allow(dead_code)] fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; /// _Hash out_ parts of the trie that aren't in `txn_ixs`. fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 0645254a8..1cac95203 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -1,13 +1,10 @@ use core::default::Default; use core::option::Option::None; -use core::str::FromStr as _; use std::collections::HashMap; use std::ops::Not as _; use std::time::Duration; use ::compat::Compat; -use __compat_primitive_types::H160; -use __compat_primitive_types::H256; use alloy::eips::BlockNumberOrTag; use alloy::primitives::Address; use alloy::providers::ext::DebugApi; @@ -22,12 +19,15 @@ use anyhow::bail; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use ruint::Uint; use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::{trace, warn}; +use crate::rpc::H256; + #[derive(Debug, Clone)] pub struct TxStructLogs(pub Option, pub Vec); @@ -124,9 +124,7 @@ pub(crate) fn generate_jumpdest_table<'a>( ); let entrypoint_code_hash: H256 = match tx.to { - Some(to_address) if is_precompile(H160::from_str(&to_address.to_string())?) => { - return Ok(jumpdest_table) - } + Some(to_address) if is_precompile(to_address.compat()) => return Ok(jumpdest_table), Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { return Ok(jumpdest_table) } @@ -173,17 +171,8 @@ pub(crate) fn generate_jumpdest_table<'a>( ctx, entry.pc, op, - // ?entry, ); - // trace!("TX: {:?}", tx.hash); - // trace!("STEP: {:?}", step); - // trace!("STEPS: {:?}", struct_log.len()); - // trace!("OPCODE: {}", entry.op.as_str()); - // trace!("CODE: {:?}", code_hash); - // trace!("CTX: {:?}", ctx); - // trace!("CURR_DEPTH: {:?}", curr_depth); - // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { @@ -209,34 +198,19 @@ pub(crate) fn generate_jumpdest_table<'a>( unreachable!() }; - // if *address > U256::from(U160::MAX) { - // trace!( - // "{op}: Callee address {} was larger than possible {}.", - // *address, - // U256::from(U160::MAX) - // ); - // // Se note above. - // continue; - // }; - let a: [u8; 32] = address.compat().into(); - // a <<= 96; - // a >>= 96; - // let aa: [u8; 32] = a.into(); - let aaa: H256 = a.into(); - let lower_20_bytes: [u8; 20] = H160::from(aaa).into(); - let callee_address = Address::from(lower_20_bytes); + let callee_address = stack_value_to_address(address); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; call_stack.push((next_code_hash, next_ctx_available)); }; - if is_precompile(H160::from_str(&callee_address.to_string())?) { + if is_precompile(callee_address.compat()) { trace!("Called precompile at address {}.", &callee_address); }; if callee_addr_to_code_hash.contains_key(&callee_address).not() - && is_precompile(H160::from_str(&callee_address.to_string())?).not() + && is_precompile(callee_address.compat()).not() { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun @@ -329,6 +303,13 @@ pub(crate) fn generate_jumpdest_table<'a>( Ok(jumpdest_table) } +fn stack_value_to_address(operand: &Uint<256, 4>) -> Address { + let all_bytes: [u8; 32] = operand.compat().into(); + let mut lower_20_bytes = [0u8; 20]; + lower_20_bytes[0..20].copy_from_slice(&all_bytes[..]); + Address::from(lower_20_bytes) +} + fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option { match trace { GethTrace::Default(structlog_frame) => { diff --git a/zero/src/tracing.rs b/zero/src/tracing.rs index 20d376d86..6c3f9a8bc 100644 --- a/zero/src/tracing.rs +++ b/zero/src/tracing.rs @@ -5,6 +5,7 @@ pub fn init() { .with( tracing_subscriber::fmt::layer() .with_ansi(false) + .compact() .with_filter(EnvFilter::from_default_env()), ) .init(); From 4707d3853067d1da0329cdc529514185b168f200 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 14:34:37 +0200 Subject: [PATCH 064/107] fix: addresses --- evm_arithmetization/src/generation/prover_input.rs | 2 +- zero/src/rpc/jumpdest.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 1e4090aa8..3a50edf2f 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -394,7 +394,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { Ok(next_jumpdest_proof.into()) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 1cac95203..01543b23b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -199,6 +199,8 @@ pub(crate) fn generate_jumpdest_table<'a>( }; let callee_address = stack_value_to_address(address); + let callee_address_old = stack_value_to_address_old(address); + assert_eq!(callee_address, callee_address_old); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; @@ -306,7 +308,17 @@ pub(crate) fn generate_jumpdest_table<'a>( fn stack_value_to_address(operand: &Uint<256, 4>) -> Address { let all_bytes: [u8; 32] = operand.compat().into(); let mut lower_20_bytes = [0u8; 20]; - lower_20_bytes[0..20].copy_from_slice(&all_bytes[..]); + // Based on `__compat_primitive_types::H160::from(H256::from(all_bytes)). + // into()`. + lower_20_bytes[0..20].copy_from_slice(&all_bytes[32 - 20..32]); + Address::from(lower_20_bytes) +} + +// Review: to be removed +fn stack_value_to_address_old(operand: &Uint<256, 4>) -> Address { + let all_bytes: [u8; 32] = operand.compat().into(); + let aaa: H256 = all_bytes.into(); + let lower_20_bytes: [u8; 20] = __compat_primitive_types::H160::from(aaa).into(); Address::from(lower_20_bytes) } From 484350105dda5be922e7f93e09dcf2ad3fccc66f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 14:58:31 +0200 Subject: [PATCH 065/107] todo: fix todo --- trace_decoder/src/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 17de5e6a1..6445dbb29 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -184,7 +184,7 @@ pub fn entrypoint( block_hashes: b_hashes.clone(), burn_addr, jumpdest_table: { - // TODO See the issue Simulate to get jumpdests on a per-transaction basis #653. + // TODO(einar-polygon): // Note that this causes any batch containing just a single `None` to collapse // into a `None`, which causing failover to simulating jumpdest analysis for the // whole batch. There is an optimization opportunity here. From 8f980d212f0101bd7cd78f12864c373b7fcaa0d9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 15:14:36 +0200 Subject: [PATCH 066/107] testing: improve prove_stdio script --- scripts/prove_stdio.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index d5496c63f..f0db105a8 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -50,6 +50,11 @@ if [[ $INPUT_FILE == "" ]]; then exit 1 fi +if [[ ! -s $INPUT_FILE ]]; then + echo "Input file $INPUT_FILE does not exist or has length 0." + exit 1 +fi + # Circuit sizes only matter in non test_only mode. if ! [[ $TEST_ONLY == "test_only" ]]; then if [[ $INPUT_FILE == *"witness_b19807080"* ]]; then @@ -98,30 +103,35 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH + nice -19 cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" #rm $TEST_OUT_PATH exit 0 + elif grep -q 'Attempted to collapse an extension node' $TEST_OUT_PATH; then + echo "ERROR: Attempted to collapse an extension node. See $TEST_OUT_PATH for more details." + exit 4 + elif grep -q 'SIMW == RPCW ? false' $TEST_OUT_PATH; then + echo "ERROR: SIMW == RPCW ? false. See $TEST_OUT_PATH for more details." + exit 5 elif grep -q 'Proving task finished with error' $TEST_OUT_PATH; then # Some error occurred, display the logs and exit. - cat $OUT_LOG_PATH - echo "Failed to create proof witnesses. See $OUT_LOG_PATH for more details." + echo "ERROR: Proving task finished with error. See $TEST_OUT_PATH for more details." exit 1 else - echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See \"zk_evm/test.out\" for more details." + echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See $TEST_OUT_PATH for more details." exit 2 fi fi cargo build --release --jobs "$num_procs" + start_time=$(date +%s%N) -"${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ +nice -19 "${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $OUTPUT_LOG end_time=$(date +%s%N) -set +o pipefail cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST if [ ! -s "$PROOFS_FILE_LIST" ]; then # Some error occurred, display the logs and exit. @@ -134,7 +144,7 @@ cat $PROOFS_FILE_LIST | while read proof_file; do echo "Verifying proof file $proof_file" verify_file=$PROOF_OUTPUT_DIR/verify_$(basename $proof_file).out - "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file + nice -19 "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file if grep -q 'All proofs verified successfully!' $verify_file; then echo "Proof verification for file $proof_file successful"; rm $verify_file # we keep the generated proof for potential reuse From ee7e5f303a18b67b68de0e70c506921025dcffde Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 15:15:44 +0200 Subject: [PATCH 067/107] testing: improve test_native script --- scripts/test_native.sh | 104 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 8 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index eb65b5758..3093735b8 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -48,7 +48,94 @@ ROUND2=" 20727650 " +ROUND3=" +20727643 +20727644 +20727648 +20727649 +20727650 +" + +ROUND4=" +19457111 +19477724 +19501672 +19508907 +19511272 +19548904 +19550401 +19553425 +19563122 +19585193 +19600168 +19603017 +19607029 +19649976 +19654474 +19657021 +19670735 +19688239 +19737540 +19767306 +19792995 +19812505 +19829370 +19835094 +19862390 +19871215 +19877263 +19877279 +19893964 +19922838 +19938970 +19971875 +20011069 +20071977 +20131326 +20173673 +20182890 +20218660 +20225259 +20229861 +20259823 +20274215 +20288828 +20291090 +20301243 +20346949 +20410573 +20462322 +20518465 +20521004 +20542632 +20543651 +20555951 +20634148 +20691605 +20714397 +20715309 +20715461 +20719386 +20720179 +20720275 +20741147 +20775888 +20804319 +20835783 +20859523 +20727643 +20727644 +20727648 +20727649 +20727650 +" +ROUND5=" +19650385 +19542391 +19578175 +19511272 +" CANCUN=19426587 @@ -60,27 +147,28 @@ RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" -BLOCKS="20727641" -#BLOCKS="$CANCUNBLOCKS" +#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" +#BLOCKS="19511272" +BLOCKS=$RANDOMBLOCKS +BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "\n\ngithash block verdict duration\n" | tee -a witnesses/native_results.txt -echo "------------------------------------" | tee -a witnesses/native_results.txt +printf "\n\ngithash block verdict r duration\n" | tee -a witnesses/native_results.txt +echo "----------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS rg "jump" $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 - timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK + timeout 10m nice -19 ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION @@ -90,5 +178,5 @@ for BLOCK in $BLOCKS; do else VERDICT="failure" fi - printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/native_results.txt + printf "%s %10i %s %3i %s\n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION | tee -a witnesses/native_results.txt done From 5451399e4f1866e3fd2d0af1a6c87d4cfd7e5cef Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 16:51:30 +0200 Subject: [PATCH 068/107] fmt --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f3f368650..66050ed4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,4 +123,4 @@ plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } [workspace.lints.clippy] -too_long_first_doc_paragraph = "allow" \ No newline at end of file +too_long_first_doc_paragraph = "allow" From e9a8702a64ee49020f9085f4f3e5a7e16a83bc32 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 17:50:41 +0200 Subject: [PATCH 069/107] Round 5 --- .cargo/config.toml | 13 ++++++++++++- Cargo.toml | 24 ++++++++++++++++++++++++ scripts/prove_stdio.sh | 2 +- scripts/test_native.sh | 2 +- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6340ce34a..526c1c9b4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,15 @@ [build] # https://github.com/rust-lang/rust/pull/124129 # https://github.com/dtolnay/linkme/pull/88 -rustflags = ["-Z", "linker-features=-lld"] + +[env] +RUST_BACKTRACE = "1" +RUST_TEST_NOCAPTURE = "1" + +[term] +verbose = true +color = 'auto' + +[target.x86_64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-Z", "linker-features=-lld", "-C", "target-cpu=native"] #, "-C", "link-arg=-fuse-ld=/usr/bin/mold", "-C", "debuginfo=2"] diff --git a/Cargo.toml b/Cargo.toml index 66050ed4b..5fb943015 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,3 +124,27 @@ starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" + +[profile.release] +opt-level = 3 +debug = true +incremental = true +debug-assertions = true +lto = false +overflow-checks = false + +[profile.test] +opt-level = 3 +debug = true +incremental = true +debug-assertions = true +lto = false +overflow-checks = false + +[profile.dev] +opt-level = 3 +debug = true +incremental = true +debug-assertions = true +lto = false +overflow-checks = false diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 48e5ecf6a..18ef39b40 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -1,6 +1,6 @@ #!/bin/bash # ------------------------------------------------------------------------------ -set -exo pipefail +set -ex # Run prover with the parsed input from the standard terminal. # To generate the json input file, use the `rpc` tool, for example: diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 3093735b8..d08446d4d 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -149,7 +149,7 @@ echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" #BLOCKS="19511272" -BLOCKS=$RANDOMBLOCKS +BLOCKS=$ROUND5 BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" From b2f66edeedac265b1a896f0e47c4aeca5c35e477 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 15 Oct 2024 09:23:51 +0200 Subject: [PATCH 070/107] testing --- scripts/prove_stdio.sh | 1 + scripts/test_native.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 18ef39b40..98daf1098 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -110,6 +110,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then exit 0 elif grep -q 'Attempted to collapse an extension node' $TEST_OUT_PATH; then echo "ERROR: Attempted to collapse an extension node. See $TEST_OUT_PATH for more details." + rm $TEST_OUT_PATH exit 4 elif grep -q 'SIMW == RPCW ? false' $TEST_OUT_PATH; then echo "ERROR: SIMW == RPCW ? false. See $TEST_OUT_PATH for more details." diff --git a/scripts/test_native.sh b/scripts/test_native.sh index d08446d4d..fb97f767b 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -141,7 +141,7 @@ ROUND5=" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 -NUMRANDOMBLOCKS=100 +NUMRANDOMBLOCKS=1000 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` @@ -149,7 +149,7 @@ echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" #BLOCKS="19511272" -BLOCKS=$ROUND5 +BLOCKS="$RANDOMBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" From cfb293c8bb3d4ec0b9e9f3bb0e00881311959d74 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 15 Oct 2024 20:02:33 +0200 Subject: [PATCH 071/107] testing: improve reporting, add error cases --- evm_arithmetization/src/generation/state.rs | 4 +- scripts/test_native.sh | 87 +++++++++++++++++++-- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index d44b7dea4..d8cc3e93b 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -486,12 +486,12 @@ impl GenerationState { // We cannot observe anything as the stack is empty. return Ok(()); } - if dst == KERNEL.global_labels["observe_new_address"] { + if dst == KERNEL.global_labels["observe_new_address"] && self.is_kernel() { let tip_u256 = stack_peek(self, 0)?; let tip_h256 = H256::from_uint(&tip_u256); let tip_h160 = H160::from(tip_h256); self.observe_address(tip_h160); - } else if dst == KERNEL.global_labels["observe_new_contract"] { + } else if dst == KERNEL.global_labels["observe_new_contract"] && self.is_kernel() { let tip_u256 = stack_peek(self, 0)?; let tip_h256 = H256::from_uint(&tip_u256); self.observe_contract(tip_h256)?; diff --git a/scripts/test_native.sh b/scripts/test_native.sh index fb97f767b..6a6b6dabe 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -137,6 +137,71 @@ ROUND5=" 19511272 " +ROUND6=" +19426872 +19427018 +19427388 +19427472 +19429634 +19430273 +19430687 +19430855 +19431223 +19431344 +19432360 +19432641 +19435607 +19435804 +19436307 +19439155 +19439754 +19440665 +19441789 +19443628 +19443673 +19444327 +19444582 +19445175 +19445286 +19445799 +19446774 +19446911 +19447598 +19447814 +19448687 +19449229 +19449755 +19450491 +19451118 +19451955 +19452325 +19452532 +19452795 +19452869 +19454136 +19455621 +19456052 +19456615 +19460281 +19460945 +19462377 +19463186 +19464727 +19466034 +19466036 +19466108 +19466509 +" + +ROUND7=" +19430273 +19431344 +19451118 +19452869 +19460945 +19464727 +19466034 +" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` @@ -149,34 +214,40 @@ echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" #BLOCKS="19511272" -BLOCKS="$RANDOMBLOCKS" +BLOCKS="$ROUND7" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "\n\ngithash block verdict r duration\n" | tee -a witnesses/native_results.txt -echo "----------------------------------------" | tee -a witnesses/native_results.txt +printf "\n\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt +echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do + TOTALTIME=0 GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace + SECONDS=0 nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - rg "jump" $WITNESS - echo "Testing blocks: $BLOCKS." + TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` + DURATION_RPC=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` + TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` + TXOK=`echo -n $(($TXALL - $TXNONE))` echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 timeout 10m nice -19 ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? - DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` - echo $DURATION + TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` + DURATION_PRV=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + TOTALTIME=`date -u -d @"$TOTALTIME" +'%-Hh%-Mm%-Ss'` if [ $EXITCODE -eq 0 ] then VERDICT="success" else VERDICT="failure" fi - printf "%s %10i %s %3i %s\n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION | tee -a witnesses/native_results.txt + printf "%s %10i %s %3i %8s %8s %8s %3i %3i %3i \n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION_RPC $DURATION_PRV $TOTALTIME $TXOK $TXNONE $TXALL | tee -a witnesses/native_results.txt done From 3c497cc9b5dfc506e900f835b511f6149f4b2c7e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 15 Oct 2024 21:15:14 +0200 Subject: [PATCH 072/107] change exit code --- scripts/prove_stdio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 98daf1098..5ee39f296 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -52,7 +52,7 @@ fi if [[ ! -s $INPUT_FILE ]]; then echo "Input file $INPUT_FILE does not exist or has length 0." - exit 1 + exit 6 fi # Circuit sizes only matter in non test_only mode. From 2dc52cbc4c2671cd3656e2ea5a033da4ed52786c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 16 Oct 2024 16:17:33 +0200 Subject: [PATCH 073/107] don't panic! --- evm_arithmetization/src/generation/prover_input.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 3a50edf2f..bfbea41b2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -359,10 +359,10 @@ impl GenerationState { } /// Returns the next used jump address. - /// todo fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // TODO(einar-polygon) // get_code from self.memory if self.jumpdest_table.is_none() { @@ -824,7 +824,7 @@ impl GenerationState { } info!("SIMW == RPCW ? {}", simw == &rpcw); info!("tx: {:?}", self.inputs.txn_hashes); - panic!(); + // panic!(); // info!("SIMP {:?}", &simp); // info!("RPCP {:?}", &rpcp); // info!("SIMP == RPCP ? {}", &simp == &rpcp); From dd89251d10421646f9ae3dbf18cc0d5e872e74aa Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 18 Oct 2024 13:00:00 +0200 Subject: [PATCH 074/107] fix type 5 errors --- evm_arithmetization/src/witness/transition.rs | 16 +++++----- scripts/prove_stdio.sh | 2 +- scripts/test_native.sh | 31 +++++++++++++++---- zero/src/rpc/jumpdest.rs | 5 ++- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/evm_arithmetization/src/witness/transition.rs b/evm_arithmetization/src/witness/transition.rs index fdcf9af65..8a6f1d39a 100644 --- a/evm_arithmetization/src/witness/transition.rs +++ b/evm_arithmetization/src/witness/transition.rs @@ -345,14 +345,6 @@ where where Self: Sized, { - self.perform_op(op, row)?; - self.incr_pc(match op { - Operation::Syscall(_, _, _) | Operation::ExitKernel => 0, - Operation::Push(n) => n as usize + 1, - Operation::Jump | Operation::Jumpi => 0, - _ => 1, - }); - self.incr_gas(gas_to_charge(op)); let registers = self.get_registers(); let gas_limit_address = MemoryAddress::new( @@ -373,6 +365,14 @@ where } } + self.perform_op(op, row)?; + self.incr_pc(match op { + Operation::Syscall(_, _, _) | Operation::ExitKernel => 0, + Operation::Push(n) => n as usize + 1, + Operation::Jump | Operation::Jumpi => 0, + _ => 1, + }); + Ok(op) } diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 5ee39f296..1e432c8f7 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -1,6 +1,6 @@ #!/bin/bash # ------------------------------------------------------------------------------ -set -ex +set -x # Run prover with the parsed input from the standard terminal. # To generate the json input file, use the `rpc` tool, for example: diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 6a6b6dabe..142687cfa 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -203,24 +203,41 @@ ROUND7=" 19466034 " +ROUND8=" +19657436 +19508991 +19500774 +19794433 +" + CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` -STATICTIP=20721266 +STATICTIP=20978815 NUMRANDOMBLOCKS=1000 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` +REPO_ROOT=$(git rev-parse --show-toplevel) + GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" -#BLOCKS="19511272" -BLOCKS="$ROUND7" +#BLOCKS="$RANDOMBLOCKS" +BLOCKS="$ROUND8" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "\n\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt -echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt + +printf "\n\nr\n" | tee -a witnesses/native_results.txt +echo "0 is success" | tee -a witnesses/native_results.txt +echo "5 [defect] is non-matching jumpdest tables" | tee -a witnesses/native_results.txt +echo "1 [unexpected] is other errors" | tee -a witnesses/native_results.txt +echo "4 [expected] is Attempted to collapse an extension node" | tee -a witnesses/native_results.txt +echo "6 [expected] is empty witness. Usually due to Error: Failed to get proof for account" | tee -a witnesses/native_results.txt +echo "Report started: $(date)" | tee -a witnesses/native_results.txt +printf "\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt +echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do TOTALTIME=0 @@ -229,12 +246,14 @@ for BLOCK in $BLOCKS; do echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace SECONDS=0 - nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs --timeout 600 fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` DURATION_RPC=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` TXOK=`echo -n $(($TXALL - $TXNONE))` + TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" + #rm $TEST_OUT_PATH echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 01543b23b..dcd2d9f81 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -169,7 +169,10 @@ pub(crate) fn generate_jumpdest_table<'a>( tx_hash = ?tx.hash, ?code_hash, ctx, - entry.pc, + pc = entry.pc, + pc_hex = format!("{:08x?}", entry.pc), + gas = entry.gas, + gas_cost = entry.gas_cost, op, ?entry, ); From 6c59c41d99aee7ab8cebce796bbe97713b88a197 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sat, 19 Oct 2024 02:20:25 +0200 Subject: [PATCH 075/107] Fix: 19548491 --- zero/src/rpc/jumpdest.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index dcd2d9f81..368c7e760 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -94,7 +94,7 @@ where /// using a Geth structlog as input. pub(crate) fn generate_jumpdest_table<'a>( tx: &Transaction, - struct_log: &[StructLog], + structlog: &[StructLog], tx_traces: impl Iterator, ) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); @@ -146,7 +146,8 @@ pub(crate) fn generate_jumpdest_table<'a>( let mut call_stack = vec![(entrypoint_code_hash, next_ctx_available)]; next_ctx_available += 1; - for (step, entry) in struct_log.iter().enumerate() { + let mut stuctlog_iter = structlog.iter().enumerate().peekable(); + while let Some((step, entry)) = stuctlog_iter.next() { let op = entry.op.as_str(); let curr_depth: usize = entry.depth.try_into().unwrap(); @@ -169,6 +170,7 @@ pub(crate) fn generate_jumpdest_table<'a>( tx_hash = ?tx.hash, ?code_hash, ctx, + next_ctx_available, pc = entry.pc, pc_hex = format!("{:08x?}", entry.pc), gas = entry.gas, @@ -224,6 +226,19 @@ pub(crate) fn generate_jumpdest_table<'a>( &callee_address ); } + + if let Some((_next_step, next_entry)) = &stuctlog_iter.peek() { + let next_depth: usize = next_entry.depth.try_into().unwrap(); + if next_depth < curr_depth { + // The call caused an exception. Skip over incrementing `next_ctx_available`. + continue; + } + } + // `peek()` only returns `None` if we are at the last entry of + // the Structlog, whether we are on a `CALL` op that throws an + // exception or not. But this is of no consequence to the + // generated Jumpdest table, so we can ignore the case. + next_ctx_available += 1; } "CREATE" | "CREATE2" => { From 0d7f6b7e809221c67837ab6c6d0a48ba03213955 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sat, 19 Oct 2024 21:54:41 +0200 Subject: [PATCH 076/107] add stats --- scripts/test_native.sh | 57 +++++++++++++++++++++++++++++++++++++--- zero/src/rpc/jumpdest.rs | 5 ++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 142687cfa..824001094 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -6,8 +6,51 @@ if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 fi + +if [ git diff --quiet --exit-code HEAD ]; then + exit 1 +fi + + mkdir -p witnesses + + +RESULT_LEN=$(cat witnesses/native_results.txt | wc -l) + + +function statistics() +{ + PREFIX_LEN=$(($RESULT_LEN + 13)) + wc -l witnesses/native_results.txt + cat witnesses/native_results.txt | tail -n +$PREFIX_LEN + + SUMOK=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f8 | paste -s -d+ - | bc) + SUMFAIL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f9 | paste -s -d+ - | bc) + SUMTOTAL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f10 | paste -s -d+ - | bc) + echo $SUMTOTAL + echo $SUMFAIL + echo "Failure rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMFAIL * 100 / $SUMTOTAL))%") + echo "Success rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMOK * 100 / $SUMTOTAL))%") + + ZEROES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "0") + ONES=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "1") + TWOS=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "2") + THREES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "3") + FOURS=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "4") + FIVES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "5") + SIXES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "6") + echo $ZEROES + echo $ONES + echo $TWOS + echo $THREES + echo $FOURS + echo $FIVES + echo $SIXES + echo "good bye" + exit 0 +} +trap statistics INT # EXIT QUIT HUP TERM # Must match the values in prove_stdio.sh or build is dirty. #export RAYON_NUM_THREADS=1 #export TOKIO_WORKER_THREADS=1 @@ -222,8 +265,8 @@ GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" -#BLOCKS="$RANDOMBLOCKS" -BLOCKS="$ROUND8" +BLOCKS="$RANDOMBLOCKS" +#BLOCKS="$ROUND8" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" @@ -252,8 +295,6 @@ for BLOCK in $BLOCKS; do TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` TXOK=`echo -n $(($TXALL - $TXNONE))` - TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" - #rm $TEST_OUT_PATH echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 @@ -269,4 +310,12 @@ for BLOCK in $BLOCKS; do VERDICT="failure" fi printf "%s %10i %s %3i %8s %8s %8s %3i %3i %3i \n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION_RPC $DURATION_PRV $TOTALTIME $TXOK $TXNONE $TXALL | tee -a witnesses/native_results.txt + + + ### Clean up + TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" + rm $TEST_OUT_PATH + rm $WITNESS + done + diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 368c7e760..250513c82 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -227,10 +227,11 @@ pub(crate) fn generate_jumpdest_table<'a>( ); } - if let Some((_next_step, next_entry)) = &stuctlog_iter.peek() { + if let Some((_next_step, next_entry)) = stuctlog_iter.peek() { let next_depth: usize = next_entry.depth.try_into().unwrap(); if next_depth < curr_depth { - // The call caused an exception. Skip over incrementing `next_ctx_available`. + // The call caused an exception. Skip over incrementing + // `next_ctx_available`. continue; } } From b8cf325d655f249c9056c95d507c586419c5a0bd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 11:06:44 +0200 Subject: [PATCH 077/107] dbg --- evm_arithmetization/src/generation/prover_input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index bfbea41b2..b5018395a 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -829,7 +829,7 @@ impl GenerationState { // info!("RPCP {:?}", &rpcp); // info!("SIMP == RPCP ? {}", &simp == &rpcp); } else { - info!("JUMPDEST tables similar"); + info!("JUMPDEST tables are equal."); } self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; From 0e02af5b0393361445bbd60aef163795934e5b55 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:36:23 +0200 Subject: [PATCH 078/107] rename a --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 31857ac61..780ef5716 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -97,7 +97,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - let (a, jdtw) = interpreter + let (jdtp, jdtw) = interpreter .generation_state .get_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); @@ -105,8 +105,8 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - interpreter.generation_state.jumpdest_table = Some(a.clone()); - Some((a, jdtw)) + interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); + Some((jdtp, jdtw)) } } } From d04f357e5711fbd96aefac32f7a80fd948d623e7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:37:47 +0200 Subject: [PATCH 079/107] add derive_more and add docs --- Cargo.lock | 1 + evm_arithmetization/Cargo.toml | 1 + .../src/cpu/kernel/interpreter.rs | 24 +++-- .../src/generation/jumpdest.rs | 97 ++++++++++--------- 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7dc061ad..7ac076699 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2025,6 +2025,7 @@ dependencies = [ "anyhow", "bytes", "criterion", + "derive_more", "env_logger 0.11.5", "ethereum-types", "hashbrown", diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index f5dfec2f2..d131d5bc2 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -17,6 +17,7 @@ keywords.workspace = true [dependencies] anyhow.workspace = true bytes.workspace = true +derive_more = "1.0.0" env_logger.workspace = true ethereum-types.workspace = true hashbrown.workspace = true diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 780ef5716..36a2034ee 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -21,7 +21,7 @@ use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; -use crate::generation::jumpdest::{ContextJumpDests, JumpDestTableProcessed, JumpDestTableWitness}; +use crate::generation::jumpdest::{Context, JumpDestTableProcessed, JumpDestTableWitness}; use crate::generation::linked_list::LinkedListsPtrs; use crate::generation::mpt::{load_linked_lists_and_txn_and_receipt_mpts, TrieRootPtrs}; use crate::generation::prover_input::get_proofs_and_jumpdests; @@ -161,6 +161,10 @@ pub(crate) fn set_registers_and_run( /// /// - `jumpdest_table_rpc`: The raw table received from RPC. /// - `code_db`: The corresponding database of contract code used in the trace. +/// +/// # Output +/// +/// Returns a [`JumpDestTableProccessed`]. pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, @@ -190,15 +194,15 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( /// /// # Arguments /// -/// - `ctx_jumpdests`: Map from `ctx` to its list of offsets to reached -/// `JUMPDEST`s. -/// - `code`: The bytecode for the contexts. This is the same for all contexts. -fn prove_context_jumpdests( - code: &[u8], - ctx_jumpdests: &ContextJumpDests, -) -> HashMap> { - ctx_jumpdests - .0 +/// - `code`: The bytecode for the context `ctx`. +/// - `ctx`: Map from `ctx` to its list of `JUMPDEST` offsets. +/// +/// # Outputs +/// +/// Returns a [`HashMap`] from `ctx` to [`Vec`] of proofs. Each proofs ia a +/// pair. +fn prove_context_jumpdests(code: &[u8], ctx: &Context) -> HashMap> { + ctx.0 .iter() .map(|(&ctx, jumpdests)| { let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 6def65e77..55f401c46 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -26,9 +26,9 @@ use std::cmp::max; use std::{ collections::{BTreeSet, HashMap}, fmt::Display, - ops::{Deref, DerefMut}, }; +use derive_more::derive::{Deref, DerefMut}; use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; @@ -36,18 +36,18 @@ use serde::{Deserialize, Serialize}; /// Each `CodeHash` can be called one or more times, /// each time creating a new `Context`. /// Each `Context` will contain one or more offsets of `JUMPDEST`. -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] -pub struct ContextJumpDests(pub HashMap>); +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] +pub struct Context(pub HashMap>); /// The result after proving a [`JumpDestTableWitness`]. -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] pub(crate) struct JumpDestTableProcessed(HashMap>); /// Map `CodeHash -> (Context -> [JumpDests])` -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] -pub struct JumpDestTableWitness(HashMap); +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] +pub struct JumpDestTableWitness(HashMap); -impl ContextJumpDests { +impl Context { pub fn insert(&mut self, ctx: usize, offset: usize) { self.entry(ctx).or_default().insert(offset); } @@ -64,7 +64,7 @@ impl JumpDestTableProcessed { } impl JumpDestTableWitness { - pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { + pub fn get(&self, code_hash: &H256) -> Option<&Context> { self.0.get(code_hash) } @@ -97,6 +97,7 @@ impl JumpDestTableWitness { } } +// The following Display instances are added to make it easier to read diffs. impl Display for JumpDestTableWitness { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(f, "\n=== JumpDestTableWitness ===")?; @@ -108,7 +109,7 @@ impl Display for JumpDestTableWitness { } } -impl Display for ContextJumpDests { +impl Display for Context { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let v: Vec<_> = self.0.iter().sorted().collect(); for (ctx, offsets) in v.into_iter() { @@ -134,45 +135,13 @@ impl Display for JumpDestTableProcessed { } } -impl Deref for ContextJumpDests { - type Target = HashMap>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for ContextJumpDests { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl Deref for JumpDestTableProcessed { - type Target = HashMap>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for JumpDestTableProcessed { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl Deref for JumpDestTableWitness { - type Target = HashMap; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for JumpDestTableWitness { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 +impl FromIterator<(H256, usize, usize)> for JumpDestTableWitness { + fn from_iter>(iter: T) -> Self { + let mut jdtw = JumpDestTableWitness::default(); + for (code_hash, ctx, offset) in iter.into_iter() { + jdtw.insert(code_hash, ctx, offset); + } + jdtw } } @@ -209,4 +178,36 @@ mod test { assert_eq!(86, max_ctx); assert_eq!(expected, actual) } + + #[test] + fn test_extend_from_iter() { + let code_hash = H256::default(); + + let ctx_map = vec![ + (code_hash, 1, 1), + (code_hash, 2, 2), + (code_hash, 42, 3), + (code_hash, 43, 4), + ]; + let table1 = JumpDestTableWitness::from_iter(ctx_map); + let table2 = table1.clone(); + + let jdts = [&table1, &table2]; + let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); + + let ctx_map_merged = vec![ + (code_hash, 1, 1), + (code_hash, 2, 2), + (code_hash, 42, 3), + (code_hash, 43, 4), + (code_hash, 44, 1), + (code_hash, 45, 2), + (code_hash, 85, 3), + (code_hash, 86, 4), + ]; + let expected = JumpDestTableWitness::from_iter(ctx_map_merged); + + assert_eq!(86, max_ctx); + assert_eq!(expected, actual) + } } From cc1d1d921bd06f7ade474c4e181e4c54de2cdbbf Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 13:21:48 +0200 Subject: [PATCH 080/107] clean up --- .../src/generation/jumpdest.rs | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 55f401c46..5f516e52b 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -151,34 +151,6 @@ mod test { use super::JumpDestTableWitness; - #[test] - fn test_extend() { - let code_hash = H256::default(); - - let mut table1 = JumpDestTableWitness::default(); - table1.insert(code_hash, 1, 1); - table1.insert(code_hash, 2, 2); - table1.insert(code_hash, 42, 3); - table1.insert(code_hash, 43, 4); - let table2 = table1.clone(); - - let jdts = [&table1, &table2]; - let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); - - let mut expected = JumpDestTableWitness::default(); - expected.insert(code_hash, 1, 1); - expected.insert(code_hash, 2, 2); - expected.insert(code_hash, 42, 3); - expected.insert(code_hash, 43, 4); - expected.insert(code_hash, 44, 1); - expected.insert(code_hash, 45, 2); - expected.insert(code_hash, 85, 3); - expected.insert(code_hash, 86, 4); - - assert_eq!(86, max_ctx); - assert_eq!(expected, actual) - } - #[test] fn test_extend_from_iter() { let code_hash = H256::default(); From a6ba0e5844a8da45e6cb4d7a9dd35ca0257dfce7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:37:02 +0200 Subject: [PATCH 081/107] remove todo --- evm_arithmetization/src/generation/prover_input.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index b5018395a..6db978073 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -362,9 +362,6 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; - // TODO(einar-polygon) - // get_code from self.memory - if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } From 1a46a43b1fa2bfe1ff66bdec15097ee3be86546b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:37:47 +0200 Subject: [PATCH 082/107] add derive_more and add docs --- .../src/generation/jumpdest.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 5f516e52b..79d0e942a 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -182,4 +182,36 @@ mod test { assert_eq!(86, max_ctx); assert_eq!(expected, actual) } + + #[test] + fn test_extend_from_iter() { + let code_hash = H256::default(); + + let ctx_map = vec![ + (code_hash, 1, 1), + (code_hash, 2, 2), + (code_hash, 42, 3), + (code_hash, 43, 4), + ]; + let table1 = JumpDestTableWitness::from_iter(ctx_map); + let table2 = table1.clone(); + + let jdts = [&table1, &table2]; + let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); + + let ctx_map_merged = vec![ + (code_hash, 1, 1), + (code_hash, 2, 2), + (code_hash, 42, 3), + (code_hash, 43, 4), + (code_hash, 44, 1), + (code_hash, 45, 2), + (code_hash, 85, 3), + (code_hash, 86, 4), + ]; + let expected = JumpDestTableWitness::from_iter(ctx_map_merged); + + assert_eq!(86, max_ctx); + assert_eq!(expected, actual) + } } From d31530df16aa23197c22c8e434042f86010b1cec Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 13:21:48 +0200 Subject: [PATCH 083/107] clean up --- Cargo.toml | 2 +- .../src/generation/jumpdest.rs | 32 ------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5fb943015..073e92fab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,4 +147,4 @@ debug = true incremental = true debug-assertions = true lto = false -overflow-checks = false +overflow-checks = false \ No newline at end of file diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 79d0e942a..5f516e52b 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -182,36 +182,4 @@ mod test { assert_eq!(86, max_ctx); assert_eq!(expected, actual) } - - #[test] - fn test_extend_from_iter() { - let code_hash = H256::default(); - - let ctx_map = vec![ - (code_hash, 1, 1), - (code_hash, 2, 2), - (code_hash, 42, 3), - (code_hash, 43, 4), - ]; - let table1 = JumpDestTableWitness::from_iter(ctx_map); - let table2 = table1.clone(); - - let jdts = [&table1, &table2]; - let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); - - let ctx_map_merged = vec![ - (code_hash, 1, 1), - (code_hash, 2, 2), - (code_hash, 42, 3), - (code_hash, 43, 4), - (code_hash, 44, 1), - (code_hash, 45, 2), - (code_hash, 85, 3), - (code_hash, 86, 4), - ]; - let expected = JumpDestTableWitness::from_iter(ctx_map_merged); - - assert_eq!(86, max_ctx); - assert_eq!(expected, actual) - } } From 6adcc9bcac483420ac3b03dd87117d902875dd4e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:42:10 +0200 Subject: [PATCH 084/107] use Hash2code --- trace_decoder/src/core.rs | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 6445dbb29..d7b5e19c0 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -143,7 +143,7 @@ pub fn entrypoint( checkpoint_state_trie_root, checkpoint_consolidated_hash, contract_code: { - let initcodes = + let init_codes = byte_code .iter() .filter_map(|nonempty_txn_bytes| -> Option> { @@ -154,31 +154,11 @@ pub fn entrypoint( TxKind::Call(_address) => None, } }); - - // TODO convert to Hash2Code - let initmap: HashMap<_, _> = initcodes - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(); - log::trace!("Initmap {:?}", initmap); - - let contractmap: HashMap<_, _> = contract_code - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(); - log::trace!("Contractmap {:?}", contractmap); - - let codemap: HashMap<_, _> = code_db - .clone() - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(); - log::trace!("Codemap {:?}", codemap); - - let mut res = codemap; - res.extend(contractmap); - res.extend(initmap); - res + let mut result = Hash2Code::default(); + result.extend(init_codes); + result.extend(contract_code); + result.extend(code_db.clone()); + result.into_hashmap() }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -863,6 +843,7 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. +#[derive(Default)] struct Hash2Code { /// Key must always be [`hash`] of value. inner: HashMap>, @@ -886,6 +867,9 @@ impl Hash2Code { pub fn insert(&mut self, code: Vec) { self.inner.insert(keccak_hash::keccak(&code), code); } + pub fn into_hashmap(self) -> HashMap> { + self.inner + } } impl Extend> for Hash2Code { From f39af6e4db8b4da6c8ff25f465a158cf0ab0b374 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:54:05 +0200 Subject: [PATCH 085/107] mv derive_more --- Cargo.toml | 1 + evm_arithmetization/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 073e92fab..fc6f5d27e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ ciborium-io = "0.2.2" clap = { version = "4.5.7", features = ["derive", "env"] } compat = { path = "compat" } criterion = "0.5.1" +derive_more = "1.0.0" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index d131d5bc2..9749eb084 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -17,7 +17,7 @@ keywords.workspace = true [dependencies] anyhow.workspace = true bytes.workspace = true -derive_more = "1.0.0" +derive_more.workspace = true env_logger.workspace = true ethereum-types.workspace = true hashbrown.workspace = true From 0b9bf0adb8b5131202c12ba091a32e58901c4822 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:57:53 +0200 Subject: [PATCH 086/107] cleanup --- zero/src/rpc/jumpdest.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 250513c82..7f323551a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -204,9 +204,6 @@ pub(crate) fn generate_jumpdest_table<'a>( }; let callee_address = stack_value_to_address(address); - let callee_address_old = stack_value_to_address_old(address); - assert_eq!(callee_address, callee_address_old); - if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; call_stack.push((next_code_hash, next_ctx_available)); @@ -333,14 +330,6 @@ fn stack_value_to_address(operand: &Uint<256, 4>) -> Address { Address::from(lower_20_bytes) } -// Review: to be removed -fn stack_value_to_address_old(operand: &Uint<256, 4>) -> Address { - let all_bytes: [u8; 32] = operand.compat().into(); - let aaa: H256 = all_bytes.into(); - let lower_20_bytes: [u8; 20] = __compat_primitive_types::H160::from(aaa).into(); - Address::from(lower_20_bytes) -} - fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option { match trace { GethTrace::Default(structlog_frame) => { From d1e6efae6c3074a029bcd34a1d7422246b775ac6 Mon Sep 17 00:00:00 2001 From: Sai <135601871+sai-deng@users.noreply.github.com> Date: Tue, 15 Oct 2024 22:20:34 +0000 Subject: [PATCH 087/107] Optimize zkVM Proving by Skipping Unused Keccak Tables (#690) This PR introduces optional Keccak table proving, which optimizes performance by skipping the proving process when the Keccak tables are empty. This change reduces unnecessary computation and requires an update to Plonky2 for full integration. --- Cargo.lock | 14 +- Cargo.toml | 6 +- .../src/fixed_recursive_verifier.rs | 247 +++++++++--------- evm_arithmetization/src/get_challenges.rs | 29 +- evm_arithmetization/src/proof.rs | 27 +- evm_arithmetization/src/prover.rs | 42 +-- evm_arithmetization/src/testing_utils.rs | 27 +- evm_arithmetization/src/verifier.rs | 73 ++++-- evm_arithmetization/tests/empty_tables.rs | 80 ++++++ zero/src/prover_state/mod.rs | 55 ++-- 10 files changed, 373 insertions(+), 227 deletions(-) create mode 100644 evm_arithmetization/tests/empty_tables.rs diff --git a/Cargo.lock b/Cargo.lock index 7ac076699..1b9ebaff6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3634,7 +3634,7 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "ahash", "anyhow", @@ -3645,7 +3645,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", "plonky2_util", "rand", "rand_chacha", @@ -3658,7 +3658,7 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3682,7 +3682,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "rayon", ] @@ -3690,7 +3690,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" [[package]] name = "plotters" @@ -4651,7 +4651,7 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed#dc77c77f2b06500e16ad4d7f1c2b057903602eed" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" dependencies = [ "ahash", "anyhow", @@ -4660,7 +4660,7 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=dc77c77f2b06500e16ad4d7f1c2b057903602eed)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", "plonky2_util", ] diff --git a/Cargo.toml b/Cargo.toml index fc6f5d27e..13906c6a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,10 +118,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } -starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } +plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 0d7bb82dd..96ecce6d3 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -131,6 +131,8 @@ where /// Holds chains of circuits for each table and for each initial /// `degree_bits`. pub by_table: [RecursiveCircuitsForTable; NUM_TABLES], + /// Dummy proofs of each table for the root circuit. + pub table_dummy_proofs: [Option>; NUM_TABLES], } /// Data for the EVM root circuit, which is used to combine each STARK's shrunk @@ -569,6 +571,43 @@ where } } +/// A struct that encapsulates both the init degree and the shrunk proof. +#[derive(Eq, PartialEq, Debug)] +pub struct ShrunkProofData, C: GenericConfig, const D: usize> +{ + /// The [`CommonCircuitData`] of the last shrinking circuit. + pub common_circuit_data: CommonCircuitData, + + /// The proof after applying shrinking recursion. + pub proof: ProofWithPublicInputs, +} + +impl, C: GenericConfig, const D: usize> + ShrunkProofData +{ + fn to_buffer( + &self, + buffer: &mut Vec, + gate_serializer: &dyn GateSerializer, + ) -> IoResult<()> { + buffer.write_common_circuit_data(&self.common_circuit_data, gate_serializer)?; + buffer.write_proof_with_public_inputs(&self.proof)?; + Ok(()) + } + + fn from_buffer( + buffer: &mut Buffer, + gate_serializer: &dyn GateSerializer, + ) -> IoResult { + let common_circuit_data = buffer.read_common_circuit_data(gate_serializer)?; + let proof = buffer.read_proof_with_public_inputs(&common_circuit_data)?; + Ok(Self { + common_circuit_data, + proof, + }) + } +} + impl AllRecursiveCircuits where F: RichField + Extendable, @@ -612,6 +651,15 @@ where table.to_buffer(&mut buffer, gate_serializer, generator_serializer)?; } } + for table in &self.table_dummy_proofs { + match table { + Some(dummy_proof_data) => { + buffer.write_bool(true)?; + dummy_proof_data.to_buffer(&mut buffer, gate_serializer)? + } + None => buffer.write_bool(false)?, + } + } Ok(buffer) } @@ -690,6 +738,14 @@ where } }; + let table_dummy_proofs = core::array::from_fn(|_| { + if buffer.read_bool().ok()? { + Some(ShrunkProofData::from_buffer(&mut buffer, gate_serializer).ok()?) + } else { + None + } + }); + Ok(Self { root, segment_aggregation, @@ -698,6 +754,7 @@ where block_wrapper, two_to_one_block, by_table, + table_dummy_proofs, }) } @@ -774,6 +831,33 @@ where let block_wrapper = Self::create_block_wrapper_circuit(&block); let two_to_one_block = Self::create_two_to_one_block_circuit(&block_wrapper); + // TODO(sdeng): enable more optional Tables + let table_dummy_proofs = core::array::from_fn(|i| { + if KECCAK_TABLES_INDICES.contains(&i) { + let init_degree = degree_bits_ranges[i].start; + let common_circuit_data = by_table[i] + .by_stark_size + .get(&init_degree) + .expect("Unable to get the shrinking circuits") + .shrinking_wrappers + .last() + .expect("Unable to get the last shrinking circuit") + .circuit + .common + .clone(); + let dummy_circuit: CircuitData = dummy_circuit(&common_circuit_data); + let dummy_pis = HashMap::new(); + let proof = dummy_proof(&dummy_circuit, dummy_pis) + .expect("Unable to generate dummy proofs"); + Some(ShrunkProofData { + common_circuit_data, + proof, + }) + } else { + None + } + }); + Self { root, segment_aggregation, @@ -782,6 +866,7 @@ where block_wrapper, two_to_one_block, by_table, + table_dummy_proofs, } } @@ -1903,7 +1988,6 @@ where abort_signal: Option>, ) -> anyhow::Result> { features_check(&generation_inputs); - let all_proof = prove::( all_stark, config, @@ -1912,37 +1996,32 @@ where timing, abort_signal.clone(), )?; + self.prove_segment_with_all_proofs(&all_proof, config, abort_signal.clone()) + } + pub fn prove_segment_with_all_proofs( + &self, + all_proof: &AllProof, + config: &StarkConfig, + abort_signal: Option>, + ) -> anyhow::Result> { let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - // generate and set a dummy `index_verifier_data` and `proof_with_pis` - let index_verifier_data = - table_circuits.by_stark_size.keys().min().ok_or_else(|| { - anyhow::format_err!("No valid size in shrinking circuits") - })?; - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_usize(*index_verifier_data), + let dummy_proof_data = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof_data.proof, ); - let table_circuit = table_circuits - .by_stark_size - .get(index_verifier_data) - .ok_or_else(|| anyhow::format_err!("No valid size in shrinking circuits"))? - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))?; - let dummy_circuit: CircuitData = - dummy_circuit(&table_circuit.circuit.common); - let dummy_pis = HashMap::new(); - let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .expect("Unable to generate dummy proofs"); - root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; + let stark_proof = &all_proof.multi_proof.stark_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; let original_degree_bits = stark_proof.proof.recover_degree_bits(config); let shrunk_proof = table_circuits .by_stark_size @@ -1993,7 +2072,7 @@ where is_agg: false, is_dummy: false, proof_with_pvs: ProofWithPublicValues { - public_values: all_proof.public_values, + public_values: all_proof.public_values.clone(), intern: root_proof, }, }) @@ -2052,40 +2131,34 @@ where pub fn prove_segment_after_initial_stark( &self, all_proof: AllProof, - table_circuits: &[(RecursiveCircuitsForTableSize, u8); NUM_TABLES], + table_circuits: &[Option<(RecursiveCircuitsForTableSize, u8)>; NUM_TABLES], abort_signal: Option>, ) -> anyhow::Result> { let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { - let (table_circuit, index_verifier_data) = &table_circuits[table]; if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - root_inputs.set_target( - self.root.index_verifier_data[table], - F::from_canonical_u8(*index_verifier_data), + let dummy_proof = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof.proof, ); - // generate and set a dummy `proof_with_pis` - let common_data = &table_circuit - .shrinking_wrappers - .last() - .ok_or_else(|| anyhow::format_err!("No shrinking circuits"))? - .circuit - .common; - let dummy_circuit: CircuitData = dummy_circuit(common_data); - let dummy_pis = HashMap::new(); - let dummy_proof = dummy_proof(&dummy_circuit, dummy_pis) - .expect("Unable to generate dummy proofs"); - root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &dummy_proof); } else { - let stark_proof = &all_proof.multi_proof.stark_proofs[table]; - - let shrunk_proof = - table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; + let (table_circuit, index_verifier_data) = &table_circuits[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get circuits"))?; root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), ); + let stark_proof = all_proof.multi_proof.stark_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; + let shrunk_proof = + table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; root_inputs .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); } @@ -2736,6 +2809,7 @@ where agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof); } } + /// A map between initial degree sizes and their associated shrinking recursion /// circuits. #[derive(Eq, PartialEq, Debug)] @@ -2767,6 +2841,7 @@ where buffer.write_usize(size)?; table.to_buffer(buffer, gate_serializer, generator_serializer)?; } + Ok(()) } @@ -2786,6 +2861,7 @@ where )?; by_stark_size.insert(key, table); } + Ok(Self { by_stark_size }) } @@ -2810,6 +2886,7 @@ where ) }) .collect(); + Self { by_stark_size } } @@ -3131,79 +3208,3 @@ pub mod testing { } } } - -#[cfg(test)] -#[cfg(not(feature = "cdk_erigon"))] -mod tests { - use plonky2::field::goldilocks_field::GoldilocksField; - use plonky2::plonk::config::PoseidonGoldilocksConfig; - use plonky2::timed; - - use super::*; - use crate::testing_utils::{empty_payload, init_logger}; - use crate::witness::operation::Operation; - - type F = GoldilocksField; - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - - #[test] - fn test_segment_proof_generation_without_keccak() -> anyhow::Result<()> { - init_logger(); - - let all_stark = AllStark::::default(); - let config = StarkConfig::standard_fast_config(); - - // Generate a dummy payload for testing - let payload = empty_payload()?; - let max_cpu_len_log = Some(7); - let mut segment_iterator = SegmentDataIterator::::new(&payload, max_cpu_len_log); - let (_, mut segment_data) = segment_iterator.next().unwrap()?; - - let opcode_counts = &segment_data.opcode_counts; - assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); - - let timing = &mut TimingTree::new( - "Segment Proof Generation Without Keccak Test", - log::Level::Info, - ); - // Process and prove segment - let all_circuits = timed!( - timing, - log::Level::Info, - "Create all recursive circuits", - AllRecursiveCircuits::::new( - &all_stark, - &[16..17, 8..9, 7..8, 4..9, 8..9, 4..7, 16..17, 16..17, 16..17], - &config, - ) - ); - - let segment_proof = timed!( - timing, - log::Level::Info, - "Prove segment", - all_circuits.prove_segment( - &all_stark, - &config, - payload.trim(), - &mut segment_data, - timing, - None, - )? - ); - - // Verify the generated segment proof - timed!( - timing, - log::Level::Info, - "Verify segment proof", - all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? - ); - - // Print timing details - timing.print(); - - Ok(()) - } -} diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index ca1f06c62..c7471471b 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -247,7 +247,7 @@ pub(crate) fn observe_public_values_target< pub mod testing { use plonky2::field::extension::Extendable; - use plonky2::hash::hash_types::RichField; + use plonky2::hash::hash_types::{RichField, NUM_HASH_OUT_ELTS}; use plonky2::iop::challenger::Challenger; use plonky2::plonk::config::GenericConfig; use starky::config::StarkConfig; @@ -278,19 +278,14 @@ pub mod testing { let stark_proofs = &self.multi_proof.stark_proofs; - for (i, proof) in stark_proofs.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { - // Observe zero merkle caps when skipping Keccak tables. - let zero_merkle_cap = proof - .proof - .trace_cap - .flatten() - .iter() - .map(|_| F::ZERO) - .collect::>(); - challenger.observe_elements(&zero_merkle_cap); + for (i, stark_proof) in stark_proofs.iter().enumerate() { + if let Some(stark_proof) = stark_proof { + challenger.observe_cap(&stark_proof.proof.trace_cap); } else { - challenger.observe_cap(&proof.proof.trace_cap); + assert!(KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables); + let zero_cap = + vec![F::ZERO; config.fri_config.num_cap_elements() * NUM_HASH_OUT_ELTS]; + challenger.observe_elements(&zero_cap); } } @@ -301,16 +296,16 @@ pub mod testing { Ok(AllProofChallenges { stark_challenges: core::array::from_fn(|i| { - if KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables { - None - } else { + if let Some(stark_proof) = &stark_proofs[i] { challenger.compact(); - Some(stark_proofs[i].proof.get_challenges( + Some(stark_proof.proof.get_challenges( &mut challenger, Some(&ctl_challenges), true, config, )) + } else { + None } }), ctl_challenges, diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index 304072aed..b0de028af 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -10,7 +10,8 @@ use plonky2::plonk::config::{GenericConfig, GenericHashOut, Hasher}; use plonky2::util::serialization::{Buffer, IoResult, Read, Write}; use serde::{Deserialize, Serialize}; use starky::config::StarkConfig; -use starky::proof::MultiProof; +use starky::lookup::GrandProductChallengeSet; +use starky::proof::StarkProofWithMetadata; use crate::all_stark::NUM_TABLES; use crate::util::{get_h160, get_h256, get_u256, h256_limbs, h2u}; @@ -21,6 +22,22 @@ pub(crate) const DEFAULT_CAP_HEIGHT: usize = 4; /// Number of elements contained in a Merkle cap with default height. pub(crate) const DEFAULT_CAP_LEN: usize = 1 << DEFAULT_CAP_HEIGHT; +/// A combination of STARK proofs for independent statements operating on +/// possibly shared variables, along with Cross-Table Lookup (CTL) challenges to +/// assert consistency of common variables across tables. +#[derive(Debug, Clone)] +pub struct MultiProof< + F: RichField + Extendable, + C: GenericConfig, + const D: usize, + const N: usize, +> { + /// Proofs for all the different STARK modules. + pub stark_proofs: [Option>; N], + /// Cross-table lookup challenges. + pub ctl_challenges: GrandProductChallengeSet, +} + /// A STARK proof for each table, plus some metadata used to create recursive /// wrapper proofs. #[derive(Debug, Clone)] @@ -38,8 +55,12 @@ pub struct AllProof, C: GenericConfig, co impl, C: GenericConfig, const D: usize> AllProof { /// Returns the degree (i.e. the trace length) of each STARK. - pub fn degree_bits(&self, config: &StarkConfig) -> [usize; NUM_TABLES] { - self.multi_proof.recover_degree_bits(config) + pub fn degree_bits(&self, config: &StarkConfig) -> [Option; NUM_TABLES] { + core::array::from_fn(|i| { + self.multi_proof.stark_proofs[i] + .as_ref() + .map(|proof| proof.proof.recover_degree_bits(config)) + }) } } diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index fdc5eca20..4192db6c8 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -16,7 +16,7 @@ use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; use starky::cross_table_lookup::{get_ctl_data, CtlData}; use starky::lookup::GrandProductChallengeSet; -use starky::proof::{MultiProof, StarkProofWithMetadata}; +use starky::proof::StarkProofWithMetadata; use starky::prover::prove_with_commitment; use starky::stark::Stark; @@ -24,7 +24,7 @@ use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; -use crate::proof::{AllProof, MemCap, PublicValues, DEFAULT_CAP_LEN}; +use crate::proof::{AllProof, MemCap, MultiProof, PublicValues, DEFAULT_CAP_LEN}; use crate::GenerationSegmentData; /// Generate traces, then create all STARK proofs. @@ -220,7 +220,7 @@ where } type ProofWithMemCaps = ( - [StarkProofWithMetadata; NUM_TABLES], + [Option>; NUM_TABLES], MerkleCap, MerkleCap, ); @@ -273,16 +273,16 @@ where let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); - let challenger_after_cpu = challenger.clone(); - // TODO(sdeng): Keccak proofs are still required for CTLs, etc. Refactor the - // code and remove the unnecessary parts. - let (keccak_proof, _) = prove_table!(keccak_stark, Table::Keccak); - let (keccak_sponge_proof, _) = prove_table!(keccak_sponge_stark, Table::KeccakSponge); - if !use_keccak_tables { - // We need to connect the challenger state of CPU and Logic tables when the - // Keccak tables are not in use. - *challenger = challenger_after_cpu; - } + let keccak_proof = if use_keccak_tables { + Some(prove_table!(keccak_stark, Table::Keccak).0) + } else { + None + }; + let keccak_sponge_proof = if use_keccak_tables { + Some(prove_table!(keccak_sponge_stark, Table::KeccakSponge).0) + } else { + None + }; let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); @@ -293,17 +293,17 @@ where Ok(( [ - arithmetic_proof, - byte_packing_proof, - cpu_proof, + Some(arithmetic_proof), + Some(byte_packing_proof), + Some(cpu_proof), keccak_proof, keccak_sponge_proof, - logic_proof, - memory_proof, - mem_before_proof, - mem_after_proof, + Some(logic_proof), + Some(memory_proof), + Some(mem_before_proof), + Some(mem_after_proof), #[cfg(feature = "cdk_erigon")] - poseidon_proof, + Some(poseidon_proof), ], mem_before_cap, mem_after_cap, diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 5f1b1b041..039dc2504 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -10,12 +10,17 @@ use mpt_trie::{ nibbles::Nibbles, partial_trie::{HashedPartialTrie, Node, PartialTrie}, }; +use plonky2::field::goldilocks_field::GoldilocksField; pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; -use crate::generation::TrieInputs; +use crate::generation::{TrieInputs, TrimmedGenerationInputs}; use crate::proof::TrieRoots; -use crate::{generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs}; +use crate::witness::operation::Operation; +use crate::{ + generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs, + GenerationSegmentData, SegmentDataIterator, +}; pub const EMPTY_NODE_HASH: H256 = H256(hex!( "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" @@ -165,7 +170,7 @@ pub fn scalable_contract_from_storage(storage_trie: &HashedPartialTrie) -> Accou } } -pub fn empty_payload() -> Result { +fn empty_payload() -> Result { // Set up default block metadata let block_metadata = BlockMetadata { block_beneficiary: Address::zero(), @@ -213,3 +218,19 @@ pub fn empty_payload() -> Result { Ok(inputs) } + +pub fn segment_with_empty_tables() -> Result<( + TrimmedGenerationInputs, + GenerationSegmentData, +)> { + let payload = empty_payload()?; + let max_cpu_len_log = Some(7); + let mut segment_iterator = + SegmentDataIterator::::new(&payload, max_cpu_len_log); + let (trimmed_inputs, segment_data) = segment_iterator.next().unwrap()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + + Ok((trimmed_inputs, segment_data)) +} diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 947516801..81e9b502b 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -85,7 +85,9 @@ pub mod testing { use plonky2::hash::hash_types::RichField; use plonky2::plonk::config::{GenericConfig, GenericHashOut}; use starky::config::StarkConfig; - use starky::cross_table_lookup::{get_ctl_vars_from_proofs, verify_cross_table_lookups}; + use starky::cross_table_lookup::verify_cross_table_lookups; + use starky::cross_table_lookup::CrossTableLookup; + use starky::cross_table_lookup::CtlCheckVars; use starky::lookup::GrandProductChallenge; use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; @@ -184,25 +186,39 @@ pub mod testing { cross_table_lookups, } = all_stark; - let ctl_vars_per_table = get_ctl_vars_from_proofs( - &all_proof.multi_proof, - cross_table_lookups, - &ctl_challenges, - &num_lookup_columns, - all_stark.arithmetic_stark.constraint_degree(), - ); - let stark_proofs = &all_proof.multi_proof.stark_proofs; macro_rules! verify_table { ($stark:ident, $table:expr) => { + let stark_proof = &stark_proofs[*$table] + .as_ref() + .expect("Missing stark_proof") + .proof; + let ctl_vars = { + let (total_num_helpers, _, num_helpers_by_ctl) = + CrossTableLookup::num_ctl_helpers_zs_all( + &all_stark.cross_table_lookups, + *$table, + config.num_challenges, + $stark.constraint_degree(), + ); + CtlCheckVars::from_proof( + *$table, + &stark_proof, + &all_stark.cross_table_lookups, + &ctl_challenges, + num_lookup_columns[*$table], + total_num_helpers, + &num_helpers_by_ctl, + ) + }; verify_stark_proof_with_challenges( $stark, - &stark_proofs[*$table].proof, + stark_proof, &stark_challenges[*$table] .as_ref() .expect("Missing challenges"), - Some(&ctl_vars_per_table[*$table]), + Some(&ctl_vars), &[], config, )?; @@ -240,12 +256,39 @@ pub mod testing { .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) .collect_vec(); + let all_ctls = &all_stark.cross_table_lookups; + verify_cross_table_lookups::( cross_table_lookups, - all_proof - .multi_proof - .stark_proofs - .map(|p| p.proof.openings.ctl_zs_first.unwrap()), + core::array::from_fn(|i| { + if let Some(stark_proof) = &stark_proofs[i] { + stark_proof + .proof + .openings + .ctl_zs_first + .as_ref() + .expect("Missing ctl_zs") + .clone() + } else if i == *Table::Keccak { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::Keccak, + config.num_challenges, + keccak_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else if i == *Table::KeccakSponge { + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + *Table::KeccakSponge, + config.num_challenges, + keccak_sponge_stark.constraint_degree(), + ); + vec![F::ZERO; n] + } else { + panic!("Unable to find stark_proof"); + } + }), Some(&extra_looking_sums), config, ) diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs new file mode 100644 index 000000000..d25901e24 --- /dev/null +++ b/evm_arithmetization/tests/empty_tables.rs @@ -0,0 +1,80 @@ +#![cfg(feature = "eth_mainnet")] + +use evm_arithmetization::fixed_recursive_verifier::AllRecursiveCircuits; +use evm_arithmetization::prover::prove; +use evm_arithmetization::testing_utils::{init_logger, segment_with_empty_tables}; +use evm_arithmetization::verifier::testing::verify_all_proofs; +use evm_arithmetization::AllStark; +use plonky2::field::goldilocks_field::GoldilocksField; +use plonky2::plonk::config::PoseidonGoldilocksConfig; +use plonky2::timed; +use plonky2::util::timing::TimingTree; +use starky::config::StarkConfig; + +/// This test focuses on testing zkVM proofs with some empty tables. +#[test] +fn empty_tables() -> anyhow::Result<()> { + type F = GoldilocksField; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + + init_logger(); + + let all_stark = AllStark::::default(); + let config = StarkConfig::standard_fast_config(); + let timing = &mut TimingTree::new("Empty Table Test", log::Level::Info); + + // Generate segment data + let (payload, mut segment_data) = segment_with_empty_tables()?; + + // Create all STARK proofs + let mut proofs = vec![]; + let proof = timed!( + timing, + log::Level::Info, + "Create all STARK proofs", + prove::( + &all_stark, + &config, + payload, + &mut segment_data, + timing, + None, + )? + ); + proofs.push(proof); + + // Verify the generated STARK proofs + verify_all_proofs(&all_stark, &proofs, &config)?; + + // Process and generate segment proof + let all_circuits = timed!( + timing, + log::Level::Info, + "Create all recursive circuits", + AllRecursiveCircuits::::new( + &all_stark, + &[16..17, 8..9, 7..8, 4..6, 8..9, 4..5, 16..17, 16..17, 16..17], + &config, + ) + ); + let segment_proof = timed!( + timing, + log::Level::Info, + "Prove segment", + all_circuits.prove_segment_with_all_proofs(&proofs[0], &config, None)? + ); + + // Verify the generated segment proof + timed!( + timing, + log::Level::Info, + "Verify segment proof", + all_circuits.verify_root(segment_proof.proof_with_pvs.intern.clone())? + ); + + // Print timing details + timing.print(); + + Ok(()) +} diff --git a/zero/src/prover_state/mod.rs b/zero/src/prover_state/mod.rs index edbad02ba..8cdfb45ce 100644 --- a/zero/src/prover_state/mod.rs +++ b/zero/src/prover_state/mod.rs @@ -174,45 +174,30 @@ impl ProverStateManager { &self, config: &StarkConfig, all_proof: &AllProof, - ) -> anyhow::Result<[(RecursiveCircuitsForTableSize, u8); NUM_TABLES]> { + ) -> anyhow::Result<[Option<(RecursiveCircuitsForTableSize, u8)>; NUM_TABLES]> { let degrees = all_proof.degree_bits(config); - /// Given a recursive circuit index (e.g., Arithmetic / 0), return a - /// tuple containing the loaded table at the specified size and - /// its offset relative to the configured range used to pre-process the - /// circuits. - macro_rules! circuit { - ($circuit_index:expr) => { - ( - RecursiveCircuitResource::get(&( - $circuit_index.into(), - degrees[$circuit_index], + // Given a recursive circuit index (e.g., Arithmetic / 0), return a + // tuple containing the loaded table at the specified size and + // its offset relative to the configured range used to pre-process the + // circuits. + let circuits = core::array::from_fn(|i| match degrees[i] { + Some(size) => RecursiveCircuitResource::get(&(i.into(), size)) + .map(|circuit_resource| { + Some(( + circuit_resource, + (size - self.circuit_config[i].start) as u8, )) - .map_err(|e| { - let circuit: $crate::prover_state::circuit::Circuit = $circuit_index.into(); - let size = degrees[$circuit_index]; - anyhow::Error::from(e).context(format!( - "Attempting to load circuit: {circuit:?} at size: {size}" - )) - })?, - (degrees[$circuit_index] - self.circuit_config[$circuit_index].start) as u8, - ) - }; - } + }) + .map_err(|e| { + anyhow::Error::from(e) + .context(format!("Attempting to load circuit: {i} at size: {size}")) + }) + .unwrap_or(None), + None => None, + }); - Ok([ - circuit!(0), - circuit!(1), - circuit!(2), - circuit!(3), - circuit!(4), - circuit!(5), - circuit!(6), - circuit!(7), - circuit!(8), - #[cfg(feature = "cdk_erigon")] - circuit!(9), - ]) + Ok(circuits) } /// Generate a segment proof using the specified input, loading From a85080d3eea367cc650fa731fd815ea50d9271a4 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Wed, 16 Oct 2024 14:43:14 +0530 Subject: [PATCH 088/107] Assign specific jobs to dedicated workers (#564) * separate workers * fix review comments * add worker run modes * change default queue * update cargo.lock * fix review comments * testing changes * separate segment proof and segment agg proof * review comments * fix review comments * Update zero/src/bin/leader/cli.rs Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> * update paladin version --------- Co-authored-by: Ben Co-authored-by: Robin Salen <30937548+Nashtare@users.noreply.github.com> --- Cargo.lock | 97 ++++++++++++++++------------------- Cargo.toml | 2 +- zero/src/bin/leader.rs | 41 +++++++++++++-- zero/src/bin/leader/cli.rs | 22 +++++++- zero/src/bin/leader/client.rs | 12 +++-- zero/src/bin/leader/http.rs | 16 +++--- zero/src/bin/leader/stdio.rs | 17 ++++-- zero/src/prover.rs | 46 +++++++++++------ 8 files changed, 157 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b9ebaff6..c546a7bcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,9 +58,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37d89f69cb43901949ba29307ada8b9e3b170f94057ad4c04d6fd169d24d65f" +checksum = "b2683873c2744f6cd72d0db51bb74fee9ed310e0476a140bdc19e82b407d8a0a" dependencies = [ "alloy-consensus", "alloy-core", @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4f201b0ac8f81315fbdc55269965a8ddadbc04ab47fa65a1a468f9a40f7a5f" +checksum = "b68b94c159bcc2ca5f758b8663d7b00fc7c5e40569984595ddf2221b0f7f7f6e" dependencies = [ "num_enum", "strum", @@ -96,9 +96,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1468e3128e07c7afe4ff13c17e8170c330d12c322f8924b8bf6986a27e0aad3d" +checksum = "c28ddd17ffb7e4d66ef3a84e7b179072a9320cdc4b26c7f6f44cbf1081631b36" dependencies = [ "alloy-eips", "alloy-primitives", @@ -142,9 +142,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c35df7b972b06f1b2f4e8b7a53328522fa788054a9d3e556faf2411c5a51d5a" +checksum = "2f6c5c0a383f14519531cf58d8440e74f10b938e289f803af870be6f79223110" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8866562186d237f1dfeaf989ef941a24764f764bf5c33311e37ead3519c6a429" +checksum = "7111af869909275cffc5c84d16b6c892d6d512773e40cbe83187d0b9c5235e91" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -174,9 +174,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe714e233f9eaf410de95a9af6bcd05d3a7f8c8de7a0817221e95a6b642a080" +checksum = "342028392a2d5050b7b93dd32a0715d3b3b9ce30072ecb69a35dd4895c005495" dependencies = [ "alloy-consensus", "alloy-eips", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c5a38117974c5776a45e140226745a0b664f79736aa900995d8e4121558e064" +checksum = "a6e66d78c049dcadd065a926a9f2d9a9b2b10981a7889449e694fac7bccd2c6f" dependencies = [ "alloy-eips", "alloy-primitives", @@ -229,9 +229,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65633d6ef83c3626913c004eaf166a6dd50406f724772ea8567135efd6dc5d3" +checksum = "79f14ccc2a3c575cb17b1b4af8c772cf9b5b93b7ce7047d6640e53954abb558d" dependencies = [ "alloy-chains", "alloy-consensus", @@ -286,9 +286,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fc328bb5d440599ba1b5aa44c0b9ab0625fbc3a403bb5ee94ed4a01ba23e07" +checksum = "4dc79aeca84abb122a2fffbc1c91fdf958dca5c95be3875977bc99672bde0027" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -307,9 +307,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f8ff679f94c497a8383f2cd09e2a099266e5f3d5e574bc82b4b379865707dbb" +checksum = "22045187a5ebf5b2af3f8b6831b66735b6556c5750ec5790aeeb45935260c1c2" dependencies = [ "alloy-rpc-types-eth", "alloy-rpc-types-trace", @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a59b1d7c86e0a653e7f3d29954f6de5a2878d8cfd1f010ff93be5c2c48cd3b1" +checksum = "238f494727ff861a803bd73b1274ef788a615bf8f8c4bfada4e6df42afa275d2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -330,17 +330,19 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", + "cfg-if", + "derive_more", + "hashbrown", "itertools 0.13.0", "serde", "serde_json", - "thiserror", ] [[package]] name = "alloy-rpc-types-trace" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54375e5a34ec5a2cf607f9ce98c0ece30dc76ad623afeb25d3953a8d7d30f20" +checksum = "64ca08b0ccc0861055ceb83a1db009c4c8a7f52a259e7cda7ca6ca36ec2b5ce8" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -352,9 +354,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51db8a6428a2159e01b7a43ec7aac801edd0c4db1d4de06f310c288940f16fd3" +checksum = "6b95b6f024a558593dd3b8628af03f7df2ca50e4c56839293ad0a7546e471db0" dependencies = [ "alloy-primitives", "serde", @@ -363,9 +365,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebc1760c13592b7ba3fcd964abba546b8d6a9f10d15e8d92a8263731be33f36" +checksum = "da64740ff0518606c514eb0e03dd0a1daa8ff94d6d491a626fd8e50efd6c4f18" dependencies = [ "alloy-primitives", "async-trait", @@ -435,9 +437,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd5dc4e902f1860d54952446d246ac05386311ad61030a2b906ae865416d36e0" +checksum = "3c7a669caa427abe8802184c8776f5103302f9337bb30a5b36bdebc332946c14" dependencies = [ "alloy-json-rpc", "base64", @@ -454,9 +456,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1742b94bb814f1ca6b322a6f9dd38a0252ff45a3119e40e888fb7029afa500ce" +checksum = "4433ffa97aab6ae643de81c7bde9a2f043496f27368a607405a5c78a610caf74" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -572,9 +574,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.88" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" dependencies = [ "backtrace", ] @@ -3268,9 +3270,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe" [[package]] name = "oorandom" @@ -3358,9 +3360,9 @@ dependencies = [ [[package]] name = "paladin-core" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5af1955eaab1506a43d046628c218b7b3915539554838feb85ed31f54aace2f2" +checksum = "6343e2767eb6ac7f55ca8b869df03c2986b88f0b9ad09cada898b3770ac436e3" dependencies = [ "anyhow", "async-trait", @@ -3378,6 +3380,7 @@ dependencies = [ "paladin-opkind-derive", "pin-project", "postcard", + "rand", "serde", "thiserror", "tokio", @@ -3386,14 +3389,13 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", - "uuid", ] [[package]] name = "paladin-opkind-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25dcb10b7c0ce99abee8694e2e79e4787d7f778b9339dc5a50ba6fc45e5cc9" +checksum = "0bde85d55c108d4eef3404b4c88d1982a270dc11146a085f4f45c1548b4d2b4c" dependencies = [ "quote", "syn 2.0.77", @@ -5357,17 +5359,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" -dependencies = [ - "getrandom", - "rand", - "serde", -] - [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 13906c6a3..6c1191196 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,7 @@ num-bigint = "0.4.5" num-traits = "0.2.19" nunny = "0.2.1" once_cell = "1.19.0" -paladin-core = "0.4.2" +paladin-core = "0.4.3" parking_lot = "0.12.3" paste = "1.0.15" pest = "2.7.10" diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index d1b7c70fd..973a180d4 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -6,10 +6,11 @@ use anyhow::Result; use clap::Parser; use cli::Command; use client::RpcParams; +use paladin::config::Config; use paladin::runtime::Runtime; use tracing::info; use zero::env::load_dotenvy_vars_if_present; -use zero::prover::ProverConfig; +use zero::prover::{ProofRuntime, ProverConfig}; use zero::{ block_interval::BlockInterval, prover_state::persistence::set_circuit_cache_dir_env_if_not_set, }; @@ -24,6 +25,10 @@ mod leader { pub mod stdio; } +const HEAVY_PROOF_ROUTING_KEY: &str = "heavy-proof"; +const LIGHT_PROOF_ROUTING_KEY: &str = "light-proof"; +const DEFAULT_ROUTING_KEY: &str = paladin::runtime::DEFAULT_ROUTING_KEY; + #[tokio::main] async fn main() -> Result<()> { load_dotenvy_vars_if_present(); @@ -36,7 +41,33 @@ async fn main() -> Result<()> { return zero::prover_state::persistence::delete_all(); } - let runtime = Arc::new(Runtime::from_config(&args.paladin, register()).await?); + let mut light_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + let mut heavy_proof_routing_key = DEFAULT_ROUTING_KEY.to_string(); + if args.worker_run_mode == cli::WorkerRunMode::Affinity { + // If we're running in affinity mode, we need to set the routing key for the + // heavy proof and light proof. + info!("Workers running in affinity mode"); + light_proof_routing_key = LIGHT_PROOF_ROUTING_KEY.to_string(); + heavy_proof_routing_key = HEAVY_PROOF_ROUTING_KEY.to_string(); + } + + let light_proof_paladin_args = Config { + task_bus_routing_key: Some(light_proof_routing_key), + ..args.paladin.clone() + }; + + let heavy_proof_paladin_args = Config { + task_bus_routing_key: Some(heavy_proof_routing_key), + ..args.paladin + }; + + let light_proof = Runtime::from_config(&light_proof_paladin_args, register()).await?; + let heavy_proof = Runtime::from_config(&heavy_proof_paladin_args, register()).await?; + + let proof_runtime = Arc::new(ProofRuntime { + light_proof, + heavy_proof, + }); let prover_config: ProverConfig = args.prover_config.into(); if prover_config.block_pool_size == 0 { panic!("block-pool-size must be greater than 0"); @@ -55,7 +86,7 @@ async fn main() -> Result<()> { match args.command { Command::Stdio { previous_proof } => { let previous_proof = get_previous_proof(previous_proof)?; - stdio::stdio_main(runtime, previous_proof, Arc::new(prover_config)).await?; + stdio::stdio_main(proof_runtime, previous_proof, Arc::new(prover_config)).await?; } Command::Http { port, output_dir } => { // check if output_dir exists, is a directory, and is writable @@ -67,7 +98,7 @@ async fn main() -> Result<()> { panic!("output-dir is not a writable directory"); } - http::http_main(runtime, port, output_dir, Arc::new(prover_config)).await?; + http::http_main(proof_runtime, port, output_dir, Arc::new(prover_config)).await?; } Command::Rpc { rpc_url, @@ -86,7 +117,7 @@ async fn main() -> Result<()> { info!("Proving interval {block_interval}"); client_main( - runtime, + proof_runtime, RpcParams { rpc_url, rpc_type, diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index ede966493..d21af25a4 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -2,12 +2,14 @@ use std::path::PathBuf; use std::time::Duration; use alloy::transports::http::reqwest::Url; -use clap::{Parser, Subcommand, ValueHint}; +use clap::{Parser, Subcommand, ValueEnum, ValueHint}; use zero::parsing::parse_duration; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; use zero::rpc::{JumpdestSrc, RpcType}; +const WORKER_HELP_HEADING: &str = "Worker Config options"; + /// zero-bin leader config #[derive(Parser)] #[command(version = zero::version(), propagate_version = true)] @@ -25,6 +27,24 @@ pub(crate) struct Cli { // mode. #[clap(flatten)] pub(crate) prover_state_config: CliProverStateConfig, + + // Mode to use for worker for setup (affinity or default) + #[arg(long = "worker-run-mode", help_heading = WORKER_HELP_HEADING, value_enum, default_value = "default")] + pub(crate) worker_run_mode: WorkerRunMode, +} + +/// Defines the mode for worker setup in terms of job allocation: +/// +/// - `Affinity`: Workers are assigned specific types of jobs based on their +/// capabilities, distinguishing between heavy and light jobs. +/// - `Default`: No job distinction is made — any worker can handle any type of +/// job, whether heavy or light. +/// +/// This enum allows for flexible worker configuration based on workload needs. +#[derive(ValueEnum, Clone, PartialEq, Debug)] +pub enum WorkerRunMode { + Affinity, + Default, } #[derive(Subcommand)] diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index b999bb584..0b4c3b95b 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -4,7 +4,6 @@ use std::time::Duration; use alloy::rpc::types::{BlockId, BlockNumberOrTag}; use alloy::transports::http::reqwest::Url; use anyhow::{anyhow, Result}; -use paladin::runtime::Runtime; use tokio::sync::mpsc; use tracing::info; use zero::block_interval::{BlockInterval, BlockIntervalStream}; @@ -14,6 +13,8 @@ use zero::prover::{self, BlockProverInput, ProverConfig}; use zero::rpc::{self, JumpdestSrc}; use zero::rpc::{retry::build_http_retry_provider, RpcType}; +use crate::ProofRuntime; + #[derive(Debug)] pub struct RpcParams { pub rpc_url: Url, @@ -34,7 +35,7 @@ pub struct LeaderConfig { /// The main function for the client. pub(crate) async fn client_main( - runtime: Arc, + proof_runtime: Arc, rpc_params: RpcParams, block_interval: BlockInterval, mut leader_config: LeaderConfig, @@ -66,10 +67,10 @@ pub(crate) async fn client_main( let (block_tx, block_rx) = mpsc::channel::<(BlockProverInput, bool)>(zero::BLOCK_CHANNEL_SIZE); // Run proving task - let runtime_ = runtime.clone(); + let proof_runtime_ = proof_runtime.clone(); let proving_task = tokio::spawn(prover::prove( block_rx, - runtime_, + proof_runtime_, leader_config.previous_proof.take(), Arc::new(leader_config.prover_config), )); @@ -117,7 +118,8 @@ pub(crate) async fn client_main( } } - runtime.close().await?; + proof_runtime.light_proof.close().await?; + proof_runtime.heavy_proof.close().await?; if test_only { info!("All proof witnesses have been generated successfully."); diff --git a/zero/src/bin/leader/http.rs b/zero/src/bin/leader/http.rs index 02e968ec0..0bc90ac40 100644 --- a/zero/src/bin/leader/http.rs +++ b/zero/src/bin/leader/http.rs @@ -3,16 +3,17 @@ use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use alloy::primitives::U256; use anyhow::{bail, Result}; use axum::{http::StatusCode, routing::post, Json, Router}; -use paladin::runtime::Runtime; use serde::{Deserialize, Serialize}; use serde_json::to_writer; use tracing::{debug, error, info}; use zero::proof_types::GeneratedBlockProof; use zero::prover::{BlockProverInput, ProverConfig}; +use crate::ProofRuntime; + /// The main function for the HTTP mode. pub(crate) async fn http_main( - runtime: Arc, + proof_runtime: Arc, port: u16, output_dir: PathBuf, prover_config: Arc, @@ -22,10 +23,7 @@ pub(crate) async fn http_main( let app = Router::new().route( "/prove", - post({ - let runtime = runtime.clone(); - move |body| prove(body, runtime, output_dir.clone(), prover_config) - }), + post(move |body| prove(body, proof_runtime, output_dir.clone(), prover_config)), ); let listener = tokio::net::TcpListener::bind(&addr).await?; Ok(axum::serve(listener, app).await?) @@ -62,7 +60,7 @@ struct HttpProverInput { async fn prove( Json(payload): Json, - runtime: Arc, + proof_runtime: Arc, output_dir: PathBuf, prover_config: Arc, ) -> StatusCode { @@ -74,7 +72,7 @@ async fn prove( payload .prover_input .prove_test( - runtime, + proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) @@ -83,7 +81,7 @@ async fn prove( payload .prover_input .prove( - runtime, + proof_runtime, payload.previous.map(futures::future::ok), prover_config, ) diff --git a/zero/src/bin/leader/stdio.rs b/zero/src/bin/leader/stdio.rs index 306a08c65..9e451e13f 100644 --- a/zero/src/bin/leader/stdio.rs +++ b/zero/src/bin/leader/stdio.rs @@ -2,15 +2,16 @@ use std::io::Read; use std::sync::Arc; use anyhow::{anyhow, Result}; -use paladin::runtime::Runtime; use tokio::sync::mpsc; use tracing::info; use zero::proof_types::GeneratedBlockProof; use zero::prover::{self, BlockProverInput, ProverConfig}; +use crate::ProofRuntime; + /// The main function for the stdio mode. pub(crate) async fn stdio_main( - runtime: Arc, + proof_runtime: Arc, previous: Option, prover_config: Arc, ) -> Result<()> { @@ -24,9 +25,14 @@ pub(crate) async fn stdio_main( let (block_tx, block_rx) = mpsc::channel::<(BlockProverInput, bool)>(zero::BLOCK_CHANNEL_SIZE); - let runtime_ = runtime.clone(); + let proof_runtime_ = proof_runtime.clone(); let prover_config_ = prover_config.clone(); - let proving_task = tokio::spawn(prover::prove(block_rx, runtime_, previous, prover_config_)); + let proving_task = tokio::spawn(prover::prove( + block_rx, + proof_runtime_, + previous, + prover_config_, + )); let interval_len = block_prover_inputs.len(); for (index, block_prover_input) in block_prover_inputs.into_iter().enumerate() { @@ -48,7 +54,8 @@ pub(crate) async fn stdio_main( } } - runtime.close().await?; + proof_runtime.light_proof.close().await?; + proof_runtime.heavy_proof.close().await?; if prover_config.test_only { info!("All proof witnesses have been generated successfully."); diff --git a/zero/src/prover.rs b/zero/src/prover.rs index 6a194ddf1..4e221709c 100644 --- a/zero/src/prover.rs +++ b/zero/src/prover.rs @@ -9,9 +9,13 @@ use std::sync::Arc; use alloy::primitives::U256; use anyhow::{Context, Result}; use evm_arithmetization::Field; -use futures::{future::BoxFuture, FutureExt, TryFutureExt, TryStreamExt}; +use evm_arithmetization::SegmentDataIterator; +use futures::{ + future, future::BoxFuture, stream::FuturesUnordered, FutureExt, TryFutureExt, TryStreamExt, +}; use hashbrown::HashMap; use num_traits::ToPrimitive as _; +use paladin::directive::{Directive, IndexedStream}; use paladin::runtime::Runtime; use plonky2::gates::noop::NoopGate; use plonky2::plonk::circuit_builder::CircuitBuilder; @@ -28,6 +32,18 @@ use crate::fs::generate_block_proof_file_name; use crate::ops; use crate::proof_types::GeneratedBlockProof; +/// `ProofRuntime` represents the runtime environments used for generating +/// different types of proofs. It contains separate runtimes for handling: +/// +/// - `light_proof`: Typically for smaller, less resource-intensive tasks, such +/// as aggregation. +/// - `heavy_proof`: For larger, more computationally expensive tasks, such as +/// STARK proof generation. +pub struct ProofRuntime { + pub light_proof: Runtime, + pub heavy_proof: Runtime, +} + // All proving tasks are executed concurrently, which can cause issues for large // block intervals, where distant future blocks may be proven first. // @@ -65,14 +81,11 @@ impl BlockProverInput { pub async fn prove( self, - runtime: Arc, + proof_runtime: Arc, previous: Option>>, prover_config: Arc, ) -> Result { use anyhow::Context as _; - use evm_arithmetization::SegmentDataIterator; - use futures::{stream::FuturesUnordered, FutureExt}; - use paladin::directive::{Directive, IndexedStream}; let ProverConfig { max_cpu_len_log, @@ -116,7 +129,7 @@ impl BlockProverInput { Directive::map(IndexedStream::from(segment_data_iterator), &seg_prove_ops) .fold(&seg_agg_ops) - .run(&runtime) + .run(&proof_runtime.heavy_proof) .map(move |e| { e.map(|p| (idx, crate::proof_types::BatchAggregatableProof::from(p))) }) @@ -126,7 +139,7 @@ impl BlockProverInput { // Fold the batch aggregated proof stream into a single proof. let final_batch_proof = Directive::fold(IndexedStream::new(batch_proof_futs), &batch_agg_ops) - .run(&runtime) + .run(&proof_runtime.light_proof) .await?; if let crate::proof_types::BatchAggregatableProof::BatchAgg(proof) = final_batch_proof { @@ -143,7 +156,7 @@ impl BlockProverInput { prev, save_inputs_on_error, }) - .run(&runtime) + .run(&proof_runtime.light_proof) .await?; info!("Successfully proved block {block_number}"); @@ -156,13 +169,12 @@ impl BlockProverInput { pub async fn prove_test( self, - runtime: Arc, + proof_runtime: Arc, previous: Option>>, prover_config: Arc, ) -> Result { use std::iter::repeat; - use futures::future; use paladin::directive::{Directive, IndexedStream}; let ProverConfig { @@ -202,7 +214,7 @@ impl BlockProverInput { ); simulation - .run(&runtime) + .run(&proof_runtime.light_proof) .await? .try_for_each(|_| future::ok(())) .await?; @@ -236,17 +248,17 @@ impl BlockProverInput { async fn prove_block( block: BlockProverInput, - runtime: Arc, + proof_runtime: Arc, previous_block_proof: Option>>, prover_config: Arc, ) -> Result { if prover_config.test_only { block - .prove_test(runtime, previous_block_proof, prover_config) + .prove_test(proof_runtime, previous_block_proof, prover_config) .await } else { block - .prove(runtime, previous_block_proof, prover_config) + .prove(proof_runtime, previous_block_proof, prover_config) .await } } @@ -257,7 +269,7 @@ async fn prove_block( /// block proofs as well. pub async fn prove( mut block_receiver: Receiver<(BlockProverInput, bool)>, - runtime: Arc, + proof_runtime: Arc, checkpoint_proof: Option, prover_config: Arc, ) -> Result<()> { @@ -277,7 +289,7 @@ pub async fn prove( let (tx, rx) = oneshot::channel::(); let prover_config = prover_config.clone(); let previous_block_proof = prev_proof.take(); - let runtime = runtime.clone(); + let proof_runtime = proof_runtime.clone(); let block_number = block_prover_input.get_block_number(); let prove_permit = PARALLEL_BLOCK_PROVING_PERMIT_POOL.acquire().await?; @@ -288,7 +300,7 @@ pub async fn prove( // Prove the block let block_proof = prove_block( block_prover_input, - runtime, + proof_runtime, previous_block_proof, prover_config.clone(), ) From 09c42488296348e06a48c518122bafdf7c04ecd4 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:34:36 +0100 Subject: [PATCH 089/107] feat: SMT support in `trace_decoder` ignores storage (#693) --- .github/workflows/lint.yml | 12 +- Cargo.lock | 11 + Cargo.toml | 3 - trace_decoder/Cargo.toml | 3 +- trace_decoder/benches/block_processing.rs | 2 + trace_decoder/src/core.rs | 226 +++++++++--- trace_decoder/src/lib.rs | 9 +- trace_decoder/src/observer.rs | 2 +- trace_decoder/src/{typed_mpt.rs => tries.rs} | 345 ++++++++++++++---- trace_decoder/src/type1.rs | 15 +- trace_decoder/src/type2.rs | 294 ++++++++------- trace_decoder/src/wire.rs | 8 +- trace_decoder/tests/consistent-with-header.rs | 2 + trace_decoder/tests/simulate-execution.rs | 16 +- zero/Cargo.toml | 1 + zero/src/bin/rpc.rs | 2 + zero/src/bin/trie_diff.rs | 2 + zero/src/prover.rs | 16 +- 18 files changed, 683 insertions(+), 286 deletions(-) rename trace_decoder/src/{typed_mpt.rs => tries.rs} (58%) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 446d6a314..13089b3d0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,8 +13,6 @@ concurrency: env: CARGO_TERM_COLOR: always - BINSTALL_NO_CONFIRM: true - RUSTDOCFLAGS: "-D warnings" jobs: clippy: @@ -42,7 +40,15 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/rust - - run: cargo doc --all --no-deps + - run: RUSTDOCFLAGS='-D warnings -A rustdoc::private_intra_doc_links' cargo doc --all --no-deps + # TODO(zero): https://github.com/0xPolygonZero/zk_evm/issues/718 + - run: > + RUSTDOCFLAGS='-D warnings -A rustdoc::private_intra_doc_links' cargo doc --no-deps --document-private-items + --package trace_decoder + --package compat + --package smt_trie + --package zk_evm_proc_macro + --package zk_evm_common cargo-fmt: runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/Cargo.lock b/Cargo.lock index c546a7bcc..8ee516001 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,6 +1151,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "build-array" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ef4e2687af237b2646687e19a0643bc369878216122e46c3f1a01c56baa9d5" +dependencies = [ + "arrayvec", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -5127,6 +5136,7 @@ dependencies = [ "assert2", "bitflags 2.6.0", "bitvec", + "build-array", "bytes", "camino", "ciborium", @@ -5816,6 +5826,7 @@ dependencies = [ "anyhow", "async-stream", "axum", + "cfg-if", "clap", "compat", "directories", diff --git a/Cargo.toml b/Cargo.toml index 6c1191196..842e3c168 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,6 @@ axum = "0.7.5" bitflags = "2.5.0" bitvec = "1.0.1" bytes = "1.6.0" -cargo_metadata = "0.18.1" ciborium = "0.2.2" ciborium-io = "0.2.2" clap = { version = "4.5.7", features = ["derive", "env"] } @@ -75,7 +74,6 @@ nunny = "0.2.1" once_cell = "1.19.0" paladin-core = "0.4.3" parking_lot = "0.12.3" -paste = "1.0.15" pest = "2.7.10" pest_derive = "2.7.10" pretty_env_logger = "0.5.0" @@ -98,7 +96,6 @@ syn = "2.0" thiserror = "1.0.61" tiny-keccak = "2.0.2" tokio = { version = "1.38.0", features = ["full"] } -toml = "0.8.14" tower = "0.4" tracing = { version = "0.1", features = ["attributes"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } diff --git a/trace_decoder/Cargo.toml b/trace_decoder/Cargo.toml index de9b389a2..5bdd24f12 100644 --- a/trace_decoder/Cargo.toml +++ b/trace_decoder/Cargo.toml @@ -15,6 +15,7 @@ alloy-compat = "0.1.0" anyhow.workspace = true bitflags.workspace = true bitvec.workspace = true +build-array = "0.1.2" bytes.workspace = true ciborium.workspace = true ciborium-io.workspace = true @@ -33,6 +34,7 @@ nunny = { workspace = true, features = ["serde"] } plonky2.workspace = true rlp.workspace = true serde.workspace = true +smt_trie.workspace = true stackstack = "0.3.0" strum = { version = "0.26.3", features = ["derive"] } thiserror.workspace = true @@ -52,7 +54,6 @@ libtest-mimic = "0.7.3" plonky2_maybe_rayon.workspace = true serde_json.workspace = true serde_path_to_error.workspace = true -smt_trie.workspace = true zero.workspace = true [features] diff --git a/trace_decoder/benches/block_processing.rs b/trace_decoder/benches/block_processing.rs index adefdae3f..4e3582e98 100644 --- a/trace_decoder/benches/block_processing.rs +++ b/trace_decoder/benches/block_processing.rs @@ -8,6 +8,7 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; use trace_decoder::observer::DummyObserver; use trace_decoder::{BlockTrace, OtherBlockData}; +use zero::prover::WIRE_DISPOSITION; #[derive(Clone, Debug, serde::Deserialize)] pub struct ProverInput { @@ -39,6 +40,7 @@ fn criterion_benchmark(c: &mut Criterion) { other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, ) .unwrap() }, diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index d7b5e19c0..ab4c08a02 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -12,6 +12,7 @@ use alloy::{ }; use alloy_compat::Compat as _; use anyhow::{anyhow, bail, ensure, Context as _}; +use either::Either; use ethereum_types::{Address, H160, U256}; use evm_arithmetization::{ generation::{mpt::AccountRlp, TrieInputs}, @@ -26,9 +27,12 @@ use mpt_trie::partial_trie::PartialTrie as _; use nunny::NonEmpty; use zk_evm_common::gwei_to_wei; -use crate::observer::Observer; use crate::{ - typed_mpt::{ReceiptTrie, StateMpt, StateTrie, StorageTrie, TransactionTrie, TrieKey}, + observer::{DummyObserver, Observer}, + tries::StateSmt, +}; +use crate::{ + tries::{MptKey, ReceiptTrie, StateMpt, StateTrie, StorageTrie, TransactionTrie}, BlockLevelData, BlockTrace, BlockTraceTriePreImages, CombinedPreImages, ContractCodeUsage, OtherBlockData, SeparateStorageTriesPreImage, SeparateTriePreImage, SeparateTriePreImages, TxnInfo, TxnMeta, TxnTrace, @@ -52,12 +56,24 @@ pub fn is_precompile(addr: H160) -> bool { == address!("0000000000000000000000000000000000000100")) } +/// Expected trie type when parsing from binary in a [`BlockTrace`]. +/// +/// See [`crate::wire`] and [`CombinedPreImages`] for more. +#[derive(Debug)] +pub enum WireDisposition { + /// MPT + Type1, + /// SMT + Type2, +} + /// TODO(0xaatif): document this after pub fn entrypoint( trace: BlockTrace, other: OtherBlockData, batch_size_hint: usize, observer: &mut impl Observer, + wire_disposition: WireDisposition, ) -> anyhow::Result> { ensure!(batch_size_hint != 0); @@ -71,8 +87,8 @@ pub fn entrypoint( BlockTraceTriePreImages::Separate(_) => FatalMissingCode(true), BlockTraceTriePreImages::Combined(_) => FatalMissingCode(false), }; + let (state, storage, mut code) = start(trie_pre_images, wire_disposition)?; - let (state, storage, mut code) = start(trie_pre_images)?; code.extend(code_db.clone()); let OtherBlockData { @@ -92,17 +108,40 @@ pub fn entrypoint( *amt = gwei_to_wei(*amt) } - let batches = middle( - state, - storage, - batch(txn_info, batch_size_hint), - &mut code, - &b_meta, - ger_data, - withdrawals, - fatal_missing_code, - observer, - )?; + let batches = match state { + Either::Left(mpt) => Either::Left( + middle( + mpt, + storage, + batch(txn_info, batch_size_hint), + &mut code, + &b_meta, + ger_data, + withdrawals, + fatal_missing_code, + observer, + )? + .into_iter() + .map(|it| it.map(Either::Left)), + ), + Either::Right(smt) => { + Either::Right( + middle( + smt, + storage, + batch(txn_info, batch_size_hint), + &mut code, + &b_meta, + ger_data, + withdrawals, + fatal_missing_code, + &mut DummyObserver::new(), // TODO(0xaatif) + )? + .into_iter() + .map(|it| it.map(Either::Right)), + ) + } + }; let mut running_gas_used = 0; Ok(batches @@ -134,7 +173,10 @@ pub fn entrypoint( withdrawals, ger_data, tries: TrieInputs { - state_trie: state.into(), + state_trie: match state { + Either::Left(mpt) => mpt.into(), + Either::Right(_) => todo!("evm_arithmetization accepts an SMT"), + }, transactions_trie: transaction.into(), receipts_trie: receipt.into(), storage_tries: storage.into_iter().map(|(k, v)| (k, v.into())).collect(), @@ -182,11 +224,16 @@ pub fn entrypoint( /// [`HashedPartialTrie`](mpt_trie::partial_trie::HashedPartialTrie), /// or a [`wire`](crate::wire)-encoded representation of one. /// -/// Turn either of those into our [`typed_mpt`](crate::typed_mpt) -/// representations. +/// Turn either of those into our [internal representations](crate::tries). +#[allow(clippy::type_complexity)] fn start( pre_images: BlockTraceTriePreImages, -) -> anyhow::Result<(StateMpt, BTreeMap, Hash2Code)> { + wire_disposition: WireDisposition, +) -> anyhow::Result<( + Either, + BTreeMap, + Hash2Code, +)> { Ok(match pre_images { // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/401 // refactor our convoluted input types @@ -197,7 +244,7 @@ fn start( let state = state.items().try_fold( StateMpt::default(), |mut acc, (nibbles, hash_or_val)| { - let path = TrieKey::from_nibbles(nibbles); + let path = MptKey::from_nibbles(nibbles); match hash_or_val { mpt_trie::trie_ops::ValOrHash::Val(bytes) => { #[expect(deprecated)] // this is MPT specific @@ -220,7 +267,7 @@ fn start( .map(|(k, SeparateTriePreImage::Direct(v))| { v.items() .try_fold(StorageTrie::default(), |mut acc, (nibbles, hash_or_val)| { - let path = TrieKey::from_nibbles(nibbles); + let path = MptKey::from_nibbles(nibbles); match hash_or_val { mpt_trie::trie_ops::ValOrHash::Val(value) => { acc.insert(path, value)?; @@ -234,17 +281,35 @@ fn start( .map(|v| (k, v)) }) .collect::>()?; - (state, storage, Hash2Code::new()) + (Either::Left(state), storage, Hash2Code::new()) } BlockTraceTriePreImages::Combined(CombinedPreImages { compact }) => { let instructions = crate::wire::parse(&compact) .context("couldn't parse instructions from binary format")?; - let crate::type1::Frontend { - state, - storage, - code, - } = crate::type1::frontend(instructions)?; - (state, storage, code.into_iter().map(Into::into).collect()) + let (state, storage, code) = match wire_disposition { + WireDisposition::Type1 => { + let crate::type1::Frontend { + state, + storage, + code, + } = crate::type1::frontend(instructions)?; + ( + Either::Left(state), + storage, + Hash2Code::from_iter(code.into_iter().map(NonEmpty::into_vec)), + ) + } + WireDisposition::Type2 => { + let crate::type2::Frontend { trie, code } = + crate::type2::frontend(instructions)?; + ( + Either::Right(trie), + BTreeMap::new(), + Hash2Code::from_iter(code.into_iter().map(NonEmpty::into_vec)), + ) + } + }; + (state, storage, code) } }) } @@ -320,6 +385,31 @@ struct Batch { pub jumpdest_tables: Vec>, } +impl Batch { + fn map(self, f: impl FnMut(T) -> U) -> Batch { + let Self { + first_txn_ix, + gas_used, + contract_code, + byte_code, + before, + after, + withdrawals, + jumpdest_tables, + } = self; + Batch { + first_txn_ix, + gas_used, + contract_code, + byte_code, + before: before.map(f), + after, + withdrawals, + jumpdest_tables, + } + } +} + /// [`evm_arithmetization::generation::TrieInputs`], /// generic over state trie representation. #[derive(Debug)] @@ -330,6 +420,22 @@ pub struct IntraBlockTries { pub receipt: ReceiptTrie, } +impl IntraBlockTries { + fn map(self, mut f: impl FnMut(T) -> U) -> IntraBlockTries { + let Self { + state, + storage, + transaction, + receipt, + } = self; + IntraBlockTries { + state: f(state), + storage, + transaction, + receipt, + } + } +} /// Hacky handling of possibly missing contract bytecode in `Hash2Code` inner /// map. /// Allows incomplete payloads fetched with the zero tracer to skip these @@ -356,12 +462,15 @@ fn middle( fatal_missing_code: FatalMissingCode, // called with the untrimmed tries after each batch observer: &mut impl Observer, -) -> anyhow::Result>> { +) -> anyhow::Result>> +where + StateTrieT::Key: Ord + From
, +{ // Initialise the storage tries. for (haddr, acct) in state_trie.iter() { let storage = storage_tries.entry(haddr).or_insert({ let mut it = StorageTrie::default(); - it.insert_hash(TrieKey::default(), acct.storage_root) + it.insert_hash(MptKey::default(), acct.storage_root) .expect("empty trie insert cannot fail"); it }); @@ -396,8 +505,8 @@ fn middle( // We want to perform mask the TrieInputs above, // but won't know the bounds until after the loop below, // so store that information here. - let mut storage_masks = BTreeMap::<_, BTreeSet>::new(); - let mut state_mask = BTreeSet::new(); + let mut storage_masks = BTreeMap::<_, BTreeSet>::new(); + let mut state_mask = BTreeSet::::new(); if txn_ix == 0 { do_pre_execution( @@ -496,7 +605,7 @@ fn middle( storage_written .keys() .chain(&storage_read) - .map(|it| TrieKey::from_hash(keccak_hash::keccak(it))), + .map(|it| MptKey::from_hash(keccak_hash::keccak(it))), ); if do_writes { @@ -543,7 +652,7 @@ fn middle( }; for (k, v) in storage_written { - let slot = TrieKey::from_hash(keccak_hash::keccak(k)); + let slot = MptKey::from_hash(keccak_hash::keccak(k)); match v.is_zero() { // this is actually a delete true => storage_mask.extend(storage.reporting_remove(slot)?), @@ -556,10 +665,10 @@ fn middle( } state_trie.insert_by_address(addr, acct)?; - state_mask.insert(TrieKey::from_address(addr)); + state_mask.insert(::from(addr)); } else { // Simple state access - state_mask.insert(TrieKey::from_address(addr)); + state_mask.insert(::from(addr)); } if self_destructed { @@ -584,7 +693,7 @@ fn middle( withdrawals: match loop_ix == loop_len { true => { for (addr, amt) in &withdrawals { - state_mask.insert(TrieKey::from_address(*addr)); + state_mask.insert(::from(*addr)); let mut acct = state_trie .get_by_address(*addr) .context(format!("missing address {addr:x} for withdrawal"))?; @@ -643,10 +752,13 @@ fn do_pre_execution( block: &BlockMetadata, ger_data: Option<(H256, H256)>, storage: &mut BTreeMap, - trim_storage: &mut BTreeMap>, - trim_state: &mut BTreeSet, + trim_storage: &mut BTreeMap>, + trim_state: &mut BTreeSet, state_trie: &mut StateTrieT, -) -> anyhow::Result<()> { +) -> anyhow::Result<()> +where + StateTrieT::Key: From
+ Ord, +{ // Ethereum mainnet: EIP-4788 if cfg!(feature = "eth_mainnet") { return do_beacon_hook( @@ -682,10 +794,13 @@ fn do_scalable_hook( block: &BlockMetadata, ger_data: Option<(H256, H256)>, storage: &mut BTreeMap, - trim_storage: &mut BTreeMap>, - trim_state: &mut BTreeSet, + trim_storage: &mut BTreeMap>, + trim_state: &mut BTreeSet, state_trie: &mut StateTrieT, -) -> anyhow::Result<()> { +) -> anyhow::Result<()> +where + StateTrieT::Key: From
+ Ord, +{ use evm_arithmetization::testing_utils::{ ADDRESS_SCALABLE_L2, ADDRESS_SCALABLE_L2_ADDRESS_HASHED, GLOBAL_EXIT_ROOT_ADDRESS, GLOBAL_EXIT_ROOT_ADDRESS_HASHED, GLOBAL_EXIT_ROOT_STORAGE_POS, LAST_BLOCK_STORAGE_POS, @@ -700,7 +815,7 @@ fn do_scalable_hook( .context("missing scalable contract storage trie")?; let scalable_trim = trim_storage.entry(ADDRESS_SCALABLE_L2).or_default(); - let timestamp_slot_key = TrieKey::from_slot_position(U256::from(TIMESTAMP_STORAGE_POS.1)); + let timestamp_slot_key = MptKey::from_slot_position(U256::from(TIMESTAMP_STORAGE_POS.1)); let timestamp = scalable_storage .get(×tamp_slot_key) @@ -714,7 +829,7 @@ fn do_scalable_hook( (U256::from(LAST_BLOCK_STORAGE_POS.1), block.block_number), (U256::from(TIMESTAMP_STORAGE_POS.1), timestamp), ] { - let slot = TrieKey::from_slot_position(ix); + let slot = MptKey::from_slot_position(ix); // These values are never 0. scalable_storage.insert(slot, alloy::rlp::encode(u.compat()))?; @@ -727,12 +842,12 @@ fn do_scalable_hook( let mut arr = [0; 64]; (block.block_number - 1).to_big_endian(&mut arr[0..32]); U256::from(STATE_ROOT_STORAGE_POS.1).to_big_endian(&mut arr[32..64]); - let slot = TrieKey::from_hash(keccak_hash::keccak(arr)); + let slot = MptKey::from_hash(keccak_hash::keccak(arr)); scalable_storage.insert(slot, alloy::rlp::encode(prev_block_root_hash.compat()))?; scalable_trim.insert(slot); - trim_state.insert(TrieKey::from_address(ADDRESS_SCALABLE_L2)); + trim_state.insert(::from(ADDRESS_SCALABLE_L2)); let mut scalable_acct = state_trie .get_by_address(ADDRESS_SCALABLE_L2) .context("missing scalable contract address")?; @@ -753,12 +868,12 @@ fn do_scalable_hook( let mut arr = [0; 64]; arr[0..32].copy_from_slice(&root.0); U256::from(GLOBAL_EXIT_ROOT_STORAGE_POS.1).to_big_endian(&mut arr[32..64]); - let slot = TrieKey::from_hash(keccak_hash::keccak(arr)); + let slot = MptKey::from_hash(keccak_hash::keccak(arr)); ger_storage.insert(slot, alloy::rlp::encode(l1blockhash.compat()))?; ger_trim.insert(slot); - trim_state.insert(TrieKey::from_address(GLOBAL_EXIT_ROOT_ADDRESS)); + trim_state.insert(::from(GLOBAL_EXIT_ROOT_ADDRESS)); let mut ger_acct = state_trie .get_by_address(GLOBAL_EXIT_ROOT_ADDRESS) .context("missing GER contract address")?; @@ -781,11 +896,14 @@ fn do_scalable_hook( fn do_beacon_hook( block_timestamp: U256, storage: &mut BTreeMap, - trim_storage: &mut BTreeMap>, + trim_storage: &mut BTreeMap>, parent_beacon_block_root: H256, - trim_state: &mut BTreeSet, + trim_state: &mut BTreeSet, state_trie: &mut StateTrieT, -) -> anyhow::Result<()> { +) -> anyhow::Result<()> +where + StateTrieT::Key: From
+ Ord, +{ use evm_arithmetization::testing_utils::{ BEACON_ROOTS_CONTRACT_ADDRESS, BEACON_ROOTS_CONTRACT_ADDRESS_HASHED, HISTORY_BUFFER_LENGTH, }; @@ -806,7 +924,7 @@ fn do_beacon_hook( U256::from_big_endian(parent_beacon_block_root.as_bytes()), ), ] { - let slot = TrieKey::from_slot_position(ix); + let slot = MptKey::from_slot_position(ix); beacon_trim.insert(slot); match u.is_zero() { @@ -817,7 +935,7 @@ fn do_beacon_hook( } } } - trim_state.insert(TrieKey::from_address(BEACON_ROOTS_CONTRACT_ADDRESS)); + trim_state.insert(::from(BEACON_ROOTS_CONTRACT_ADDRESS)); let mut beacon_acct = state_trie .get_by_address(BEACON_ROOTS_CONTRACT_ADDRESS) .context("missing beacon contract address")?; @@ -845,7 +963,7 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// get added here as we process the deltas. #[derive(Default)] struct Hash2Code { - /// Key must always be [`hash`] of value. + /// Key must always be [`hash`](keccak_hash) of value. inner: HashMap>, } diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index bf7ff71cc..aa2f0a0f7 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -56,17 +56,12 @@ mod interface; pub use interface::*; +mod tries; mod type1; -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 -// add backend/prod support for type 2 -#[cfg(test)] -#[allow(dead_code)] mod type2; -mod typed_mpt; mod wire; -pub use core::entrypoint; -pub use core::is_precompile; +pub use core::{entrypoint, is_precompile, WireDisposition}; mod core; diff --git a/trace_decoder/src/observer.rs b/trace_decoder/src/observer.rs index 320019e55..f9811e87c 100644 --- a/trace_decoder/src/observer.rs +++ b/trace_decoder/src/observer.rs @@ -4,7 +4,7 @@ use std::marker::PhantomData; use ethereum_types::{H256, U256}; use crate::core::IntraBlockTries; -use crate::typed_mpt::{ReceiptTrie, StorageTrie, TransactionTrie}; +use crate::tries::{ReceiptTrie, StorageTrie, TransactionTrie}; /// Observer API for the trace decoder. /// Observer is used to collect various debugging and metadata info diff --git a/trace_decoder/src/typed_mpt.rs b/trace_decoder/src/tries.rs similarity index 58% rename from trace_decoder/src/typed_mpt.rs rename to trace_decoder/src/tries.rs index 8baf3cf29..91add4d98 100644 --- a/trace_decoder/src/typed_mpt.rs +++ b/trace_decoder/src/tries.rs @@ -1,10 +1,12 @@ -//! Principled MPT types used in this library. +//! Principled trie types and abstractions used in this library. use core::fmt; -use std::{collections::BTreeMap, marker::PhantomData}; +use std::{cmp, collections::BTreeMap, marker::PhantomData}; +use anyhow::ensure; +use bitvec::{array::BitArray, slice::BitSlice}; use copyvec::CopyVec; -use ethereum_types::{Address, H256, U256}; +use ethereum_types::{Address, BigEndianHash as _, H256, U256}; use evm_arithmetization::generation::mpt::AccountRlp; use mpt_trie::partial_trie::{HashedPartialTrie, Node, OnOrphanedHashNode, PartialTrie as _}; use u4::{AsNibbles, U4}; @@ -30,27 +32,26 @@ impl TypedMpt { /// Insert a node which represents an out-of-band sub-trie. /// /// See [module documentation](super) for more. - fn insert_hash(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { + fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { self.inner.insert(key.into_nibbles(), hash)?; Ok(()) } - /// Returns an [`Error`] if the `key` crosses into a part of the trie that - /// isn't hydrated. - fn insert(&mut self, key: TrieKey, value: T) -> anyhow::Result> + /// Returns [`Err`] if the `key` crosses into a part of the trie that + /// is hashed out. + fn insert(&mut self, key: MptKey, value: T) -> anyhow::Result<()> where T: rlp::Encodable + rlp::Decodable, { - let prev = self.get(key); self.inner .insert(key.into_nibbles(), rlp::encode(&value).to_vec())?; - Ok(prev) + Ok(()) } /// Note that this returns [`None`] if `key` crosses into a part of the - /// trie that isn't hydrated. + /// trie that is hashed out. /// /// # Panics /// - If [`rlp::decode`]-ing for `T` doesn't round-trip. - fn get(&self, key: TrieKey) -> Option + fn get(&self, key: MptKey) -> Option where T: rlp::Decodable, { @@ -67,12 +68,12 @@ impl TypedMpt { self.inner.hash() } /// Note that this returns owned paths and items. - fn iter(&self) -> impl Iterator + '_ + fn iter(&self) -> impl Iterator + '_ where T: rlp::Decodable, { self.inner.keys().filter_map(|nib| { - let path = TrieKey::from_nibbles(nib); + let path = MptKey::from_nibbles(nib); Some((path, self.get(path)?)) }) } @@ -88,7 +89,7 @@ impl<'a, T> IntoIterator for &'a TypedMpt where T: rlp::Decodable, { - type Item = (TrieKey, T); + type Item = (MptKey, T); type IntoIter = Box + 'a>; fn into_iter(self) -> Self::IntoIter { Box::new(self.iter()) @@ -100,9 +101,9 @@ where /// /// Semantically equivalent to [`mpt_trie::nibbles::Nibbles`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -pub struct TrieKey(CopyVec); +pub struct MptKey(CopyVec); -impl fmt::Display for TrieKey { +impl fmt::Display for MptKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for u in self.0 { f.write_fmt(format_args!("{:x}", u))? @@ -111,9 +112,9 @@ impl fmt::Display for TrieKey { } } -impl TrieKey { +impl MptKey { pub fn new(components: impl IntoIterator) -> anyhow::Result { - Ok(TrieKey(CopyVec::try_from_iter(components)?)) + Ok(MptKey(CopyVec::try_from_iter(components)?)) } pub fn into_hash_left_padded(mut self) -> H256 { for _ in 0..self.0.spare_capacity_mut().len() { @@ -136,7 +137,7 @@ impl TrieKey { } pub fn from_txn_ix(txn_ix: usize) -> Self { - TrieKey::new(AsNibbles(rlp::encode(&txn_ix))).expect( + MptKey::new(AsNibbles(rlp::encode(&txn_ix))).expect( "\ rlp of an usize goes through a u64, which is 8 bytes, which will be 9 bytes RLP'ed. @@ -170,17 +171,111 @@ impl TrieKey { } } +impl From
for MptKey { + fn from(value: Address) -> Self { + Self::from_hash(keccak_hash::keccak(value)) + } +} + #[test] -fn key_into_hash() { - assert_eq!(TrieKey::new([]).unwrap().into_hash(), None); +fn mpt_key_into_hash() { + assert_eq!(MptKey::new([]).unwrap().into_hash(), None); assert_eq!( - TrieKey::new(itertools::repeat_n(u4::u4!(0), 64)) + MptKey::new(itertools::repeat_n(u4::u4!(0), 64)) .unwrap() .into_hash(), Some(H256::zero()) ) } +/// Bounded sequence of bits, +/// used as a key for [`StateSmt`]. +/// +/// Semantically equivalent to [`smt_trie::bits::Bits`]. +#[derive(Clone, Copy)] +pub struct SmtKey { + bits: bitvec::array::BitArray<[u8; 32]>, + len: usize, +} + +impl SmtKey { + fn as_bitslice(&self) -> &BitSlice { + self.bits.as_bitslice().get(..self.len).unwrap() + } +} + +impl fmt::Debug for SmtKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries(self.as_bitslice().iter().map(|it| match *it { + true => 1, + false => 0, + })) + .finish() + } +} + +impl fmt::Display for SmtKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for bit in self.as_bitslice() { + f.write_str(match *bit { + true => "1", + false => "0", + })? + } + Ok(()) + } +} + +impl SmtKey { + pub fn new(components: impl IntoIterator) -> anyhow::Result { + let mut bits = bitvec::array::BitArray::default(); + let mut len = 0; + for (ix, bit) in components.into_iter().enumerate() { + ensure!( + bits.get(ix).is_some(), + "expected at most {} components", + bits.len() + ); + bits.set(ix, bit); + len += 1 + } + Ok(Self { bits, len }) + } + + pub fn into_smt_bits(self) -> smt_trie::bits::Bits { + let mut bits = smt_trie::bits::Bits::default(); + for bit in self.as_bitslice() { + bits.push_bit(*bit) + } + bits + } +} + +impl From
for SmtKey { + fn from(addr: Address) -> Self { + let H256(bytes) = keccak_hash::keccak(addr); + Self::new(BitArray::<_>::new(bytes)).expect("SmtKey has room for 256 bits") + } +} + +impl Ord for SmtKey { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.as_bitslice().cmp(other.as_bitslice()) + } +} +impl PartialOrd for SmtKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Eq for SmtKey {} +impl PartialEq for SmtKey { + fn eq(&self, other: &Self) -> bool { + self.as_bitslice().eq(other.as_bitslice()) + } +} + /// Per-block, `txn_ix -> [u8]`. /// /// See @@ -196,10 +291,10 @@ impl TransactionTrie { pub fn insert(&mut self, txn_ix: usize, val: Vec) -> anyhow::Result>> { let prev = self .untyped - .get(TrieKey::from_txn_ix(txn_ix).into_nibbles()) + .get(MptKey::from_txn_ix(txn_ix).into_nibbles()) .map(Vec::from); self.untyped - .insert(TrieKey::from_txn_ix(txn_ix).into_nibbles(), val)?; + .insert(MptKey::from_txn_ix(txn_ix).into_nibbles(), val)?; Ok(prev) } pub fn root(&self) -> H256 { @@ -214,7 +309,7 @@ impl TransactionTrie { &self.untyped, txn_ixs .into_iter() - .map(|it| TrieKey::from_txn_ix(it).into_nibbles()), + .map(|it| MptKey::from_txn_ix(it).into_nibbles()), )?; Ok(()) } @@ -241,10 +336,10 @@ impl ReceiptTrie { pub fn insert(&mut self, txn_ix: usize, val: Vec) -> anyhow::Result>> { let prev = self .untyped - .get(TrieKey::from_txn_ix(txn_ix).into_nibbles()) + .get(MptKey::from_txn_ix(txn_ix).into_nibbles()) .map(Vec::from); self.untyped - .insert(TrieKey::from_txn_ix(txn_ix).into_nibbles(), val)?; + .insert(MptKey::from_txn_ix(txn_ix).into_nibbles(), val)?; Ok(prev) } pub fn root(&self) -> H256 { @@ -259,7 +354,7 @@ impl ReceiptTrie { &self.untyped, txn_ixs .into_iter() - .map(|it| TrieKey::from_txn_ix(it).into_nibbles()), + .map(|it| MptKey::from_txn_ix(it).into_nibbles()), )?; Ok(()) } @@ -271,18 +366,14 @@ impl From for HashedPartialTrie { } } -/// TODO(0xaatif): document this after refactoring is done https://github.com/0xPolygonZero/zk_evm/issues/275 +/// TODO(0xaatif): document this after refactoring is done pub trait StateTrie { - fn insert_by_address( - &mut self, - address: Address, - account: AccountRlp, - ) -> anyhow::Result>; - fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()>; + type Key; + fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()>; fn get_by_address(&self, address: Address) -> Option; - fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; - /// _Hash out_ parts of the trie that aren't in `txn_ixs`. - fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; + fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; + /// _Hash out_ parts of the trie that aren't in `addresses`. + fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; fn iter(&self) -> impl Iterator + '_; fn root(&self) -> H256; } @@ -304,13 +395,17 @@ impl StateMpt { }, } } + /// Insert a _hashed out_ part of the trie + pub fn insert_hash_by_key(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { + self.typed.insert_hash(key, hash) + } #[deprecated = "prefer operations on `Address` where possible, as SMT support requires this"] pub fn insert_by_hashed_address( &mut self, key: H256, account: AccountRlp, - ) -> anyhow::Result> { - self.typed.insert(TrieKey::from_hash(key), account) + ) -> anyhow::Result<()> { + self.typed.insert(MptKey::from_hash(key), account) } pub fn iter(&self) -> impl Iterator + '_ { self.typed @@ -323,34 +418,27 @@ impl StateMpt { } impl StateTrie for StateMpt { - fn insert_by_address( - &mut self, - address: Address, - account: AccountRlp, - ) -> anyhow::Result> { + type Key = MptKey; + fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()> { #[expect(deprecated)] self.insert_by_hashed_address(keccak_hash::keccak(address), account) } - /// Insert an _hashed out_ part of the trie - fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { - self.typed.insert_hash(key, hash) - } fn get_by_address(&self, address: Address) -> Option { self.typed - .get(TrieKey::from_hash(keccak_hash::keccak(address))) + .get(MptKey::from_hash(keccak_hash::keccak(address))) } /// Delete the account at `address`, returning any remaining branch on /// collapse - fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { + fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { delete_node_and_report_remaining_key_if_branch_collapsed( self.typed.as_mut_hashed_partial_trie_unchecked(), - TrieKey::from_address(address), + MptKey::from_address(address), ) } - fn mask(&mut self, addresses: impl IntoIterator) -> anyhow::Result<()> { + fn mask(&mut self, addresses: impl IntoIterator) -> anyhow::Result<()> { let inner = mpt_trie::trie_subsets::create_trie_subset( self.typed.as_hashed_partial_trie(), - addresses.into_iter().map(TrieKey::into_nibbles), + addresses.into_iter().map(MptKey::into_nibbles), )?; self.typed = TypedMpt { inner, @@ -377,31 +465,30 @@ impl From for HashedPartialTrie { } } +// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/706 +// We're covering for [`smt_trie`] in a couple of ways: +// - insertion operations aren't fallible, they just panic. +// - it documents a requirement that `set_hash` is called before `set`. +#[derive(Clone, Debug)] pub struct StateSmt { address2state: BTreeMap, - hashed_out: BTreeMap, + hashed_out: BTreeMap, } impl StateTrie for StateSmt { - fn insert_by_address( - &mut self, - address: Address, - account: AccountRlp, - ) -> anyhow::Result> { - Ok(self.address2state.insert(address, account)) - } - fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { - self.hashed_out.insert(key, hash); + type Key = SmtKey; + fn insert_by_address(&mut self, address: Address, account: AccountRlp) -> anyhow::Result<()> { + self.address2state.insert(address, account); Ok(()) } fn get_by_address(&self, address: Address) -> Option { self.address2state.get(&address).copied() } - fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { + fn reporting_remove(&mut self, address: Address) -> anyhow::Result> { self.address2state.remove(&address); Ok(None) } - fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()> { + fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()> { let _ = address; Ok(()) } @@ -411,7 +498,111 @@ impl StateTrie for StateSmt { .map(|(addr, acct)| (keccak_hash::keccak(addr), *acct)) } fn root(&self) -> H256 { - todo!() + conv_hash::smt2eth(self.as_smt().root) + } +} + +impl StateSmt { + pub(crate) fn new_unchecked( + address2state: BTreeMap, + hashed_out: BTreeMap, + ) -> Self { + Self { + address2state, + hashed_out, + } + } + + fn as_smt(&self) -> smt_trie::smt::Smt { + let Self { + address2state, + hashed_out, + } = self; + let mut smt = smt_trie::smt::Smt::::default(); + for (k, v) in hashed_out { + smt.set_hash(k.into_smt_bits(), conv_hash::eth2smt(*v)); + } + for ( + addr, + AccountRlp { + nonce, + balance, + storage_root, + code_hash, + }, + ) in address2state + { + smt.set(smt_trie::keys::key_nonce(*addr), *nonce); + smt.set(smt_trie::keys::key_balance(*addr), *balance); + smt.set(smt_trie::keys::key_code(*addr), code_hash.into_uint()); + smt.set( + // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 + // combined abstraction for state and storage + smt_trie::keys::key_storage(*addr, U256::zero()), + storage_root.into_uint(), + ); + } + smt + } +} + +mod conv_hash { + //! We [`u64::to_le_bytes`] because: + //! - Reference go code just puns the bytes: + //! - It's better to fix the endianness for correctness. + //! - Most (consumer) CPUs are little-endian. + + use std::array; + + use ethereum_types::H256; + use itertools::Itertools as _; + use plonky2::{ + field::{ + goldilocks_field::GoldilocksField, + types::{Field as _, PrimeField64}, + }, + hash::hash_types::HashOut, + }; + + /// # Panics + /// - On certain inputs if `debug_assertions` are enabled. See + /// [`GoldilocksField::from_canonical_u64`] for more. + pub fn eth2smt(H256(bytes): H256) -> smt_trie::smt::HashOut { + let mut bytes = bytes.into_iter(); + // (no unsafe, no unstable) + let ret = HashOut { + elements: array::from_fn(|_ix| { + let (a, b, c, d, e, f, g, h) = bytes.next_tuple().unwrap(); + GoldilocksField::from_canonical_u64(u64::from_le_bytes([a, b, c, d, e, f, g, h])) + }), + }; + assert_eq!(bytes.len(), 0); + ret + } + pub fn smt2eth(HashOut { elements }: smt_trie::smt::HashOut) -> H256 { + H256( + build_array::ArrayBuilder::from_iter( + elements + .iter() + .map(GoldilocksField::to_canonical_u64) + .flat_map(u64::to_le_bytes), + ) + .build_exact() + .unwrap(), + ) + } + + #[test] + fn test() { + use plonky2::field::types::Field64 as _; + let mut max = std::iter::repeat(GoldilocksField::ORDER - 1).flat_map(u64::to_le_bytes); + for h in [ + H256::zero(), + H256(array::from_fn(|ix| ix as u8)), + H256(array::from_fn(|_| max.next().unwrap())), + ] { + assert_eq!(smt2eth(eth2smt(h)), h); + } } } @@ -428,15 +619,15 @@ impl StorageTrie { untyped: HashedPartialTrie::new_with_strategy(Node::Empty, strategy), } } - pub fn get(&mut self, key: &TrieKey) -> Option<&[u8]> { + pub fn get(&mut self, key: &MptKey) -> Option<&[u8]> { self.untyped.get(key.into_nibbles()) } - pub fn insert(&mut self, key: TrieKey, value: Vec) -> anyhow::Result>> { + pub fn insert(&mut self, key: MptKey, value: Vec) -> anyhow::Result>> { let prev = self.get(&key).map(Vec::from); self.untyped.insert(key.into_nibbles(), value)?; Ok(prev) } - pub fn insert_hash(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()> { + pub fn insert_hash(&mut self, key: MptKey, hash: H256) -> anyhow::Result<()> { self.untyped.insert(key.into_nibbles(), hash)?; Ok(()) } @@ -446,17 +637,17 @@ impl StorageTrie { pub const fn as_hashed_partial_trie(&self) -> &HashedPartialTrie { &self.untyped } - pub fn reporting_remove(&mut self, key: TrieKey) -> anyhow::Result> { + pub fn reporting_remove(&mut self, key: MptKey) -> anyhow::Result> { delete_node_and_report_remaining_key_if_branch_collapsed(&mut self.untyped, key) } pub fn as_mut_hashed_partial_trie_unchecked(&mut self) -> &mut HashedPartialTrie { &mut self.untyped } /// _Hash out_ the parts of the trie that aren't in `paths`. - pub fn mask(&mut self, paths: impl IntoIterator) -> anyhow::Result<()> { + pub fn mask(&mut self, paths: impl IntoIterator) -> anyhow::Result<()> { self.untyped = mpt_trie::trie_subsets::create_trie_subset( &self.untyped, - paths.into_iter().map(TrieKey::into_nibbles), + paths.into_iter().map(MptKey::into_nibbles), )?; Ok(()) } @@ -473,18 +664,18 @@ impl From for HashedPartialTrie { /// plonky2. Returns the key to the remaining child if a collapse occurred. fn delete_node_and_report_remaining_key_if_branch_collapsed( trie: &mut HashedPartialTrie, - key: TrieKey, -) -> anyhow::Result> { + key: MptKey, +) -> anyhow::Result> { let old_trace = get_trie_trace(trie, key); trie.delete(key.into_nibbles())?; let new_trace = get_trie_trace(trie, key); Ok( node_deletion_resulted_in_a_branch_collapse(&old_trace, &new_trace) - .map(TrieKey::from_nibbles), + .map(MptKey::from_nibbles), ) } -fn get_trie_trace(trie: &HashedPartialTrie, k: TrieKey) -> mpt_trie::utils::TriePath { +fn get_trie_trace(trie: &HashedPartialTrie, k: MptKey) -> mpt_trie::utils::TriePath { mpt_trie::special_query::path_for_query(trie, k.into_nibbles(), true).collect() } diff --git a/trace_decoder/src/type1.rs b/trace_decoder/src/type1.rs index aeea0dbb6..c44beaec7 100644 --- a/trace_decoder/src/type1.rs +++ b/trace_decoder/src/type1.rs @@ -12,7 +12,7 @@ use mpt_trie::partial_trie::OnOrphanedHashNode; use nunny::NonEmpty; use u4::U4; -use crate::typed_mpt::{StateMpt, StateTrie as _, StorageTrie, TrieKey}; +use crate::tries::{MptKey, StateMpt, StorageTrie}; use crate::wire::{Instruction, SmtLeaf}; #[derive(Debug, Clone)] @@ -66,10 +66,10 @@ fn visit( Node::Hash(Hash { raw_hash }) => { frontend .state - .insert_hash_by_key(TrieKey::new(path.iter().copied())?, raw_hash.into())?; + .insert_hash_by_key(MptKey::new(path.iter().copied())?, raw_hash.into())?; } Node::Leaf(Leaf { key, value }) => { - let path = TrieKey::new(path.iter().copied().chain(key))? + let path = MptKey::new(path.iter().copied().chain(key))? .into_hash() .context("invalid depth for leaf of state trie")?; match value { @@ -106,8 +106,7 @@ fn visit( }, }; #[expect(deprecated)] // this is MPT-specific code - let clobbered = frontend.state.insert_by_hashed_address(path, account)?; - ensure!(clobbered.is_none(), "duplicate account"); + frontend.state.insert_by_hashed_address(path, account)?; } } } @@ -141,12 +140,12 @@ fn node2storagetrie(node: Node) -> anyhow::Result { ) -> anyhow::Result<()> { match node { Node::Hash(Hash { raw_hash }) => { - mpt.insert_hash(TrieKey::new(path.iter().copied())?, raw_hash.into())?; + mpt.insert_hash(MptKey::new(path.iter().copied())?, raw_hash.into())?; } Node::Leaf(Leaf { key, value }) => { match value { Either::Left(Value { raw_value }) => mpt.insert( - TrieKey::new(path.iter().copied().chain(key))?, + MptKey::new(path.iter().copied().chain(key))?, rlp::encode(&raw_value.as_slice()).to_vec(), )?, Either::Right(_) => bail!("unexpected account node in storage trie"), @@ -380,6 +379,8 @@ fn finish_stack(v: &mut Vec) -> anyhow::Result { #[test] fn test_tries() { + use crate::tries::StateTrie as _; + for (ix, case) in serde_json::from_str::>(include_str!("cases/zero_jerigon.json")) .unwrap() diff --git a/trace_decoder/src/type2.rs b/trace_decoder/src/type2.rs index dd3e45c4b..a71761533 100644 --- a/trace_decoder/src/type2.rs +++ b/trace_decoder/src/type2.rs @@ -1,35 +1,34 @@ //! Frontend for the witness format emitted by e.g [`0xPolygonHermez/cdk-erigon`](https://github.com/0xPolygonHermez/cdk-erigon/) //! Ethereum node. -use std::{ - collections::{HashMap, HashSet}, - iter, -}; +use std::collections::{BTreeMap, HashSet}; use anyhow::{bail, ensure, Context as _}; -use bitvec::vec::BitVec; -use either::Either; -use ethereum_types::BigEndianHash as _; -use itertools::{EitherOrBoth, Itertools as _}; +use ethereum_types::{Address, U256}; +use evm_arithmetization::generation::mpt::AccountRlp; +use itertools::EitherOrBoth; +use keccak_hash::H256; use nunny::NonEmpty; -use plonky2::field::types::Field; - -use crate::wire::{Instruction, SmtLeaf, SmtLeafType}; +use stackstack::Stack; -type SmtTrie = smt_trie::smt::Smt; +use crate::{ + tries::{SmtKey, StateSmt}, + wire::{Instruction, SmtLeaf, SmtLeafType}, +}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +/// Combination of all the [`SmtLeaf::node_type`]s +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] pub struct CollatedLeaf { pub balance: Option, pub nonce: Option, - pub code_hash: Option, - pub storage_root: Option, + pub code: Option, + pub code_length: Option, + pub storage: BTreeMap, } pub struct Frontend { - pub trie: SmtTrie, + pub trie: StateSmt, pub code: HashSet>>, - pub collation: HashMap, } /// # Panics @@ -37,18 +36,13 @@ pub struct Frontend { /// NOT call this function on untrusted inputs. pub fn frontend(instructions: impl IntoIterator) -> anyhow::Result { let (node, code) = fold(instructions).context("couldn't fold smt from instructions")?; - let (trie, collation) = - node2trie(node).context("couldn't construct trie and collation from folded node")?; - Ok(Frontend { - trie, - code, - collation, - }) + let trie = node2trie(node).context("couldn't construct trie and collation from folded node")?; + Ok(Frontend { trie, code }) } /// Node in a binary (SMT) tree. /// -/// This is an intermediary type on the way to [`SmtTrie`]. +/// This is an intermediary type on the way to [`StateSmt`]. enum Node { Branch(EitherOrBoth>), Hash([u8; 32]), @@ -105,9 +99,9 @@ fn fold1(instructions: impl IntoIterator) -> anyhow::Result< Ok(Some(match mask { // note that the single-child bits are reversed... - 0b0001 => Node::Branch(EitherOrBoth::Left(get_child()?)), - 0b0010 => Node::Branch(EitherOrBoth::Right(get_child()?)), - 0b0011 => Node::Branch(EitherOrBoth::Both(get_child()?, get_child()?)), + 0b_01 => Node::Branch(EitherOrBoth::Left(get_child()?)), + 0b_10 => Node::Branch(EitherOrBoth::Right(get_child()?)), + 0b_11 => Node::Branch(EitherOrBoth::Both(get_child()?, get_child()?)), other => bail!("unexpected bit pattern in Branch mask: {:#b}", other), })) } @@ -119,113 +113,162 @@ fn fold1(instructions: impl IntoIterator) -> anyhow::Result< } } -/// Pack a [`Node`] tree into an [`SmtTrie`]. -/// Also summarizes the [`Node::Leaf`]s out-of-band. -/// -/// # Panics -/// - if the tree is too deep. -/// - if [`SmtLeaf::address`] or [`SmtLeaf::value`] are the wrong length. -/// - if [`SmtLeafType::Storage`] is the wrong length. -/// - [`SmtTrie`] panics internally. -fn node2trie( - node: Node, -) -> anyhow::Result<(SmtTrie, HashMap)> { - let mut trie = SmtTrie::default(); - - let (hashes, leaves) = - iter_leaves(node).partition_map::, Vec<_>, _, _, _>(|(path, leaf)| match leaf { - Either::Left(it) => Either::Left((path, it)), - Either::Right(it) => Either::Right(it), - }); - - for (path, hash) in hashes { - // needs to be called before `set`, below, "to avoid any issues" according - // to the smt docs. - trie.set_hash( - bits2bits(path), - smt_trie::smt::HashOut { - elements: { - let ethereum_types::U256(arr) = ethereum_types::H256(hash).into_uint(); - arr.map(smt_trie::smt::F::from_canonical_u64) +fn node2trie(node: Node) -> anyhow::Result { + let mut hashes = BTreeMap::new(); + let mut leaves = BTreeMap::new(); + visit(&mut hashes, &mut leaves, Stack::new(), node)?; + Ok(StateSmt::new_unchecked( + leaves + .into_iter() + .map( + |( + addr, + CollatedLeaf { + balance, + nonce, + // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 + // we shouldn't ignore these fields + code: _, + code_length: _, + storage: _, + }, + )| { + ( + addr, + AccountRlp { + nonce: nonce.unwrap_or_default(), + balance: balance.unwrap_or_default(), + storage_root: H256::zero(), + code_hash: H256::zero(), + }, + ) }, - }, - ) - } + ) + .collect(), + hashes, + )) +} - let mut collated = HashMap::::new(); - for SmtLeaf { - node_type, - address, - value, - } in leaves - { - let address = ethereum_types::Address::from_slice(&address); - let collated = collated.entry(address).or_default(); - let value = ethereum_types::U256::from_big_endian(&value); - let key = match node_type { - SmtLeafType::Balance => { - ensure!(collated.balance.is_none(), "double write of field"); - collated.balance = Some(value); - smt_trie::keys::key_balance(address) - } - SmtLeafType::Nonce => { - ensure!(collated.nonce.is_none(), "double write of field"); - collated.nonce = Some(value); - smt_trie::keys::key_nonce(address) +fn visit( + hashes: &mut BTreeMap, + leaves: &mut BTreeMap, + path: Stack, + node: Node, +) -> anyhow::Result<()> { + match node { + Node::Branch(children) => { + let (left, right) = children.left_and_right(); + if let Some(left) = left { + visit(hashes, leaves, path.pushed(false), *left)?; } - SmtLeafType::Code => { - ensure!(collated.code_hash.is_none(), "double write of field"); - collated.code_hash = Some({ - let mut it = ethereum_types::H256::zero(); - value.to_big_endian(it.as_bytes_mut()); - it - }); - smt_trie::keys::key_code(address) + if let Some(right) = right { + visit(hashes, leaves, path.pushed(true), *right)?; } - SmtLeafType::Storage(it) => { - ensure!(collated.storage_root.is_none(), "double write of field"); - // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 - // do we not do anything with the storage here? - smt_trie::keys::key_storage(address, ethereum_types::U256::from_big_endian(&it)) + } + Node::Hash(hash) => { + hashes.insert(SmtKey::new(path.iter().copied())?, H256(hash)); + } + Node::Leaf(SmtLeaf { + node_type, + address, + value, + }) => { + let address = Address::from_slice(&address); + let collated = leaves.entry(address).or_default(); + let value = U256::from_big_endian(&value); + macro_rules! ensure { + ($expr:expr) => { + ::anyhow::ensure!($expr, "double write of field for address {}", address) + }; } - SmtLeafType::CodeLength => smt_trie::keys::key_code_length(address), - }; - trie.set(key, value) + match node_type { + SmtLeafType::Balance => { + ensure!(collated.balance.is_none()); + collated.balance = Some(value) + } + SmtLeafType::Nonce => { + ensure!(collated.nonce.is_none()); + collated.nonce = Some(value) + } + SmtLeafType::Code => { + ensure!(collated.code.is_none()); + collated.code = Some(value) + } + SmtLeafType::Storage(slot) => { + let clobbered = collated.storage.insert(U256::from_big_endian(&slot), value); + ensure!(clobbered.is_none()) + } + SmtLeafType::CodeLength => { + ensure!(collated.code_length.is_none()); + collated.code_length = Some(value) + } + }; + } } - Ok((trie, collated)) + Ok(()) } -/// # Panics -/// - on overcapacity -fn bits2bits(ours: BitVec) -> smt_trie::bits::Bits { - let mut theirs = smt_trie::bits::Bits::empty(); - for it in ours { - theirs.push_bit(it) - } - theirs -} +#[test] +fn test_tries() { + type Smt = smt_trie::smt::Smt; + use ethereum_types::BigEndianHash as _; + use plonky2::field::types::{Field, Field64 as _}; -/// Simple, inefficient visitor of all leaves of the [`Node`] tree. -#[allow(clippy::type_complexity)] -fn iter_leaves(node: Node) -> Box)>> { - match node { - Node::Hash(it) => Box::new(iter::once((BitVec::new(), Either::Left(it)))), - Node::Branch(it) => { - let (left, right) = it.left_and_right(); - let left = left - .into_iter() - .flat_map(|it| iter_leaves(*it).update(|(path, _)| path.insert(0, false))); - let right = right - .into_iter() - .flat_map(|it| iter_leaves(*it).update(|(path, _)| path.insert(0, true))); - Box::new(left.chain(right)) + // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/707 + // this logic should live in StateSmt, but we need to + // - abstract over state and storage tries + // - parameterize the account types + // we preserve this code as a tested record of how it _should_ + // be done. + fn node2trie(node: Node) -> anyhow::Result { + let mut trie = Smt::default(); + let mut hashes = BTreeMap::new(); + let mut leaves = BTreeMap::new(); + visit(&mut hashes, &mut leaves, Stack::new(), node)?; + for (key, hash) in hashes { + trie.set_hash( + key.into_smt_bits(), + smt_trie::smt::HashOut { + elements: { + let ethereum_types::U256(arr) = hash.into_uint(); + for u in arr { + ensure!(u < smt_trie::smt::F::ORDER); + } + arr.map(smt_trie::smt::F::from_canonical_u64) + }, + }, + ); } - Node::Leaf(it) => Box::new(iter::once((BitVec::new(), Either::Right(it)))), + for ( + addr, + CollatedLeaf { + balance, + nonce, + code, + code_length, + storage, + }, + ) in leaves + { + use smt_trie::keys::{key_balance, key_code, key_code_length, key_nonce, key_storage}; + + for (value, key_fn) in [ + (balance, key_balance as fn(_) -> _), + (nonce, key_nonce), + (code, key_code), + (code_length, key_code_length), + ] { + if let Some(value) = value { + trie.set(key_fn(addr), value); + } + } + for (slot, value) in storage { + trie.set(key_storage(addr, slot), value); + } + } + Ok(trie) } -} -#[test] -fn test_tries() { for (ix, case) in serde_json::from_str::>(include_str!("cases/hermez_cdk_erigon.json")) .unwrap() @@ -234,10 +277,11 @@ fn test_tries() { { println!("case {}", ix); let instructions = crate::wire::parse(&case.bytes).unwrap(); - let frontend = frontend(instructions).unwrap(); + let (node, _code) = fold(instructions).unwrap(); + let trie = node2trie(node).unwrap(); assert_eq!(case.expected_state_root, { let mut it = [0; 32]; - smt_trie::utils::hashout2u(frontend.trie.root).to_big_endian(&mut it); + smt_trie::utils::hashout2u(trie.root).to_big_endian(&mut it); ethereum_types::H256(it) }); } diff --git a/trace_decoder/src/wire.rs b/trace_decoder/src/wire.rs index 6f56f1e44..63dee6040 100644 --- a/trace_decoder/src/wire.rs +++ b/trace_decoder/src/wire.rs @@ -1,6 +1,6 @@ //! We support two wire formats: -//! - Type 1, based on [this specification](https://gist.github.com/mandrigin/ff7eccf30d0ef9c572bafcb0ab665cff#the-bytes-layout). -//! - Type 2, loosely based on [this specification](https://github.com/0xPolygonHermez/cdk-erigon/blob/d1d6b3c7a4c81c46fd995c1baa5c1f8069ff0348/turbo/trie/WITNESS.md) +//! - Type 1 (AKA MPT), based on [this specification](https://gist.github.com/mandrigin/ff7eccf30d0ef9c572bafcb0ab665cff#the-bytes-layout). +//! - Type 2 (AKA SMT), loosely based on [this specification](https://github.com/0xPolygonHermez/cdk-erigon/blob/d1d6b3c7a4c81c46fd995c1baa5c1f8069ff0348/turbo/trie/WITNESS.md) //! //! Fortunately, their opcodes don't conflict, so we can have a single //! [`Instruction`] type, with shared parsing logic in this module, and bail on @@ -80,6 +80,8 @@ pub enum Instruction { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 +// `address` and `value` should be fixed length fields pub struct SmtLeaf { pub node_type: SmtLeafType, pub address: NonEmpty>, @@ -87,6 +89,8 @@ pub struct SmtLeaf { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 +// `Storage` should contain a fixed length field pub enum SmtLeafType { Balance, Nonce, diff --git a/trace_decoder/tests/consistent-with-header.rs b/trace_decoder/tests/consistent-with-header.rs index 609fd57bb..63df41b4f 100644 --- a/trace_decoder/tests/consistent-with-header.rs +++ b/trace_decoder/tests/consistent-with-header.rs @@ -11,6 +11,7 @@ use itertools::Itertools; use libtest_mimic::{Arguments, Trial}; use mpt_trie::partial_trie::PartialTrie as _; use trace_decoder::observer::DummyObserver; +use zero::prover::WIRE_DISPOSITION; fn main() -> anyhow::Result<()> { let mut trials = vec![]; @@ -29,6 +30,7 @@ fn main() -> anyhow::Result<()> { other.clone(), batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, ) .map_err(|e| format!("{e:?}"))?; // get the full cause chain check!(gen_inputs.len() >= 2); diff --git a/trace_decoder/tests/simulate-execution.rs b/trace_decoder/tests/simulate-execution.rs index d0476c2b7..fc7136c34 100644 --- a/trace_decoder/tests/simulate-execution.rs +++ b/trace_decoder/tests/simulate-execution.rs @@ -9,6 +9,7 @@ use common::{cases, Case}; use libtest_mimic::{Arguments, Trial}; use plonky2::field::goldilocks_field::GoldilocksField; use trace_decoder::observer::DummyObserver; +use zero::prover::WIRE_DISPOSITION; fn main() -> anyhow::Result<()> { let mut trials = vec![]; @@ -20,11 +21,16 @@ fn main() -> anyhow::Result<()> { other, } in cases()? { - let gen_inputs = - trace_decoder::entrypoint(trace, other, batch_size, &mut DummyObserver::new()) - .context(format!( - "error in `trace_decoder` for {name} at batch size {batch_size}" - ))?; + let gen_inputs = trace_decoder::entrypoint( + trace, + other, + batch_size, + &mut DummyObserver::new(), + WIRE_DISPOSITION, + ) + .context(format!( + "error in `trace_decoder` for {name} at batch size {batch_size}" + ))?; for (ix, gi) in gen_inputs.into_iter().enumerate() { trials.push(Trial::test( format!("{name}@{batch_size}/{ix}"), diff --git a/zero/Cargo.toml b/zero/Cargo.toml index 18da12fc3..362125e22 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -17,6 +17,7 @@ alloy-serde.workspace = true anyhow.workspace = true async-stream.workspace = true axum.workspace = true +cfg-if = "1.0.0" clap = { workspace = true, features = ["derive", "string"] } compat.workspace = true directories = "5.0.1" diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index e87e9e34d..fa9d9eab1 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -16,6 +16,7 @@ use zero::block_interval::BlockInterval; use zero::block_interval::BlockIntervalStream; use zero::parsing::parse_duration; use zero::prover::BlockProverInput; +use zero::prover::WIRE_DISPOSITION; use zero::provider::CachedProvider; use zero::rpc; use zero::rpc::JumpdestSrc; @@ -194,6 +195,7 @@ impl Cli { block_prover_input.other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, )?; if let Some(index) = tx_info.transaction_index { diff --git a/zero/src/bin/trie_diff.rs b/zero/src/bin/trie_diff.rs index 4c00d2ca3..c211cc528 100644 --- a/zero/src/bin/trie_diff.rs +++ b/zero/src/bin/trie_diff.rs @@ -26,6 +26,7 @@ use regex::Regex; use trace_decoder::observer::TriesObserver; use tracing::{error, info}; use zero::ops::register; +use zero::prover::WIRE_DISPOSITION; use zero::prover::{cli::CliProverConfig, BlockProverInput, ProverConfig}; /// This binary is a debugging tool used to compare @@ -97,6 +98,7 @@ async fn main() -> Result<()> { block_prover_input.other_data.clone(), prover_config.batch_size, &mut observer, + WIRE_DISPOSITION, )?; info!( "Number of collected batch tries for block {}: {}", diff --git a/zero/src/prover.rs b/zero/src/prover.rs index 4e221709c..7cc840f02 100644 --- a/zero/src/prover.rs +++ b/zero/src/prover.rs @@ -25,7 +25,7 @@ use tokio::io::AsyncWriteExt; use tokio::sync::mpsc::Receiver; use tokio::sync::{oneshot, Semaphore}; use trace_decoder::observer::DummyObserver; -use trace_decoder::{BlockTrace, OtherBlockData}; +use trace_decoder::{BlockTrace, OtherBlockData, WireDisposition}; use tracing::{error, info}; use crate::fs::generate_block_proof_file_name; @@ -55,6 +55,18 @@ pub struct ProofRuntime { // batches as soon as they are generated. static PARALLEL_BLOCK_PROVING_PERMIT_POOL: Semaphore = Semaphore::const_new(0); +pub const WIRE_DISPOSITION: WireDisposition = { + cfg_if::cfg_if! { + if #[cfg(feature = "eth_mainnet")] { + WireDisposition::Type1 + } else if #[cfg(feature = "cdk_erigon")] { + WireDisposition::Type2 + } else { + compile_error!("must select a feature"); + } + } +}; + #[derive(Debug, Clone)] pub struct ProverConfig { pub batch_size: usize, @@ -101,6 +113,7 @@ impl BlockProverInput { self.other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, )?; // Create segment proof. @@ -193,6 +206,7 @@ impl BlockProverInput { self.other_data, batch_size, &mut DummyObserver::new(), + WIRE_DISPOSITION, )?; let seg_ops = ops::SegmentProofTestOnly { From 60e32fb9439d704b4c3ae5fcb86c70731cdafda1 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 17:59:01 +0200 Subject: [PATCH 090/107] Merge remote-tracking branch 'origin/develop' into einar/prefetch_transaction_jumps/pr --- Cargo.lock | 15 +- Cargo.toml | 6 +- evm_arithmetization/src/all_stark.rs | 40 ++- .../src/cpu/kernel/interpreter.rs | 16 +- .../src/fixed_recursive_verifier.rs | 257 ++++++++------ evm_arithmetization/src/generation/mod.rs | 37 +- .../src/generation/segments.rs | 20 +- evm_arithmetization/src/generation/state.rs | 9 + evm_arithmetization/src/get_challenges.rs | 4 +- evm_arithmetization/src/proof.rs | 24 +- evm_arithmetization/src/prover.rs | 151 ++++---- evm_arithmetization/src/recursive_verifier.rs | 322 +++++++++++------- evm_arithmetization/src/testing_utils.rs | 24 +- evm_arithmetization/src/verifier.rs | 158 +++++---- evm_arithmetization/src/witness/errors.rs | 7 + evm_arithmetization/src/witness/traces.rs | 31 +- evm_arithmetization/tests/empty_tables.rs | 32 ++ evm_arithmetization/tests/two_to_one_block.rs | 2 +- trace_decoder/src/type2.rs | 3 + trace_decoder/src/wire.rs | 4 - 20 files changed, 718 insertions(+), 444 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ee516001..94c89138c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3645,7 +3645,7 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plonky2" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "ahash", "anyhow", @@ -3656,7 +3656,7 @@ dependencies = [ "log", "num", "plonky2_field", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b)", "plonky2_util", "rand", "rand_chacha", @@ -3669,7 +3669,7 @@ dependencies = [ [[package]] name = "plonky2_field" version = "0.2.2" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "anyhow", "itertools 0.11.0", @@ -3693,7 +3693,7 @@ dependencies = [ [[package]] name = "plonky2_maybe_rayon" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "rayon", ] @@ -3701,7 +3701,7 @@ dependencies = [ [[package]] name = "plonky2_util" version = "0.2.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" [[package]] name = "plotters" @@ -4662,7 +4662,7 @@ checksum = "8acdd7dbfcfb5dd6e46c63512508bf71c2043f70b8f143813ad75cb5e8a589f2" [[package]] name = "starky" version = "0.4.0" -source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee#8463effe0dd1472a52906cd12ffb047885db42ee" +source = "git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b#2488cdacd49ede15737bc1172546d82e9521b79b" dependencies = [ "ahash", "anyhow", @@ -4671,8 +4671,9 @@ dependencies = [ "log", "num-bigint", "plonky2", - "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=8463effe0dd1472a52906cd12ffb047885db42ee)", + "plonky2_maybe_rayon 0.2.0 (git+https://github.com/0xPolygonZero/plonky2.git?rev=2488cdacd49ede15737bc1172546d82e9521b79b)", "plonky2_util", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 842e3c168..2027eb8bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,10 +115,10 @@ zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } # plonky2-related dependencies -plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +plonky2 = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "2488cdacd49ede15737bc1172546d82e9521b79b" } plonky2_maybe_rayon = "0.2.0" -plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } -starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "8463effe0dd1472a52906cd12ffb047885db42ee" } +plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "2488cdacd49ede15737bc1172546d82e9521b79b" } +starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "2488cdacd49ede15737bc1172546d82e9521b79b" } [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" diff --git a/evm_arithmetization/src/all_stark.rs b/evm_arithmetization/src/all_stark.rs index 0ec87f3db..41f4010fe 100644 --- a/evm_arithmetization/src/all_stark.rs +++ b/evm_arithmetization/src/all_stark.rs @@ -106,9 +106,24 @@ pub const NUM_TABLES: usize = if cfg!(feature = "cdk_erigon") { Table::MemAfter as usize + 1 }; -/// Indices of Keccak Tables -pub const KECCAK_TABLES_INDICES: [usize; 2] = - [Table::Keccak as usize, Table::KeccakSponge as usize]; +/// Indices of optional Tables +#[cfg(not(feature = "cdk_erigon"))] +pub const OPTIONAL_TABLE_INDICES: [usize; 5] = [ + Table::BytePacking as usize, + Table::Keccak as usize, + Table::KeccakSponge as usize, + Table::Logic as usize, + Table::MemAfter as usize, +]; +#[cfg(feature = "cdk_erigon")] +pub const OPTIONAL_TABLE_INDICES: [usize; 6] = [ + Table::BytePacking as usize, + Table::Keccak as usize, + Table::KeccakSponge as usize, + Table::Logic as usize, + Table::MemAfter as usize, + Table::Poseidon as usize, +]; impl Table { /// Returns all STARK table indices. @@ -129,6 +144,11 @@ impl Table { } } +/// The total number of CTLs used by the zkEVM. +pub(crate) const NUM_CTLS: usize = if cfg!(feature = "cdk_erigon") { 13 } else { 10 }; +/// The position of the Memory CTL within all CTLs of the zkEVM. +pub(crate) const MEMORY_CTL_IDX: usize = 6; + /// Returns all the `CrossTableLookups` used for proving the EVM. pub(crate) fn all_cross_table_lookups() -> Vec> { vec![ @@ -419,3 +439,17 @@ fn ctl_poseidon_general_output() -> CrossTableLookup { poseidon_stark::ctl_looked_general_output(), ) } + +#[cfg(test)] +mod tests { + use plonky2::field::goldilocks_field::GoldilocksField; + + use super::*; + + type F = GoldilocksField; + + #[test] + fn check_num_ctls() { + assert_eq!(all_cross_table_lookups::().len(), NUM_CTLS); + } +} diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 36a2034ee..b1bb3b4f4 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -58,8 +58,6 @@ pub(crate) struct Interpreter { /// The interpreter will halt only if the current context matches /// halt_context pub(crate) halt_context: Option, - /// Counts the number of appearances of each opcode. For debugging purposes. - pub(crate) opcode_count: HashMap, /// A table of call contexts and the JUMPDEST offsets that they jumped to. jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. @@ -69,6 +67,10 @@ pub(crate) struct Interpreter { pub(crate) clock: usize, /// Log of the maximal number of CPU cycles in one segment execution. max_cpu_len_log: Option, + + #[cfg(test)] + // Counts the number of appearances of each opcode. For debugging purposes. + pub(crate) opcode_count: HashMap, } /// Simulates the CPU execution from `state` until the program counter reaches @@ -241,6 +243,7 @@ impl Interpreter { // while the label `halt` is the halting label in the kernel. halt_offsets: vec![DEFAULT_HALT_OFFSET, KERNEL.global_labels["halt_final"]], halt_context: None, + #[cfg(test)] opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: false, @@ -272,6 +275,7 @@ impl Interpreter { generation_state: state.soft_clone(), halt_offsets: vec![halt_offset], halt_context: Some(halt_context), + #[cfg(test)] opcode_count: HashMap::new(), jumpdest_table: HashMap::new(), is_jumpdest_analysis: true, @@ -491,6 +495,7 @@ impl Interpreter { self.max_cpu_len_log } + #[cfg(test)] pub(crate) fn reset_opcode_counts(&mut self) { self.opcode_count = HashMap::new(); } @@ -732,8 +737,11 @@ impl State for Interpreter { let op = decode(registers, opcode)?; - // Increment the opcode count - *self.opcode_count.entry(op).or_insert(0) += 1; + #[cfg(test)] + { + // Increment the opcode count + *self.opcode_count.entry(op).or_insert(0) += 1; + } fill_op_flag(op, &mut row); diff --git a/evm_arithmetization/src/fixed_recursive_verifier.rs b/evm_arithmetization/src/fixed_recursive_verifier.rs index 96ecce6d3..adde60d1b 100644 --- a/evm_arithmetization/src/fixed_recursive_verifier.rs +++ b/evm_arithmetization/src/fixed_recursive_verifier.rs @@ -37,7 +37,8 @@ use starky::proof::StarkProofWithMetadata; use starky::stark::Stark; use crate::all_stark::{ - all_cross_table_lookups, AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES, + all_cross_table_lookups, AllStark, Table, MEMORY_CTL_IDX, NUM_CTLS, NUM_TABLES, + OPTIONAL_TABLE_INDICES, }; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::segments::{GenerationSegmentData, SegmentDataIterator}; @@ -155,8 +156,8 @@ where /// for EVM root proofs; the circuit has them just to match the /// structure of aggregation proofs. cyclic_vk: VerifierCircuitTarget, - /// We can skip verifying Keccak tables when they are not in use. - use_keccak_tables: BoolTarget, + /// We can skip verifying tables when they are not in use. + table_in_use: [BoolTarget; NUM_TABLES], } impl RootCircuitData @@ -179,7 +180,9 @@ where } self.public_values.to_buffer(buffer)?; buffer.write_target_verifier_circuit(&self.cyclic_vk)?; - buffer.write_target_bool(self.use_keccak_tables)?; + for table_in_use in self.table_in_use { + buffer.write_target_bool(table_in_use)?; + } Ok(()) } @@ -199,7 +202,10 @@ where } let public_values = PublicValuesTarget::from_buffer(buffer)?; let cyclic_vk = buffer.read_target_verifier_circuit()?; - let use_keccak_tables = buffer.read_target_bool()?; + let mut table_in_use = Vec::with_capacity(NUM_TABLES); + for _ in 0..NUM_TABLES { + table_in_use.push(buffer.read_target_bool()?); + } Ok(Self { circuit, @@ -207,7 +213,7 @@ where index_verifier_data: index_verifier_data.try_into().unwrap(), public_values, cyclic_vk, - use_keccak_tables, + table_in_use: table_in_use.try_into().unwrap(), }) } } @@ -831,26 +837,24 @@ where let block_wrapper = Self::create_block_wrapper_circuit(&block); let two_to_one_block = Self::create_two_to_one_block_circuit(&block_wrapper); - // TODO(sdeng): enable more optional Tables let table_dummy_proofs = core::array::from_fn(|i| { - if KECCAK_TABLES_INDICES.contains(&i) { + if OPTIONAL_TABLE_INDICES.contains(&i) { let init_degree = degree_bits_ranges[i].start; - let common_circuit_data = by_table[i] + let chain = by_table[i] .by_stark_size .get(&init_degree) - .expect("Unable to get the shrinking circuits") + .expect("Unable to get the shrinking circuits"); + let common_circuit_data = chain .shrinking_wrappers .last() - .expect("Unable to get the last shrinking circuit") - .circuit - .common - .clone(); - let dummy_circuit: CircuitData = dummy_circuit(&common_circuit_data); + .map(|wrapper| &wrapper.circuit.common) + .unwrap_or(&chain.initial_wrapper.circuit.common); + let dummy_circuit: CircuitData = dummy_circuit(common_circuit_data); let dummy_pis = HashMap::new(); let proof = dummy_proof(&dummy_circuit, dummy_pis) .expect("Unable to generate dummy proofs"); Some(ShrunkProofData { - common_circuit_data, + common_circuit_data: common_circuit_data.clone(), proof, }) } else { @@ -899,8 +903,10 @@ where let mut builder = CircuitBuilder::new(CircuitConfig::standard_recursion_config()); - let use_keccak_tables = builder.add_virtual_bool_target_safe(); - let skip_keccak_tables = builder.not(use_keccak_tables); + let table_in_use: [BoolTarget; NUM_TABLES] = + core::array::from_fn(|_| builder.add_virtual_bool_target_safe()); + let table_not_in_use: [BoolTarget; NUM_TABLES] = + core::array::from_fn(|i| builder.not(table_in_use[i])); let public_values = add_virtual_public_values_public_input(&mut builder); let recursive_proofs = @@ -920,11 +926,17 @@ where } } + for (i, table) in table_in_use.iter().enumerate() { + if !OPTIONAL_TABLE_INDICES.contains(&i) { + builder.assert_one(table.target); + } + } + // Ensures that the trace cap is set to 0 when skipping Keccak tables. - for i in KECCAK_TABLES_INDICES { + for i in OPTIONAL_TABLE_INDICES { for h in &pis[i].trace_cap { for t in h { - let trace_cap_check = builder.mul(skip_keccak_tables.target, *t); + let trace_cap_check = builder.mul(table_not_in_use[i].target, *t); builder.assert_zero(trace_cap_check); } } @@ -940,16 +952,16 @@ where // Check that the correct CTL challenges are used in every proof. for (i, pi) in pis.iter().enumerate() { for j in 0..stark_config.num_challenges { - if KECCAK_TABLES_INDICES.contains(&i) { - // Ensures that the correct CTL challenges are used in Keccak tables when - // `enable_keccak_tables` is true. + if OPTIONAL_TABLE_INDICES.contains(&i) { + // Ensures that the correct CTL challenges are used when an optional table + // is in use. builder.conditional_assert_eq( - use_keccak_tables.target, + table_in_use[i].target, ctl_challenges.challenges[j].beta, pi.ctl_challenges.challenges[j].beta, ); builder.conditional_assert_eq( - use_keccak_tables.target, + table_in_use[i].target, ctl_challenges.challenges[j].gamma, pi.ctl_challenges.challenges[j].gamma, ); @@ -977,18 +989,18 @@ where let current_state_before = pis[i].challenger_state_before.as_ref(); let current_state_after = pis[i].challenger_state_after.as_ref(); for j in 0..state_len { - if KECCAK_TABLES_INDICES.contains(&i) { + if OPTIONAL_TABLE_INDICES.contains(&i) { // Ensure the challenger state: - // 1) prev == current_before when using Keccak + // 1) prev == current_before when using this table builder.conditional_assert_eq( - use_keccak_tables.target, + table_in_use[i].target, prev_state[j], current_state_before[j], ); - // 2) Update prev <- current_after when using Keccak - // 3) Keep prev <- prev when skipping Keccak + // 2) Update prev <- current_after when using this table + // 3) Keep prev <- prev when skipping this table prev_state[j] = - builder.select(use_keccak_tables, current_state_after[j], prev_state[j]); + builder.select(table_in_use[i], current_state_after[j], prev_state[j]); } else { builder.connect(prev_state[j], current_state_before[j]); prev_state[j] = current_state_after[j]; @@ -998,25 +1010,28 @@ where // Extra sums to add to the looked last value. // Only necessary for the Memory values. - let mut extra_looking_sums = - vec![vec![builder.zero(); stark_config.num_challenges]; NUM_TABLES]; + let mut extra_looking_sums = HashMap::from_iter( + (0..NUM_CTLS).map(|i| (i, vec![builder.zero(); stark_config.num_challenges])), + ); // Memory - extra_looking_sums[*Table::Memory] = (0..stark_config.num_challenges) - .map(|c| { - get_memory_extra_looking_sum_circuit( - &mut builder, - &public_values, - ctl_challenges.challenges[c], - ) - }) - .collect_vec(); + extra_looking_sums.insert( + MEMORY_CTL_IDX, + (0..stark_config.num_challenges) + .map(|c| { + get_memory_extra_looking_sum_circuit( + &mut builder, + &public_values, + ctl_challenges.challenges[c], + ) + }) + .collect_vec(), + ); - // Ensure that when Keccak tables are skipped, the Keccak tables' ctl_zs_first - // are all zeros. - for &i in KECCAK_TABLES_INDICES.iter() { + // Ensure that when a table is skipped, the table's ctl_zs_first are all zeros. + for &i in OPTIONAL_TABLE_INDICES.iter() { for &t in pis[i].ctl_zs_first.iter() { - let ctl_check = builder.mul(skip_keccak_tables.target, t); + let ctl_check = builder.mul(table_not_in_use[i].target, t); builder.assert_zero(ctl_check); } } @@ -1026,7 +1041,7 @@ where &mut builder, all_cross_table_lookups(), pis.map(|p| p.ctl_zs_first), - Some(&extra_looking_sums), + &extra_looking_sums, stark_config, ); @@ -1050,10 +1065,10 @@ where let inner_verifier_data = builder.random_access_verifier_data(index_verifier_data[i], possible_vks); - if KECCAK_TABLES_INDICES.contains(&i) { + if OPTIONAL_TABLE_INDICES.contains(&i) { builder .conditionally_verify_proof_or_dummy::( - use_keccak_tables, + table_in_use[i], &recursive_proofs[i], &inner_verifier_data, inner_common_data[i], @@ -1096,7 +1111,7 @@ where index_verifier_data, public_values, cyclic_vk, - use_keccak_tables, + table_in_use, } } @@ -2009,16 +2024,7 @@ where for table in 0..NUM_TABLES { let table_circuits = &self.by_table[table]; - if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - let dummy_proof_data = self.table_dummy_proofs[table] - .as_ref() - .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; - root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); - root_inputs.set_proof_with_pis_target( - &self.root.proof_with_pis[table], - &dummy_proof_data.proof, - ); - } else { + if all_proof.table_in_use[table] { let stark_proof = &all_proof.multi_proof.stark_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; @@ -2042,9 +2048,19 @@ where root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_usize(index_verifier_data), - ); + )?; root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof)?; + } else { + assert!(OPTIONAL_TABLE_INDICES.contains(&table)); + let dummy_proof_data = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("No dummy_proof_data"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof_data.proof, + )?; } check_abort_signal(abort_signal.clone())?; @@ -2053,7 +2069,7 @@ where root_inputs.set_verifier_data_target( &self.root.cyclic_vk, &self.segment_aggregation.circuit.verifier_only, - ); + )?; set_public_value_targets( &mut root_inputs, @@ -2064,7 +2080,11 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); + self.root + .table_in_use + .iter() + .zip(all_proof.table_in_use.iter()) + .try_for_each(|(target, value)| root_inputs.set_bool_target(*target, *value))?; let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2137,30 +2157,31 @@ where let mut root_inputs = PartialWitness::new(); for table in 0..NUM_TABLES { - if KECCAK_TABLES_INDICES.contains(&table) && !all_proof.use_keccak_tables { - let dummy_proof = self.table_dummy_proofs[table] - .as_ref() - .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; - root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO); - root_inputs.set_proof_with_pis_target( - &self.root.proof_with_pis[table], - &dummy_proof.proof, - ); - } else { + if all_proof.table_in_use[table] { let (table_circuit, index_verifier_data) = &table_circuits[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get circuits"))?; root_inputs.set_target( self.root.index_verifier_data[table], F::from_canonical_u8(*index_verifier_data), - ); + )?; let stark_proof = all_proof.multi_proof.stark_proofs[table] .as_ref() .ok_or_else(|| anyhow::format_err!("Unable to get stark proof"))?; let shrunk_proof = table_circuit.shrink(stark_proof, &all_proof.multi_proof.ctl_challenges)?; root_inputs - .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof); + .set_proof_with_pis_target(&self.root.proof_with_pis[table], &shrunk_proof)?; + } else { + assert!(OPTIONAL_TABLE_INDICES.contains(&table)); + let dummy_proof = self.table_dummy_proofs[table] + .as_ref() + .ok_or_else(|| anyhow::format_err!("Unable to get dummpy proof"))?; + root_inputs.set_target(self.root.index_verifier_data[table], F::ZERO)?; + root_inputs.set_proof_with_pis_target( + &self.root.proof_with_pis[table], + &dummy_proof.proof, + )?; } check_abort_signal(abort_signal.clone())?; @@ -2169,7 +2190,7 @@ where root_inputs.set_verifier_data_target( &self.root.cyclic_vk, &self.segment_aggregation.circuit.verifier_only, - ); + )?; set_public_value_targets( &mut root_inputs, @@ -2180,7 +2201,11 @@ where anyhow::Error::msg("Invalid conversion when setting public values targets.") })?; - root_inputs.set_bool_target(self.root.use_keccak_tables, all_proof.use_keccak_tables); + self.root + .table_in_use + .iter() + .zip(all_proof.table_in_use.iter()) + .try_for_each(|(target, value)| root_inputs.set_bool_target(*target, *value))?; let root_proof = self.root.circuit.prove(root_inputs)?; @@ -2229,7 +2254,7 @@ where &self.segment_aggregation.circuit, &mut agg_inputs, lhs_proof, - ); + )?; // If rhs is dummy, the rhs proof is also set to be the lhs. let real_rhs_proof = if rhs_is_dummy { lhs_proof } else { rhs_proof }; @@ -2241,12 +2266,12 @@ where &self.segment_aggregation.circuit, &mut agg_inputs, real_rhs_proof, - ); + )?; agg_inputs.set_verifier_data_target( &self.segment_aggregation.cyclic_vk, &self.segment_aggregation.circuit.verifier_only, - ); + )?; // Aggregates both `PublicValues` from the provided proofs into a single one. let lhs_public_values = &lhs.proof_with_pvs.public_values; @@ -2339,7 +2364,7 @@ where &self.batch_aggregation.circuit, &mut batch_inputs, &lhs.intern, - ); + )?; Self::set_dummy_if_necessary( &self.batch_aggregation.rhs, @@ -2347,12 +2372,12 @@ where &self.batch_aggregation.circuit, &mut batch_inputs, &rhs.intern, - ); + )?; batch_inputs.set_verifier_data_target( &self.batch_aggregation.cyclic_vk, &self.batch_aggregation.circuit.verifier_only, - ); + )?; let lhs_pvs = &lhs.public_values; let batch_public_values = PublicValues { @@ -2393,20 +2418,20 @@ where circuit: &CircuitData, agg_inputs: &mut PartialWitness, proof: &ProofWithPublicInputs, - ) { - agg_inputs.set_bool_target(agg_child.is_agg, is_agg); - agg_inputs.set_bool_target(agg_child.is_dummy, is_dummy); + ) -> anyhow::Result<()> { + agg_inputs.set_bool_target(agg_child.is_agg, is_agg)?; + agg_inputs.set_bool_target(agg_child.is_dummy, is_dummy)?; if is_agg { - agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof)?; } else { Self::set_dummy_proof_with_cyclic_vk_pis( circuit, agg_inputs, &agg_child.agg_proof, proof, - ); + )?; } - agg_inputs.set_proof_with_pis_target(&agg_child.real_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.real_proof, proof) } /// Create a final block proof, once all transactions of a given block have @@ -2438,10 +2463,10 @@ where block_inputs.set_bool_target( self.block.has_parent_block, opt_parent_block_proof.is_some(), - ); + )?; if let Some(parent_block_proof) = opt_parent_block_proof { block_inputs - .set_proof_with_pis_target(&self.block.parent_block_proof, parent_block_proof); + .set_proof_with_pis_target(&self.block.parent_block_proof, parent_block_proof)?; } else { if agg_root_proof.public_values.trie_roots_before.state_root != agg_root_proof @@ -2584,13 +2609,14 @@ where &self.block.circuit.verifier_only, nonzero_pis, ), - ); + )?; } - block_inputs.set_proof_with_pis_target(&self.block.agg_root_proof, &agg_root_proof.intern); + block_inputs + .set_proof_with_pis_target(&self.block.agg_root_proof, &agg_root_proof.intern)?; block_inputs - .set_verifier_data_target(&self.block.cyclic_vk, &self.block.circuit.verifier_only); + .set_verifier_data_target(&self.block.cyclic_vk, &self.block.circuit.verifier_only)?; // This is basically identical to this block public values, apart from the // `trie_roots_before` that may come from the previous proof, if any. @@ -2649,13 +2675,15 @@ where )> { let mut block_wrapper_inputs = PartialWitness::new(); - block_wrapper_inputs - .set_proof_with_pis_target(&self.block_wrapper.parent_block_proof, &block_proof.intern); + block_wrapper_inputs.set_proof_with_pis_target( + &self.block_wrapper.parent_block_proof, + &block_proof.intern, + )?; block_wrapper_inputs.set_verifier_data_target( &self.block_wrapper.cyclic_vk, // dummy &self.block_wrapper.circuit.verifier_only, - ); + )?; let final_pvs = block_proof.public_values.clone().into(); set_final_public_value_targets( @@ -2709,7 +2737,7 @@ where &self.two_to_one_block.circuit, &mut witness, lhs, - ); + )?; Self::set_dummy_if_necessary( &self.two_to_one_block.rhs, @@ -2717,15 +2745,14 @@ where &self.two_to_one_block.circuit, &mut witness, rhs, - ); + )?; witness.set_verifier_data_target( &self.two_to_one_block.cyclic_vk, &self.two_to_one_block.circuit.verifier_only, - ); + )?; - let proof = self.two_to_one_block.circuit.prove(witness)?; - Ok(proof) + self.two_to_one_block.circuit.prove(witness) } /// Verifies an existing block aggregation proof. @@ -2756,7 +2783,7 @@ where witness: &mut PartialWitness, agg_proof_with_pis: &ProofWithPublicInputsTarget, base_proof_with_pis: &ProofWithPublicInputs, - ) { + ) -> anyhow::Result<()> { let ProofWithPublicInputs { proof: base_proof, public_inputs: _, @@ -2767,7 +2794,7 @@ where } = agg_proof_with_pis; // The proof remains the same. - witness.set_proof_target(agg_proof_targets, base_proof); + witness.set_proof_target(agg_proof_targets, base_proof)?; let cyclic_verifying_data = &circuit_agg.verifier_only; let mut cyclic_vk = cyclic_verifying_data.circuit_digest.to_vec(); @@ -2778,8 +2805,10 @@ where // Set dummy public inputs. for (&pi_t, pi) in agg_pi_targets.iter().zip_eq(dummy_pis) { - witness.set_target(pi_t, pi); + witness.set_target(pi_t, pi)?; } + + Ok(()) } /// If the [`AggregationChild`] is a base proof and not an aggregation @@ -2794,19 +2823,19 @@ where circuit: &CircuitData, agg_inputs: &mut PartialWitness, proof: &ProofWithPublicInputs, - ) { - agg_inputs.set_bool_target(agg_child.is_agg, is_agg); + ) -> anyhow::Result<()> { + agg_inputs.set_bool_target(agg_child.is_agg, is_agg)?; if is_agg { - agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.agg_proof, proof)?; } else { Self::set_dummy_proof_with_cyclic_vk_pis( circuit, agg_inputs, &agg_child.agg_proof, proof, - ); + )?; } - agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof); + agg_inputs.set_proof_with_pis_target(&agg_child.base_proof, proof) } } @@ -3038,6 +3067,12 @@ where }); } + log::debug!( + "Table: {:?}, degree: {}, shrinking_wrappers_len: {}", + table, + degree_bits, + shrinking_wrappers.len() + ); Self { initial_wrapper, shrinking_wrappers, diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index dee3adaf5..66253e4a4 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -20,7 +20,8 @@ use GlobalMetadata::{ StateTrieRootDigestBefore, TransactionTrieRootDigestAfter, TransactionTrieRootDigestBefore, }; -use crate::all_stark::{AllStark, NUM_TABLES}; +use crate::all_stark::Table::MemAfter; +use crate::all_stark::{AllStark, Table, NUM_TABLES, OPTIONAL_TABLE_INDICES}; use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; @@ -499,7 +500,7 @@ fn get_all_memory_address_and_values(memory_before: &MemoryState) -> Vec<(Memory pub struct TablesWithPVs { pub tables: [Vec>; NUM_TABLES], - pub use_keccak_tables: bool, + pub table_in_use: [bool; NUM_TABLES], pub public_values: PublicValues, } @@ -594,9 +595,30 @@ pub fn generate_traces, const D: usize>( mem_after: MemCap::default(), }; - let use_keccak_tables = !state.traces.keccak_inputs.is_empty(); + let mut table_in_use = [true; NUM_TABLES]; + if state.traces.keccak_inputs.is_empty() && OPTIONAL_TABLE_INDICES.contains(&Table::Keccak) { + assert!(OPTIONAL_TABLE_INDICES.contains(&Table::KeccakSponge)); + log::debug!("Keccak and KeccakSponge tables not in use"); + table_in_use[*Table::Keccak] = false; + table_in_use[*Table::KeccakSponge] = false; + } + if state.traces.logic_ops.is_empty() && OPTIONAL_TABLE_INDICES.contains(&Table::Logic) { + log::debug!("Logic table not in use"); + table_in_use[*Table::Logic] = false; + } + if state.traces.byte_packing_ops.is_empty() + && OPTIONAL_TABLE_INDICES.contains(&Table::BytePacking) + { + log::debug!("BytePacking table not in use"); + table_in_use[*Table::BytePacking] = false; + } + #[cfg(feature = "cdk_erigon")] + if state.traces.poseidon_ops.is_empty() && OPTIONAL_TABLE_INDICES.contains(&Table::Poseidon) { + log::debug!("Poseidon table not in use"); + table_in_use[*Table::Poseidon] = false; + } - let tables = timed!( + let (tables, final_len) = timed!( timing, "convert trace data to tables", state.traces.into_tables( @@ -609,9 +631,14 @@ pub fn generate_traces, const D: usize>( ) ); + if final_len == 0 && OPTIONAL_TABLE_INDICES.contains(&MemAfter) { + log::debug!("MemAfter table not in use"); + table_in_use[*MemAfter] = false; + } + Ok(TablesWithPVs { tables, - use_keccak_tables, + table_in_use, public_values, }) } diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index 36bac8f8a..1df63af29 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -1,6 +1,7 @@ //! Module defining the logic around proof segmentation into chunks, //! which allows what is commonly known as zk-continuations. +#[cfg(test)] use std::collections::HashMap; use anyhow::Result; @@ -13,6 +14,7 @@ use crate::cpu::kernel::interpreter::{set_registers_and_run, ExtraSegmentData, I use crate::generation::state::State; use crate::generation::{collect_debug_tries, debug_inputs, ErrorWithTries, GenerationInputs}; use crate::witness::memory::MemoryState; +#[cfg(test)] use crate::witness::operation::Operation; use crate::witness::state::RegistersState; @@ -32,7 +34,9 @@ pub struct GenerationSegmentData { pub(crate) extra_data: ExtraSegmentData, /// Log of the maximal cpu length. pub(crate) max_cpu_len_log: Option, - /// Counts the number of appearances of each opcode. For debugging purposes. + + #[cfg(test)] + // Counts the number of appearances of each opcode. For debugging purposes. pub(crate) opcode_counts: HashMap, } @@ -82,6 +86,7 @@ fn build_segment_data( access_lists_ptrs: interpreter.generation_state.access_lists_ptrs.clone(), state_ptrs: interpreter.generation_state.state_ptrs.clone(), }, + #[cfg(test)] opcode_counts: interpreter.opcode_count.clone(), } } @@ -139,8 +144,11 @@ impl SegmentDataIterator { let segment_index = segment_data.segment_index; - // Reset opcode counts before executing the segment - self.interpreter.reset_opcode_counts(); + #[cfg(test)] + { + // Reset opcode counts before executing the segment + self.interpreter.reset_opcode_counts(); + } // Run the interpreter to get `registers_after` and the partial data for the // next segment. @@ -156,7 +164,11 @@ impl SegmentDataIterator { )); segment_data.registers_after = updated_registers; - segment_data.opcode_counts = self.interpreter.opcode_count.clone(); + + #[cfg(test)] + { + segment_data.opcode_counts = self.interpreter.opcode_count.clone(); + } Ok(Some(Box::new((segment_data, partial_segment_data)))) } else { diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index d8cc3e93b..1ff7b3f7c 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -189,6 +189,15 @@ pub(crate) trait State { { let halt_offsets = self.get_halt_offsets(); + if let Some(max_len_log) = max_cpu_len_log { + assert!( + (1 << max_len_log) >= NUM_EXTRA_CYCLES_AFTER, + "Target length (2^{}) is less than NUM_EXTRA_CYCLES_AFTER ({})", + max_len_log, + NUM_EXTRA_CYCLES_AFTER + ); + } + let cycle_limit = max_cpu_len_log.map(|max_len_log| (1 << max_len_log) - NUM_EXTRA_CYCLES_AFTER); diff --git a/evm_arithmetization/src/get_challenges.rs b/evm_arithmetization/src/get_challenges.rs index c7471471b..23a46d5c5 100644 --- a/evm_arithmetization/src/get_challenges.rs +++ b/evm_arithmetization/src/get_challenges.rs @@ -254,7 +254,7 @@ pub mod testing { use starky::lookup::{get_grand_product_challenge_set, GrandProductChallengeSet}; use starky::proof::StarkProofChallenges; - use crate::all_stark::KECCAK_TABLES_INDICES; + use crate::all_stark::OPTIONAL_TABLE_INDICES; use crate::get_challenges::observe_public_values; use crate::proof::*; use crate::witness::errors::ProgramError; @@ -282,7 +282,7 @@ pub mod testing { if let Some(stark_proof) = stark_proof { challenger.observe_cap(&stark_proof.proof.trace_cap); } else { - assert!(KECCAK_TABLES_INDICES.contains(&i) && !self.use_keccak_tables); + assert!(OPTIONAL_TABLE_INDICES.contains(&i) && !self.table_in_use[i]); let zero_cap = vec![F::ZERO; config.fri_config.num_cap_elements() * NUM_HASH_OUT_ELTS]; challenger.observe_elements(&zero_cap); diff --git a/evm_arithmetization/src/proof.rs b/evm_arithmetization/src/proof.rs index b0de028af..f501a96d3 100644 --- a/evm_arithmetization/src/proof.rs +++ b/evm_arithmetization/src/proof.rs @@ -4,6 +4,7 @@ use ethereum_types::{Address, H256, U256}; use itertools::Itertools; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::{HashOutTarget, MerkleCapTarget, RichField, NUM_HASH_OUT_ELTS}; +use plonky2::hash::merkle_tree::MerkleCap; use plonky2::iop::target::{BoolTarget, Target}; use plonky2::plonk::circuit_builder::CircuitBuilder; use plonky2::plonk::config::{GenericConfig, GenericHashOut, Hasher}; @@ -47,10 +48,9 @@ pub struct AllProof, C: GenericConfig, co pub multi_proof: MultiProof, /// Public memory values used for the recursive proofs. pub public_values: PublicValues, - /// A flag indicating whether the Keccak and KeccakSponge tables contain - /// only padding values (i.e., no meaningful data). This is set to false - /// when no actual Keccak operations were performed. - pub use_keccak_tables: bool, + /// A flag indicating whether the table only contains padding values (i.e., + /// no meaningful data). + pub table_in_use: [bool; NUM_TABLES], } impl, C: GenericConfig, const D: usize> AllProof { @@ -603,6 +603,22 @@ impl MemCap { Self { mem_cap } } + + pub fn from_merkle_cap>(merkle_cap: MerkleCap) -> Self { + let mem_cap = merkle_cap + .0 + .iter() + .map(|h| { + h.to_vec() + .iter() + .map(|hi| hi.to_canonical_u64().into()) + .collect::>() + .try_into() + .unwrap() + }) + .collect::>(); + Self { mem_cap } + } } /// Memory values which are public. diff --git a/evm_arithmetization/src/prover.rs b/evm_arithmetization/src/prover.rs index 4192db6c8..698c23934 100644 --- a/evm_arithmetization/src/prover.rs +++ b/evm_arithmetization/src/prover.rs @@ -2,15 +2,15 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use anyhow::{anyhow, Result}; +use ethereum_types::U256; use itertools::Itertools; use once_cell::sync::Lazy; use plonky2::field::extension::Extendable; use plonky2::field::polynomial::PolynomialValues; use plonky2::fri::oracle::PolynomialBatch; use plonky2::hash::hash_types::RichField; -use plonky2::hash::merkle_tree::MerkleCap; use plonky2::iop::challenger::Challenger; -use plonky2::plonk::config::{GenericConfig, GenericHashOut}; +use plonky2::plonk::config::GenericConfig; use plonky2::timed; use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; @@ -20,7 +20,7 @@ use starky::proof::StarkProofWithMetadata; use starky::prover::prove_with_commitment; use starky::stark::Stark; -use crate::all_stark::{AllStark, Table, KECCAK_TABLES_INDICES, NUM_TABLES}; +use crate::all_stark::{AllStark, Table, NUM_TABLES, OPTIONAL_TABLE_INDICES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::generation::{generate_traces, GenerationInputs, TrimmedGenerationInputs}; use crate::get_challenges::observe_public_values; @@ -59,7 +59,7 @@ where all_stark, config, tables_with_pvs.tables, - tables_with_pvs.use_keccak_tables, + tables_with_pvs.table_in_use, &mut tables_with_pvs.public_values, timing, abort_signal, @@ -73,7 +73,7 @@ pub(crate) fn prove_with_traces( all_stark: &AllStark, config: &StarkConfig, trace_poly_values: [Vec>; NUM_TABLES], - use_keccak_tables: bool, + table_in_use: [bool; NUM_TABLES], public_values: &mut PublicValues, timing: &mut TimingTree, abort_signal: Option>, @@ -117,7 +117,7 @@ where .collect::>(); let mut challenger = Challenger::::new(); for (i, cap) in trace_caps.iter().enumerate() { - if KECCAK_TABLES_INDICES.contains(&i) && !use_keccak_tables { + if OPTIONAL_TABLE_INDICES.contains(&i) && !table_in_use[i] { // Observe zero merkle caps when skipping Keccak tables. let zero_merkle_cap = cap.flatten().iter().map(|_| F::ZERO).collect::>(); challenger.observe_elements(&zero_merkle_cap); @@ -151,7 +151,7 @@ where config, &trace_poly_values, trace_commitments, - use_keccak_tables, + table_in_use, ctl_data_per_table, &mut challenger, &ctl_challenges, @@ -159,34 +159,8 @@ where abort_signal, )? ); - public_values.mem_before = MemCap { - mem_cap: mem_before_cap - .0 - .iter() - .map(|h| { - h.to_vec() - .iter() - .map(|hi| hi.to_canonical_u64().into()) - .collect::>() - .try_into() - .unwrap() - }) - .collect::>(), - }; - public_values.mem_after = MemCap { - mem_cap: mem_after_cap - .0 - .iter() - .map(|h| { - h.to_vec() - .iter() - .map(|hi| hi.to_canonical_u64().into()) - .collect::>() - .try_into() - .unwrap() - }) - .collect::>(), - }; + public_values.mem_before = mem_before_cap; + public_values.mem_after = mem_after_cap; // This is an expensive check, hence is only run when `debug_assertions` are // enabled. @@ -215,14 +189,14 @@ where ctl_challenges, }, public_values: public_values.clone(), - use_keccak_tables, + table_in_use, }) } -type ProofWithMemCaps = ( +type ProofWithMemCaps = ( [Option>; NUM_TABLES], - MerkleCap, - MerkleCap, + MemCap, + MemCap, ); /// Generates a proof for each STARK. @@ -239,80 +213,85 @@ fn prove_with_commitments( config: &StarkConfig, trace_poly_values: &[Vec>; NUM_TABLES], trace_commitments: Vec>, - use_keccak_tables: bool, + table_in_use: [bool; NUM_TABLES], ctl_data_per_table: [CtlData; NUM_TABLES], challenger: &mut Challenger, ctl_challenges: &GrandProductChallengeSet, timing: &mut TimingTree, abort_signal: Option>, -) -> Result> +) -> Result> where F: RichField + Extendable, C: GenericConfig, { macro_rules! prove_table { ($stark:ident, $table:expr) => { - timed!( - timing, - &format!("prove {} STARK", stringify!($stark)), - prove_single_table( - &all_stark.$stark, - config, - &trace_poly_values[*$table], - &trace_commitments[*$table], - &ctl_data_per_table[*$table], - ctl_challenges, - challenger, + if table_in_use[*$table] { + Some(timed!( timing, - abort_signal.clone(), - )? - ) + &format!("prove {} STARK", stringify!($stark)), + prove_single_table( + &all_stark.$stark, + config, + &trace_poly_values[*$table], + &trace_commitments[*$table], + &ctl_data_per_table[*$table], + ctl_challenges, + challenger, + timing, + abort_signal.clone(), + )? + )) + } else { + None + } }; } - let (arithmetic_proof, _) = prove_table!(arithmetic_stark, Table::Arithmetic); - let (byte_packing_proof, _) = prove_table!(byte_packing_stark, Table::BytePacking); - let (cpu_proof, _) = prove_table!(cpu_stark, Table::Cpu); - let keccak_proof = if use_keccak_tables { - Some(prove_table!(keccak_stark, Table::Keccak).0) - } else { - None - }; - let keccak_sponge_proof = if use_keccak_tables { - Some(prove_table!(keccak_sponge_stark, Table::KeccakSponge).0) - } else { - None - }; - let (logic_proof, _) = prove_table!(logic_stark, Table::Logic); - let (memory_proof, _) = prove_table!(memory_stark, Table::Memory); - let (mem_before_proof, mem_before_cap) = prove_table!(mem_before_stark, Table::MemBefore); - let (mem_after_proof, mem_after_cap) = prove_table!(mem_after_stark, Table::MemAfter); + let arithmetic_proof = prove_table!(arithmetic_stark, Table::Arithmetic); + let byte_packing_proof = prove_table!(byte_packing_stark, Table::BytePacking); + let cpu_proof = prove_table!(cpu_stark, Table::Cpu); + let keccak_proof = prove_table!(keccak_stark, Table::Keccak); + let keccak_sponge_proof = prove_table!(keccak_sponge_stark, Table::KeccakSponge); + let logic_proof = prove_table!(logic_stark, Table::Logic); + let memory_proof = prove_table!(memory_stark, Table::Memory); + let mem_before_proof = prove_table!(mem_before_stark, Table::MemBefore); + let mem_after_proof = prove_table!(mem_after_stark, Table::MemAfter); + + let mem_before_cap = trace_commitments[*Table::MemBefore].merkle_tree.cap.clone(); + let mem_before_cap = MemCap::from_merkle_cap(mem_before_cap); + let mem_after_cap = trace_commitments[*Table::MemAfter].merkle_tree.cap.clone(); + let mut mem_after_cap = MemCap::from_merkle_cap(mem_after_cap); + if !table_in_use[*Table::MemAfter] { + for hash in &mut mem_after_cap.mem_cap { + for element in hash.iter_mut() { + *element = U256::zero(); + } + } + } #[cfg(feature = "cdk_erigon")] - let (poseidon_proof, _) = prove_table!(poseidon_stark, Table::Poseidon); + let poseidon_proof = prove_table!(poseidon_stark, Table::Poseidon); Ok(( [ - Some(arithmetic_proof), - Some(byte_packing_proof), - Some(cpu_proof), + arithmetic_proof, + byte_packing_proof, + cpu_proof, keccak_proof, keccak_sponge_proof, - Some(logic_proof), - Some(memory_proof), - Some(mem_before_proof), - Some(mem_after_proof), + logic_proof, + memory_proof, + mem_before_proof, + mem_after_proof, #[cfg(feature = "cdk_erigon")] - Some(poseidon_proof), + poseidon_proof, ], mem_before_cap, mem_after_cap, )) } -type ProofSingleWithCap = - (StarkProofWithMetadata, MerkleCap); - /// Computes a proof for a single STARK table, including: /// - the initial state of the challenger, /// - all the requires Merkle caps, @@ -329,7 +308,7 @@ pub(crate) fn prove_single_table( challenger: &mut Challenger, timing: &mut TimingTree, abort_signal: Option>, -) -> Result> +) -> Result> where F: RichField + Extendable, C: GenericConfig, @@ -356,7 +335,7 @@ where init_challenger_state, })?; - Ok((proof, trace_commitment.merkle_tree.cap.clone())) + Ok(proof) } /// Utility method that checks whether a kill signal has been emitted by one of diff --git a/evm_arithmetization/src/recursive_verifier.rs b/evm_arithmetization/src/recursive_verifier.rs index 14ed5957e..f70b9f166 100644 --- a/evm_arithmetization/src/recursive_verifier.rs +++ b/evm_arithmetization/src/recursive_verifier.rs @@ -156,7 +156,7 @@ where &self.stark_proof_target, &proof_with_metadata.proof, self.zero_target, - ); + )?; for (challenge_target, challenge) in self .ctl_challenges_target @@ -164,14 +164,14 @@ where .iter() .zip(&ctl_challenges.challenges) { - inputs.set_target(challenge_target.beta, challenge.beta); - inputs.set_target(challenge_target.gamma, challenge.gamma); + inputs.set_target(challenge_target.beta, challenge.beta)?; + inputs.set_target(challenge_target.gamma, challenge.gamma)?; } inputs.set_target_arr( self.init_challenger_state_target.as_ref(), proof_with_metadata.init_challenger_state.as_ref(), - ); + )?; self.circuit.prove(inputs) } @@ -199,7 +199,7 @@ where proof: &ProofWithPublicInputs, ) -> Result> { let mut inputs = PartialWitness::new(); - inputs.set_proof_with_pis_target(&self.proof_with_pis_target, proof); + inputs.set_proof_with_pis_target(&self.proof_with_pis_target, proof)?; self.circuit.prove(inputs) } } @@ -842,12 +842,12 @@ where witness, &public_values_target.trie_roots_before, &public_values.trie_roots_before, - ); + )?; set_trie_roots_target( witness, &public_values_target.trie_roots_after, &public_values.trie_roots_after, - ); + )?; set_block_metadata_target( witness, &public_values_target.block_metadata, @@ -857,7 +857,7 @@ where witness, &public_values_target.block_hashes, &public_values.block_hashes, - ); + )?; set_extra_public_values_target( witness, &public_values_target.extra_block_data, @@ -906,10 +906,12 @@ where H: Hasher, W: Witness, { - witness.set_target( - public_values_target.chain_id, - F::from_canonical_u64(public_values.chain_id.low_u64()), - ); + witness + .set_target( + public_values_target.chain_id, + F::from_canonical_u64(public_values.chain_id.low_u64()), + ) + .map_err(ProgramError::from)?; for (i, limb) in public_values .checkpoint_state_trie_root @@ -918,14 +920,18 @@ where .into_iter() .enumerate() { - witness.set_target( - public_values_target.checkpoint_state_trie_root[2 * i], - F::from_canonical_u32(limb as u32), - ); - witness.set_target( - public_values_target.checkpoint_state_trie_root[2 * i + 1], - F::from_canonical_u32((limb >> 32) as u32), - ); + witness + .set_target( + public_values_target.checkpoint_state_trie_root[2 * i], + F::from_canonical_u32(limb as u32), + ) + .map_err(ProgramError::from)?; + witness + .set_target( + public_values_target.checkpoint_state_trie_root[2 * i + 1], + F::from_canonical_u32((limb >> 32) as u32), + ) + .map_err(ProgramError::from)?; } for (i, limb) in public_values @@ -935,18 +941,24 @@ where .into_iter() .enumerate() { - witness.set_target( - public_values_target.new_state_trie_root[2 * i], - F::from_canonical_u32(limb as u32), - ); - witness.set_target( - public_values_target.new_state_trie_root[2 * i + 1], - F::from_canonical_u32((limb >> 32) as u32), - ); + witness + .set_target( + public_values_target.new_state_trie_root[2 * i], + F::from_canonical_u32(limb as u32), + ) + .map_err(ProgramError::from)?; + witness + .set_target( + public_values_target.new_state_trie_root[2 * i + 1], + F::from_canonical_u32((limb >> 32) as u32), + ) + .map_err(ProgramError::from)?; } for (i, limb) in public_values.new_consolidated_hash.iter().enumerate() { - witness.set_target(public_values_target.new_consolidated_hash[i], *limb); + witness + .set_target(public_values_target.new_consolidated_hash[i], *limb) + .map_err(ProgramError::from)?; } Ok(()) @@ -956,7 +968,8 @@ pub(crate) fn set_trie_roots_target( witness: &mut W, trie_roots_target: &TrieRootsTarget, trie_roots: &TrieRoots, -) where +) -> Result<()> +where F: RichField + Extendable, W: Witness, { @@ -964,11 +977,11 @@ pub(crate) fn set_trie_roots_target( witness.set_target( trie_roots_target.state_root[2 * i], F::from_canonical_u32(limb as u32), - ); + )?; witness.set_target( trie_roots_target.state_root[2 * i + 1], F::from_canonical_u32((limb >> 32) as u32), - ); + )?; } for (i, limb) in trie_roots @@ -981,11 +994,11 @@ pub(crate) fn set_trie_roots_target( witness.set_target( trie_roots_target.transactions_root[2 * i], F::from_canonical_u32(limb as u32), - ); + )?; witness.set_target( trie_roots_target.transactions_root[2 * i + 1], F::from_canonical_u32((limb >> 32) as u32), - ); + )?; } for (i, limb) in trie_roots @@ -998,12 +1011,14 @@ pub(crate) fn set_trie_roots_target( witness.set_target( trie_roots_target.receipts_root[2 * i], F::from_canonical_u32(limb as u32), - ); + )?; witness.set_target( trie_roots_target.receipts_root[2 * i + 1], F::from_canonical_u32((limb >> 32) as u32), - ); + )?; } + + Ok(()) } #[cfg(feature = "cdk_erigon")] @@ -1019,7 +1034,9 @@ where match burn_addr_target { BurnAddrTarget::BurnAddr(addr_target) => { let burn_addr_limbs: [F; 8] = u256_limbs::(burn_addr); - witness.set_target_arr(addr_target, &burn_addr_limbs); + witness + .set_target_arr(addr_target, &burn_addr_limbs) + .map_err(ProgramError::from)?; } BurnAddrTarget::Burnt() => panic!("There should be an address target set in cdk_erigon."), } @@ -1040,73 +1057,105 @@ where u256_limbs::(U256::from_big_endian(&block_metadata.block_beneficiary.0))[..5] .try_into() .unwrap(); - witness.set_target_arr(&block_metadata_target.block_beneficiary, &beneficiary_limbs); - witness.set_target( - block_metadata_target.block_timestamp, - u256_to_u32(block_metadata.block_timestamp)?, - ); - witness.set_target( - block_metadata_target.block_number, - u256_to_u32(block_metadata.block_number)?, - ); - witness.set_target( - block_metadata_target.block_difficulty, - u256_to_u32(block_metadata.block_difficulty)?, - ); - witness.set_target_arr( - &block_metadata_target.block_random, - &h256_limbs(block_metadata.block_random), - ); - witness.set_target( - block_metadata_target.block_gaslimit, - u256_to_u32(block_metadata.block_gaslimit)?, - ); - witness.set_target( - block_metadata_target.block_chain_id, - u256_to_u32(block_metadata.block_chain_id)?, - ); + witness + .set_target_arr(&block_metadata_target.block_beneficiary, &beneficiary_limbs) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_timestamp, + u256_to_u32(block_metadata.block_timestamp)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_number, + u256_to_u32(block_metadata.block_number)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_difficulty, + u256_to_u32(block_metadata.block_difficulty)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target_arr( + &block_metadata_target.block_random, + &h256_limbs(block_metadata.block_random), + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_gaslimit, + u256_to_u32(block_metadata.block_gaslimit)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_chain_id, + u256_to_u32(block_metadata.block_chain_id)?, + ) + .map_err(ProgramError::from)?; // Basefee fits in 2 limbs let basefee = u256_to_u64(block_metadata.block_base_fee)?; - witness.set_target(block_metadata_target.block_base_fee[0], basefee.0); - witness.set_target(block_metadata_target.block_base_fee[1], basefee.1); - witness.set_target( - block_metadata_target.block_gas_used, - u256_to_u32(block_metadata.block_gas_used)?, - ); + witness + .set_target(block_metadata_target.block_base_fee[0], basefee.0) + .map_err(ProgramError::from)?; + witness + .set_target(block_metadata_target.block_base_fee[1], basefee.1) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_gas_used, + u256_to_u32(block_metadata.block_gas_used)?, + ) + .map_err(ProgramError::from)?; #[cfg(feature = "eth_mainnet")] { // BlobGasUsed fits in 2 limbs let blob_gas_used = u256_to_u64(block_metadata.block_blob_gas_used)?; - witness.set_target( - block_metadata_target.block_blob_gas_used[0], - blob_gas_used.0, - ); - witness.set_target( - block_metadata_target.block_blob_gas_used[1], - blob_gas_used.1, - ); + witness + .set_target( + block_metadata_target.block_blob_gas_used[0], + blob_gas_used.0, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_blob_gas_used[1], + blob_gas_used.1, + ) + .map_err(ProgramError::from)?; // ExcessBlobGas fits in 2 limbs let excess_blob_gas = u256_to_u64(block_metadata.block_excess_blob_gas)?; - witness.set_target( - block_metadata_target.block_excess_blob_gas[0], - excess_blob_gas.0, - ); - witness.set_target( - block_metadata_target.block_excess_blob_gas[1], - excess_blob_gas.1, - ); - - witness.set_target_arr( - &block_metadata_target.parent_beacon_block_root, - &h256_limbs(block_metadata.parent_beacon_block_root), - ); + witness + .set_target( + block_metadata_target.block_excess_blob_gas[0], + excess_blob_gas.0, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + block_metadata_target.block_excess_blob_gas[1], + excess_blob_gas.1, + ) + .map_err(ProgramError::from)?; + + witness + .set_target_arr( + &block_metadata_target.parent_beacon_block_root, + &h256_limbs(block_metadata.parent_beacon_block_root), + ) + .map_err(ProgramError::from)?; } let mut block_bloom_limbs = [F::ZERO; 64]; for (i, limbs) in block_bloom_limbs.chunks_exact_mut(8).enumerate() { limbs.copy_from_slice(&u256_limbs(block_metadata.block_bloom[i])); } - witness.set_target_arr(&block_metadata_target.block_bloom, &block_bloom_limbs); + witness + .set_target_arr(&block_metadata_target.block_bloom, &block_bloom_limbs) + .map_err(ProgramError::from)?; Ok(()) } @@ -1115,7 +1164,8 @@ pub(crate) fn set_block_hashes_target( witness: &mut W, block_hashes_target: &BlockHashesTarget, block_hashes: &BlockHashes, -) where +) -> Result<()> +where F: RichField + Extendable, W: Witness, { @@ -1124,10 +1174,10 @@ pub(crate) fn set_block_hashes_target( witness.set_target_arr( &block_hashes_target.prev_hashes[8 * i..8 * (i + 1)], &block_hash_limbs, - ); + )?; } let cur_block_hash_limbs: [F; 8] = h256_limbs::(block_hashes.cur_hash); - witness.set_target_arr(&block_hashes_target.cur_hash, &cur_block_hash_limbs); + witness.set_target_arr(&block_hashes_target.cur_hash, &cur_block_hash_limbs) } pub(crate) fn set_extra_public_values_target( @@ -1139,24 +1189,36 @@ where F: RichField + Extendable, W: Witness, { - witness.set_target_arr( - &ed_target.checkpoint_state_trie_root, - &h256_limbs::(ed.checkpoint_state_trie_root), - ); - witness.set_target_arr( - &ed_target.checkpoint_consolidated_hash, - &ed.checkpoint_consolidated_hash, - ); - witness.set_target( - ed_target.txn_number_before, - u256_to_u32(ed.txn_number_before)?, - ); - witness.set_target( - ed_target.txn_number_after, - u256_to_u32(ed.txn_number_after)?, - ); - witness.set_target(ed_target.gas_used_before, u256_to_u32(ed.gas_used_before)?); - witness.set_target(ed_target.gas_used_after, u256_to_u32(ed.gas_used_after)?); + witness + .set_target_arr( + &ed_target.checkpoint_state_trie_root, + &h256_limbs::(ed.checkpoint_state_trie_root), + ) + .map_err(ProgramError::from)?; + witness + .set_target_arr( + &ed_target.checkpoint_consolidated_hash, + &ed.checkpoint_consolidated_hash, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + ed_target.txn_number_before, + u256_to_u32(ed.txn_number_before)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target( + ed_target.txn_number_after, + u256_to_u32(ed.txn_number_after)?, + ) + .map_err(ProgramError::from)?; + witness + .set_target(ed_target.gas_used_before, u256_to_u32(ed.gas_used_before)?) + .map_err(ProgramError::from)?; + witness + .set_target(ed_target.gas_used_after, u256_to_u32(ed.gas_used_after)?) + .map_err(ProgramError::from)?; Ok(()) } @@ -1170,12 +1232,24 @@ where F: RichField + Extendable, W: Witness, { - witness.set_target(rd_target.program_counter, u256_to_u32(rd.program_counter)?); - witness.set_target(rd_target.is_kernel, u256_to_u32(rd.is_kernel)?); - witness.set_target(rd_target.stack_len, u256_to_u32(rd.stack_len)?); - witness.set_target_arr(&rd_target.stack_top, &u256_limbs(rd.stack_top)); - witness.set_target(rd_target.context, u256_to_u32(rd.context)?); - witness.set_target(rd_target.gas_used, u256_to_u32(rd.gas_used)?); + witness + .set_target(rd_target.program_counter, u256_to_u32(rd.program_counter)?) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.is_kernel, u256_to_u32(rd.is_kernel)?) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.stack_len, u256_to_u32(rd.stack_len)?) + .map_err(ProgramError::from)?; + witness + .set_target_arr(&rd_target.stack_top, &u256_limbs(rd.stack_top)) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.context, u256_to_u32(rd.context)?) + .map_err(ProgramError::from)?; + witness + .set_target(rd_target.gas_used, u256_to_u32(rd.gas_used)?) + .map_err(ProgramError::from)?; Ok(()) } @@ -1190,12 +1264,14 @@ where W: Witness, { for i in 0..mc.mem_cap.len() { - witness.set_hash_target( - mc_target.mem_cap.0[i], - HashOut { - elements: mc.mem_cap[i].map(|elt| F::from_canonical_u64(elt.as_u64())), - }, - ); + witness + .set_hash_target( + mc_target.mem_cap.0[i], + HashOut { + elements: mc.mem_cap[i].map(|elt| F::from_canonical_u64(elt.as_u64())), + }, + ) + .map_err(ProgramError::from)?; } Ok(()) } diff --git a/evm_arithmetization/src/testing_utils.rs b/evm_arithmetization/src/testing_utils.rs index 039dc2504..321e7fe07 100644 --- a/evm_arithmetization/src/testing_utils.rs +++ b/evm_arithmetization/src/testing_utils.rs @@ -16,6 +16,7 @@ pub use crate::cpu::kernel::cancun_constants::*; pub use crate::cpu::kernel::constants::global_exit_root::*; use crate::generation::{TrieInputs, TrimmedGenerationInputs}; use crate::proof::TrieRoots; +#[cfg(test)] use crate::witness::operation::Operation; use crate::{ generation::mpt::AccountRlp, proof::BlockMetadata, util::h2u, GenerationInputs, @@ -229,8 +230,25 @@ pub fn segment_with_empty_tables() -> Result<( SegmentDataIterator::::new(&payload, max_cpu_len_log); let (trimmed_inputs, segment_data) = segment_iterator.next().unwrap()?; - let opcode_counts = &segment_data.opcode_counts; - assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); - Ok((trimmed_inputs, segment_data)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::logic; + + // Ensures that there are no Keccak and Logic ops in the segment. + #[test] + fn test_segment_with_empty_tables() -> Result<()> { + let (_, segment_data) = segment_with_empty_tables()?; + + let opcode_counts = &segment_data.opcode_counts; + assert!(!opcode_counts.contains_key(&Operation::KeccakGeneral)); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::And))); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Or))); + assert!(!opcode_counts.contains_key(&Operation::BinaryLogic(logic::Op::Xor))); + + Ok(()) + } +} diff --git a/evm_arithmetization/src/verifier.rs b/evm_arithmetization/src/verifier.rs index 81e9b502b..1d4fdf387 100644 --- a/evm_arithmetization/src/verifier.rs +++ b/evm_arithmetization/src/verifier.rs @@ -80,6 +80,7 @@ pub(crate) fn initial_memory_merkle_cap< pub mod testing { use anyhow::{ensure, Result}; use ethereum_types::{BigEndianHash, U256}; + use hashbrown::HashMap; use itertools::Itertools; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::RichField; @@ -92,7 +93,7 @@ pub mod testing { use starky::stark::Stark; use starky::verifier::verify_stark_proof_with_challenges; - use crate::all_stark::Table; + use crate::all_stark::{Table, MEMORY_CTL_IDX, NUM_CTLS, OPTIONAL_TABLE_INDICES}; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::get_challenges::testing::AllProofChallenges; @@ -123,6 +124,22 @@ pub mod testing { self.poseidon_stark.num_lookup_helper_columns(config), ] } + + pub fn get_constraint_degree(&self, table: Table) -> usize { + match table { + Table::Arithmetic => self.arithmetic_stark.constraint_degree(), + Table::BytePacking => self.byte_packing_stark.constraint_degree(), + Table::Cpu => self.cpu_stark.constraint_degree(), + Table::Keccak => self.keccak_stark.constraint_degree(), + Table::KeccakSponge => self.keccak_sponge_stark.constraint_degree(), + Table::Logic => self.logic_stark.constraint_degree(), + Table::Memory => self.memory_stark.constraint_degree(), + Table::MemBefore => self.mem_before_stark.constraint_degree(), + Table::MemAfter => self.mem_after_stark.constraint_degree(), + #[cfg(feature = "cdk_erigon")] + Table::Poseidon => self.poseidon_stark.constraint_degree(), + } + } } fn verify_initial_memory< @@ -190,48 +207,48 @@ pub mod testing { macro_rules! verify_table { ($stark:ident, $table:expr) => { - let stark_proof = &stark_proofs[*$table] - .as_ref() - .expect("Missing stark_proof") - .proof; - let ctl_vars = { - let (total_num_helpers, _, num_helpers_by_ctl) = - CrossTableLookup::num_ctl_helpers_zs_all( - &all_stark.cross_table_lookups, - *$table, - config.num_challenges, - $stark.constraint_degree(), - ); - CtlCheckVars::from_proof( - *$table, - &stark_proof, - &all_stark.cross_table_lookups, - &ctl_challenges, - num_lookup_columns[*$table], - total_num_helpers, - &num_helpers_by_ctl, - ) - }; - verify_stark_proof_with_challenges( - $stark, - stark_proof, - &stark_challenges[*$table] + if !OPTIONAL_TABLE_INDICES.contains(&*$table) || all_proof.table_in_use[*$table] { + let stark_proof = &stark_proofs[*$table] .as_ref() - .expect("Missing challenges"), - Some(&ctl_vars), - &[], - config, - )?; + .expect("Missing stark_proof") + .proof; + let ctl_vars = { + let (total_num_helpers, _, num_helpers_by_ctl) = + CrossTableLookup::num_ctl_helpers_zs_all( + &all_stark.cross_table_lookups, + *$table, + config.num_challenges, + $stark.constraint_degree(), + ); + CtlCheckVars::from_proof( + *$table, + &stark_proof, + &all_stark.cross_table_lookups, + &ctl_challenges, + num_lookup_columns[*$table], + total_num_helpers, + &num_helpers_by_ctl, + ) + }; + verify_stark_proof_with_challenges( + $stark, + stark_proof, + &stark_challenges[*$table] + .as_ref() + .expect("Missing challenges"), + Some(&ctl_vars), + &[], + config, + )?; + } }; } verify_table!(arithmetic_stark, Table::Arithmetic); verify_table!(byte_packing_stark, Table::BytePacking); verify_table!(cpu_stark, Table::Cpu); - if all_proof.use_keccak_tables { - verify_table!(keccak_stark, Table::Keccak); - verify_table!(keccak_sponge_stark, Table::KeccakSponge); - } + verify_table!(keccak_stark, Table::Keccak); + verify_table!(keccak_sponge_stark, Table::KeccakSponge); verify_table!(logic_stark, Table::Logic); verify_table!(memory_stark, Table::Memory); verify_table!(mem_before_stark, Table::MemBefore); @@ -249,47 +266,48 @@ pub mod testing { // Extra sums to add to the looked last value. // Only necessary for the Memory values. - let mut extra_looking_sums = vec![vec![F::ZERO; config.num_challenges]; NUM_TABLES]; + let mut extra_looking_sums = + HashMap::from_iter((0..NUM_CTLS).map(|i| (i, vec![F::ZERO; config.num_challenges]))); // Memory - extra_looking_sums[*Table::Memory] = (0..config.num_challenges) - .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) - .collect_vec(); + extra_looking_sums.insert( + MEMORY_CTL_IDX, + (0..config.num_challenges) + .map(|i| get_memory_extra_looking_sum(&public_values, ctl_challenges.challenges[i])) + .collect_vec(), + ); let all_ctls = &all_stark.cross_table_lookups; + let table_all = Table::all(); + let ctl_zs_first_values = core::array::from_fn(|i| { + let table = table_all[i]; + if let Some(stark_proof) = &stark_proofs[i] { + stark_proof + .proof + .openings + .ctl_zs_first + .as_ref() + .expect("Missing ctl_zs") + .clone() + } else if OPTIONAL_TABLE_INDICES.contains(&table) { + let degree = all_stark.get_constraint_degree(table); + let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( + all_ctls, + i, + config.num_challenges, + degree, + ); + vec![F::ZERO; n] + } else { + panic!("Unable to find stark_proof for table {:?}", table); + } + }); + verify_cross_table_lookups::( cross_table_lookups, - core::array::from_fn(|i| { - if let Some(stark_proof) = &stark_proofs[i] { - stark_proof - .proof - .openings - .ctl_zs_first - .as_ref() - .expect("Missing ctl_zs") - .clone() - } else if i == *Table::Keccak { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::Keccak, - config.num_challenges, - keccak_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else if i == *Table::KeccakSponge { - let (_, n, _) = CrossTableLookup::num_ctl_helpers_zs_all( - all_ctls, - *Table::KeccakSponge, - config.num_challenges, - keccak_sponge_stark.constraint_degree(), - ); - vec![F::ZERO; n] - } else { - panic!("Unable to find stark_proof"); - } - }), - Some(&extra_looking_sums), + ctl_zs_first_values, + &extra_looking_sums, config, ) } diff --git a/evm_arithmetization/src/witness/errors.rs b/evm_arithmetization/src/witness/errors.rs index ef1e9f73b..adb9dd94d 100644 --- a/evm_arithmetization/src/witness/errors.rs +++ b/evm_arithmetization/src/witness/errors.rs @@ -16,6 +16,13 @@ pub enum ProgramError { IntegerTooLarge, ProverInputError(ProverInputError), UnknownContractCode, + Other(String), +} + +impl From for ProgramError { + fn from(e: anyhow::Error) -> Self { + ProgramError::Other(e.to_string()) + } } #[allow(clippy::enum_variant_names)] diff --git a/evm_arithmetization/src/witness/traces.rs b/evm_arithmetization/src/witness/traces.rs index cb9e605e8..e1daa53b0 100644 --- a/evm_arithmetization/src/witness/traces.rs +++ b/evm_arithmetization/src/witness/traces.rs @@ -144,7 +144,7 @@ impl Traces { mut trace_lengths: TraceCheckpoint, config: &StarkConfig, timing: &mut TimingTree, - ) -> [Vec>; NUM_TABLES] + ) -> ([Vec>; NUM_TABLES], usize) where T: RichField + Extendable, { @@ -240,19 +240,22 @@ impl Traces { final_values.len() ); - [ - arithmetic_trace, - byte_packing_trace, - cpu_trace, - keccak_trace, - keccak_sponge_trace, - logic_trace, - memory_trace, - mem_before_trace, - mem_after_trace, - #[cfg(feature = "cdk_erigon")] - poseidon_trace, - ] + ( + [ + arithmetic_trace, + byte_packing_trace, + cpu_trace, + keccak_trace, + keccak_sponge_trace, + logic_trace, + memory_trace, + mem_before_trace, + mem_after_trace, + #[cfg(feature = "cdk_erigon")] + poseidon_trace, + ], + final_values.len(), + ) } } diff --git a/evm_arithmetization/tests/empty_tables.rs b/evm_arithmetization/tests/empty_tables.rs index d25901e24..e99c48731 100644 --- a/evm_arithmetization/tests/empty_tables.rs +++ b/evm_arithmetization/tests/empty_tables.rs @@ -1,5 +1,7 @@ #![cfg(feature = "eth_mainnet")] +use std::time::Duration; + use evm_arithmetization::fixed_recursive_verifier::AllRecursiveCircuits; use evm_arithmetization::prover::prove; use evm_arithmetization::testing_utils::{init_logger, segment_with_empty_tables}; @@ -8,6 +10,7 @@ use evm_arithmetization::AllStark; use plonky2::field::goldilocks_field::GoldilocksField; use plonky2::plonk::config::PoseidonGoldilocksConfig; use plonky2::timed; +use plonky2::util::serialization::{DefaultGateSerializer, DefaultGeneratorSerializer}; use plonky2::util::timing::TimingTree; use starky::config::StarkConfig; @@ -58,6 +61,7 @@ fn empty_tables() -> anyhow::Result<()> { &config, ) ); + let segment_proof = timed!( timing, log::Level::Info, @@ -76,5 +80,33 @@ fn empty_tables() -> anyhow::Result<()> { // Print timing details timing.print(); + // Test serialization of preprocessed circuits + { + let gate_serializer = DefaultGateSerializer; + let generator_serializer = DefaultGeneratorSerializer::::default(); + + let timing = TimingTree::new("serialize AllRecursiveCircuits", log::Level::Info); + let all_circuits_bytes = all_circuits + .to_bytes(false, &gate_serializer, &generator_serializer) + .map_err(|_| anyhow::Error::msg("AllRecursiveCircuits serialization failed."))?; + timing.filter(Duration::from_millis(100)).print(); + log::info!( + "AllRecursiveCircuits length: {} bytes", + all_circuits_bytes.len() + ); + + let timing = TimingTree::new("deserialize AllRecursiveCircuits", log::Level::Info); + let all_circuits_from_bytes = AllRecursiveCircuits::from_bytes( + &all_circuits_bytes, + false, + &gate_serializer, + &generator_serializer, + ) + .map_err(|_| anyhow::Error::msg("AllRecursiveCircuits deserialization failed."))?; + timing.filter(Duration::from_millis(100)).print(); + + assert_eq!(all_circuits, all_circuits_from_bytes); + } + Ok(()) } diff --git a/evm_arithmetization/tests/two_to_one_block.rs b/evm_arithmetization/tests/two_to_one_block.rs index 378aaf3f8..e808a33e4 100644 --- a/evm_arithmetization/tests/two_to_one_block.rs +++ b/evm_arithmetization/tests/two_to_one_block.rs @@ -172,7 +172,7 @@ fn test_two_to_one_block_aggregation() -> anyhow::Result<()> { let all_circuits = AllRecursiveCircuits::new( &all_stark, - &[16..17, 8..9, 12..13, 8..9, 8..9, 6..7, 17..18, 17..18, 7..8], + &[16..17, 8..9, 12..13, 8..9, 8..9, 6..7, 17..18, 16..17, 7..8], &config, ); diff --git a/trace_decoder/src/type2.rs b/trace_decoder/src/type2.rs index a71761533..44d13e89a 100644 --- a/trace_decoder/src/type2.rs +++ b/trace_decoder/src/type2.rs @@ -173,8 +173,10 @@ fn visit( address, value, }) => { + ensure!(address.len() == Address::len_bytes()); let address = Address::from_slice(&address); let collated = leaves.entry(address).or_default(); + ensure!(value.len() <= 32); let value = U256::from_big_endian(&value); macro_rules! ensure { ($expr:expr) => { @@ -195,6 +197,7 @@ fn visit( collated.code = Some(value) } SmtLeafType::Storage(slot) => { + ensure!(slot.len() <= 32); let clobbered = collated.storage.insert(U256::from_big_endian(&slot), value); ensure!(clobbered.is_none()) } diff --git a/trace_decoder/src/wire.rs b/trace_decoder/src/wire.rs index 63dee6040..359043057 100644 --- a/trace_decoder/src/wire.rs +++ b/trace_decoder/src/wire.rs @@ -80,8 +80,6 @@ pub enum Instruction { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 -// `address` and `value` should be fixed length fields pub struct SmtLeaf { pub node_type: SmtLeafType, pub address: NonEmpty>, @@ -89,8 +87,6 @@ pub struct SmtLeaf { } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/705 -// `Storage` should contain a fixed length field pub enum SmtLeafType { Balance, Nonce, From 029983c36889f1e079f05866fb88914ba143f1ef Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 25 Oct 2024 18:42:47 +0200 Subject: [PATCH 091/107] stats --- .../src/generation/prover_input.rs | 12 +++++++- scripts/test_native.sh | 30 +++++++++++-------- zero/src/prover/cli.rs | 3 ++ 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 6db978073..cc4461b09 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -797,7 +797,10 @@ impl GenerationState { // REVIEW: This will be rewritten to only run simulation when // `self.inputs.jumpdest_table` is `None`. - info!("Generating JUMPDEST tables"); + info!( + "Generating JUMPDEST tables for block: {}, tx: {:?}", + self.inputs.block_metadata.block_number, self.inputs.txn_hashes + ); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() @@ -821,6 +824,13 @@ impl GenerationState { } info!("SIMW == RPCW ? {}", simw == &rpcw); info!("tx: {:?}", self.inputs.txn_hashes); + // let is_equal = simw == &rpcw; + // let block_num = self.inputs.block_metadata.block_number; + // tracing::error!( + // block_num, + // tables_match = is_equal, + // tx = self.inputs.txn_hashes + // ) // panic!(); // info!("SIMP {:?}", &simp); // info!("RPCP {:?}", &rpcp); diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 824001094..20506aafa 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -21,25 +21,25 @@ RESULT_LEN=$(cat witnesses/native_results.txt | wc -l) function statistics() { - PREFIX_LEN=$(($RESULT_LEN + 13)) + PREFIX_LEN=1000 wc -l witnesses/native_results.txt - cat witnesses/native_results.txt | tail -n +$PREFIX_LEN + cat witnesses/native_results.txt | tail -n $PREFIX_LEN - SUMOK=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f8 | paste -s -d+ - | bc) - SUMFAIL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f9 | paste -s -d+ - | bc) - SUMTOTAL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f10 | paste -s -d+ - | bc) + SUMOK=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f8 | paste -s -d+ - | bc) + SUMFAIL=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f9 | paste -s -d+ - | bc) + SUMTOTAL=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f10 | paste -s -d+ - | bc) echo $SUMTOTAL echo $SUMFAIL echo "Failure rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMFAIL * 100 / $SUMTOTAL))%") echo "Success rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMOK * 100 / $SUMTOTAL))%") - ZEROES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "0") - ONES=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "1") - TWOS=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "2") - THREES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "3") - FOURS=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "4") - FIVES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "5") - SIXES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "6") + ZEROES=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "0") + ONES=$( cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "1") + TWOS=$( cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "2") + THREES=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "3") + FOURS=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "4") + FIVES=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "5") + SIXES=$(cat witnesses/native_results.txt | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "6") echo $ZEROES echo $ONES echo $TWOS @@ -50,7 +50,11 @@ function statistics() echo "good bye" exit 0 } -trap statistics INT # EXIT QUIT HUP TERM +trap statistics INT EXIT QUIT HUP TERM + +statistics + +exit 0 # Must match the values in prove_stdio.sh or build is dirty. #export RAYON_NUM_THREADS=1 #export TOKIO_WORKER_THREADS=1 diff --git a/zero/src/prover/cli.rs b/zero/src/prover/cli.rs index 87e79bc65..09a64ba7d 100644 --- a/zero/src/prover/cli.rs +++ b/zero/src/prover/cli.rs @@ -23,6 +23,9 @@ pub struct CliProverConfig { /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] save_inputs_on_error: bool, + /// Keep going if a block proof fails. + #[arg(short='K', long, help_heading = HELP_HEADING, default_value_t = false)] + keep_going: bool, /// If true, only test the trace decoder and witness generation without /// generating a proof. #[arg(long, help_heading = HELP_HEADING, default_value_t = false)] From 462fd1eec7d76d55cacbd2ab7449785e4a73a1bb Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 29 Oct 2024 14:47:46 +0100 Subject: [PATCH 092/107] Revert "remove todo" This reverts commit a6ba0e5844a8da45e6cb4d7a9dd35ca0257dfce7. --- evm_arithmetization/src/generation/prover_input.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index cc4461b09..d22d01a59 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -362,6 +362,9 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // TODO(einar-polygon) + // get_code from self.memory + if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } From 1f9ad53e06b117d1b139c750dde379ac29c7b8dc Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 30 Oct 2024 14:30:20 +0100 Subject: [PATCH 093/107] wip --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 15 +++++++-------- evm_arithmetization/src/generation/jumpdest.rs | 4 ++++ .../src/generation/prover_input.rs | 15 ++++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index b1bb3b4f4..a6872f57d 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -170,15 +170,15 @@ pub(crate) fn set_registers_and_run( pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, + ctx_codes: &HashMap>>, ) -> JumpDestTableProcessed { - let ctx_proofs = (*jumpdest_table_rpc) + let ctx_proofs = jumpdest_table_rpc .iter() .flat_map(|(code_addr, ctx_jumpdests)| { - let code = if code_map.contains_key(code_addr) { - &code_map[code_addr] - } else { - &vec![] - }; + let code = &code_map + .get(code_addr) + .map(|x| x.clone()) + .unwrap_or_default(); trace!( "code: {:?}, code_addr: {:?}, {:?} <============", &code, @@ -204,8 +204,7 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( /// Returns a [`HashMap`] from `ctx` to [`Vec`] of proofs. Each proofs ia a /// pair. fn prove_context_jumpdests(code: &[u8], ctx: &Context) -> HashMap> { - ctx.0 - .iter() + ctx.iter() .map(|(&ctx, jumpdests)| { let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 5f516e52b..29cfac3e6 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -95,6 +95,10 @@ impl JumpDestTableWitness { jdts.into_iter() .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) } + + pub(crate) fn get_all_contexts(&self) -> Vec { + todo!() + } } // The following Display instances are added to make it easier to read diffs. diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index d22d01a59..3177e46db 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -360,10 +360,13 @@ impl GenerationState { /// Returns the next used jump address. fn run_next_jumpdest_table_address(&mut self) -> Result { + // QUESTION: Is there a reason we do not use `self.registers.context` here? let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; // TODO(einar-polygon) // get_code from self.memory + let code = self.get_code(context)?; + let current_code = self.get_current_code()?; if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; @@ -805,9 +808,15 @@ impl GenerationState { self.inputs.block_metadata.block_number, self.inputs.txn_hashes ); let rpcw = self.inputs.jumpdest_table.clone(); - let rpcp: Option = rpcw - .as_ref() - .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + let rpcp: Option = rpcw.as_ref().map(|jdt| { + let all_contexts: Vec = jdt.get_all_contexts(); + let ctx_codes: HashMap>> = all_contexts + .into_iter() + .map(|ctx| (ctx, self.get_code(ctx).ok())) + .collect(); + + get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, &ctx_codes) + }); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; From 01df87a8ae46181c927c782edfb70f2f3c4b159a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 30 Oct 2024 16:37:50 +0100 Subject: [PATCH 094/107] readd CREATE1/2 --- zero/src/rpc/jumpdest.rs | 66 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 7f323551a..d9574701b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -240,10 +240,68 @@ pub(crate) fn generate_jumpdest_table<'a>( next_ctx_available += 1; } "CREATE" | "CREATE2" => { - bail!(format!( - "{} requires memory, aborting JUMPDEST-table generation.", - tx.hash - )); + prev_jump = None; + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands_used = 3; + + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + }; + + let [_value, offset, size, ..] = evm_stack[..] else { + unreachable!() + }; + if *offset > U256::from(usize::MAX) { + trace!( + "{op}: Offset {offset} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; + let offset: usize = offset.to(); + + if *size > U256::from(usize::MAX) { + trace!( + "{op}: Size {size} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; + let size: usize = size.to(); + + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; + + if entry.memory.is_none() || offset + size > memory_size { + trace!("Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size); + continue; + } + let memory_raw: &[String] = entry.memory.as_ref().unwrap(); + let memory_parsed: Vec> = memory_raw + .iter() + .map(|mem_line| { + let mem_line_parsed = U256::from_str_radix(mem_line, 16)?; + Ok(mem_line_parsed.to_be_bytes()) + }) + .collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + if mem_res.is_err() { + trace!( + "{op}: Parsing memory failed with error: {}", + mem_res.unwrap_err() + ); + continue; + } + let memory: Vec = mem_res.unwrap().concat(); + + let init_code = &memory[offset..offset + size]; + code_db.insert(init_code.to_vec()); + let init_code_hash = keccak(init_code); + call_stack.push((init_code_hash, next_ctx_available)); + + next_ctx_available += 1; } "JUMP" => { prev_jump = None; From aecb2eb4f61fd40830226606a9dc4438352afc0b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sat, 2 Nov 2024 17:17:29 +0100 Subject: [PATCH 095/107] don't insert code --- zero/src/rpc/jumpdest.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d9574701b..83b086c75 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -272,6 +272,10 @@ pub(crate) fn generate_jumpdest_table<'a>( }; let size: usize = size.to(); + /// Structure of Etheruem memory + type Word = [u8; 32]; + const WORDSIZE: usize = std::mem::size_of::(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; if entry.memory.is_none() || offset + size > memory_size { @@ -297,7 +301,7 @@ pub(crate) fn generate_jumpdest_table<'a>( let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; - code_db.insert(init_code.to_vec()); + //code_db.insert(init_code.to_vec()); let init_code_hash = keccak(init_code); call_stack.push((init_code_hash, next_ctx_available)); From 1f61ee441872d31da213930dd8bc784a8d59fe94 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sat, 2 Nov 2024 18:10:45 +0100 Subject: [PATCH 096/107] WIP --- zero/src/rpc/jumpdest.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 83b086c75..f1db5b80b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -227,8 +227,11 @@ pub(crate) fn generate_jumpdest_table<'a>( if let Some((_next_step, next_entry)) = stuctlog_iter.peek() { let next_depth: usize = next_entry.depth.try_into().unwrap(); if next_depth < curr_depth { - // The call caused an exception. Skip over incrementing - // `next_ctx_available`. + // The call caused an exception. Skip over incrementing + // `next_ctx_available`. Note that calling an invalid + // contract address should still increment + // `next_ctx_available`, although we stay in the current + // context. continue; } } @@ -303,8 +306,25 @@ pub(crate) fn generate_jumpdest_table<'a>( let init_code = &memory[offset..offset + size]; //code_db.insert(init_code.to_vec()); let init_code_hash = keccak(init_code); + let mut init_code_hash; + call_stack.push((init_code_hash, next_ctx_available)); + if let Some((_next_step, next_entry)) = stuctlog_iter.peek() { + let next_depth: usize = next_entry.depth.try_into().unwrap(); + if next_depth < curr_depth { + // The call caused an exception. Skip over incrementing + // `next_ctx_available`. + continue; + } + else { + init_code_hash = next_entry.co.try_into().unwrap(); + } + } + + + + next_ctx_available += 1; } "JUMP" => { From a1f688e12399fd71fa9a7607082f15a99668d551 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 12:41:39 +0100 Subject: [PATCH 097/107] progressive --- .../src/cpu/kernel/interpreter.rs | 50 +++++++++++-------- .../src/generation/jumpdest.rs | 15 +++++- .../src/generation/prover_input.rs | 47 +++++++++++------ zero/src/rpc/jumpdest.rs | 11 ++-- 4 files changed, 77 insertions(+), 46 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index a6872f57d..e947f2f49 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -167,29 +167,37 @@ pub(crate) fn set_registers_and_run( /// # Output /// /// Returns a [`JumpDestTableProccessed`]. -pub(crate) fn get_jumpdest_analysis_inputs_rpc( +pub(crate) fn get_jumpdest_analysis_inputs_rpc_progressive( jumpdest_table_rpc: &JumpDestTableWitness, - code_map: &HashMap>, - ctx_codes: &HashMap>>, + generation_state: &GenerationState, ) -> JumpDestTableProcessed { - let ctx_proofs = jumpdest_table_rpc - .iter() - .flat_map(|(code_addr, ctx_jumpdests)| { - let code = &code_map - .get(code_addr) - .map(|x| x.clone()) - .unwrap_or_default(); - trace!( - "code: {:?}, code_addr: {:?}, {:?} <============", - &code, - &code_addr, - code_map.contains_key(code_addr), - ); - trace!("code_map: {:?}", &code_map); - prove_context_jumpdests(code, ctx_jumpdests) - }) - .collect(); - JumpDestTableProcessed::new(ctx_proofs) + let current_ctx = generation_state.registers.context; + let current_code = generation_state.get_current_code().unwrap(); + let current_code_hash = generation_state.get_current_code_hash().unwrap(); + let code_map: &HashMap> = &generation_state.inputs.contract_code; + + trace!( + "current_code: {:?}, current_code_hash: {:?}, {:?} <============", + ¤t_code, + ¤t_code_hash, + code_map.contains_key(¤t_code_hash), + ); + trace!("code_map: {:?}", &code_map); + dbg!(current_ctx, current_code_hash, jumpdest_table_rpc.clone()); + let mut ctx_proof = HashMap::>::new(); + if jumpdest_table_rpc.contains_key(¤t_code_hash) { + let cc = &(*jumpdest_table_rpc)[¤t_code_hash].0; + if cc.contains_key(¤t_ctx) { + let current_offsets = cc[¤t_ctx].clone(); + //let ctx_proof = prove_context_jumpdests(¤t_code, &offsets); + let largest_address = current_offsets.last().unwrap().clone(); + let offset_proofs = + get_proofs_and_jumpdests(¤t_code, largest_address, current_offsets); + ctx_proof.insert(current_ctx, offset_proofs); + } + } + + JumpDestTableProcessed::new(ctx_proof) } /// Orchestrates the proving of all contexts in a specific bytecode. diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 29cfac3e6..30f974525 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -61,6 +61,15 @@ impl JumpDestTableProcessed { pub fn new(ctx_map: HashMap>) -> Self { Self(ctx_map) } + + pub fn merge<'a>(jdts: impl IntoIterator) -> Self { + jdts.into_iter().fold(Default::default(), |acc, next| { + let mut inner = acc.0.clone(); + let b = next.iter().map(|(a,b)| (a.clone(), b.clone())); + inner.extend(b); + JumpDestTableProcessed(inner) + }) + } } impl JumpDestTableWitness { @@ -91,9 +100,11 @@ impl JumpDestTableWitness { (self, curr_max_ctx) } - pub fn merge<'a>(jdts: impl IntoIterator) -> (Self, usize) { + pub fn merge<'a>(jdts: impl IntoIterator) -> (Self, usize) { jdts.into_iter() - .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) + .fold((Default::default(), 0), |(acc, cnt), next| { + acc.extend(next, cnt) + }) } pub(crate) fn get_all_contexts(&self) -> Vec { diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 3177e46db..a2f8ff86b 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -26,7 +26,7 @@ use crate::cpu::kernel::constants::cancun_constants::{ }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; use crate::cpu::kernel::interpreter::{ - get_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, + get_jumpdest_analysis_inputs_rpc_progressive, simulate_cpu_and_get_user_jumps, }; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; @@ -369,7 +369,21 @@ impl GenerationState { let current_code = self.get_current_code()?; if self.jumpdest_table.is_none() { - self.generate_jumpdest_table()?; + self.jumpdest_table = Some(JumpDestTableProcessed::default()); + } + + if self.jumpdest_table.is_some() + && self + .jumpdest_table + .as_ref() + .unwrap() + .get(&context) + .is_none() + { + let ctx_table = self.generate_jumpdest_table()?; + self.jumpdest_table = Some(JumpDestTableProcessed::merge( + [self.jumpdest_table.clone().unwrap(), ctx_table].iter(), + )); } let Some(jumpdest_table) = &mut self.jumpdest_table else { @@ -797,10 +811,8 @@ impl GenerationState { impl GenerationState { /// Simulate the user's code and store all the jump addresses with their /// respective contexts. - fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { - // Simulate the user's code and (unnecessarily) part of the kernel code, - // skipping the validate table call - + fn generate_jumpdest_table(&mut self) -> Result { + dbg!(&self.inputs.jumpdest_table); // REVIEW: This will be rewritten to only run simulation when // `self.inputs.jumpdest_table` is `None`. info!( @@ -808,19 +820,18 @@ impl GenerationState { self.inputs.block_metadata.block_number, self.inputs.txn_hashes ); let rpcw = self.inputs.jumpdest_table.clone(); - let rpcp: Option = rpcw.as_ref().map(|jdt| { - let all_contexts: Vec = jdt.get_all_contexts(); - let ctx_codes: HashMap>> = all_contexts - .into_iter() - .map(|ctx| (ctx, self.get_code(ctx).ok())) - .collect(); + let rpcp: Option = rpcw + .as_ref() + .map(|jdt| get_jumpdest_analysis_inputs_rpc_progressive(jdt, &self)); + if rpcp.is_some() { + return Ok(rpcp.unwrap()); + } - get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, &ctx_codes) - }); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); + //.ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; let (simp, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); @@ -851,9 +862,9 @@ impl GenerationState { info!("JUMPDEST tables are equal."); } - self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; + // self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - Ok(()) + Ok(simp.unwrap()) } /// Given a HashMap containing the contexts and the jumpdest addresses, @@ -887,6 +898,10 @@ impl GenerationState { self.get_code(self.registers.context) } + pub(crate) fn get_current_code_hash(&self) -> Result { + Ok(keccak(self.get_code(self.registers.context)?)) + } + fn get_code(&self, context: usize) -> Result, ProgramError> { let code_len = self.get_code_len(context)?; let code = (0..code_len) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index f1db5b80b..3f7a90755 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -306,7 +306,7 @@ pub(crate) fn generate_jumpdest_table<'a>( let init_code = &memory[offset..offset + size]; //code_db.insert(init_code.to_vec()); let init_code_hash = keccak(init_code); - let mut init_code_hash; + // let mut init_code_hash; call_stack.push((init_code_hash, next_ctx_available)); @@ -316,15 +316,12 @@ pub(crate) fn generate_jumpdest_table<'a>( // The call caused an exception. Skip over incrementing // `next_ctx_available`. continue; - } - else { - init_code_hash = next_entry.co.try_into().unwrap(); + } else { + // init_code_hash = + // next_entry.code_hash.try_into().unwrap(); } } - - - next_ctx_available += 1; } "JUMP" => { From ce984304e4fddef0c87cfec3c19eaab03768c743 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 13:11:38 +0100 Subject: [PATCH 098/107] test_new_chain --- scripts/test_new_chain.sh | 203 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100755 scripts/test_new_chain.sh diff --git a/scripts/test_new_chain.sh b/scripts/test_new_chain.sh new file mode 100755 index 000000000..ac43c17b2 --- /dev/null +++ b/scripts/test_new_chain.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +set -uo pipefail + +if [ -z $RPC ]; then + echo You must set an RPC endpoint + exit 1 +fi + +git diff --quiet --exit-code HEAD +if [ $? -ne 0 ]; then + echo Uncommited changes, please commit to make githash consistent + exit 1 +fi + +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p witnesses + +RESULTS="witnesses/jerigon_new_chain.txt" +RESULT_LEN=$(cat $RESULTS | wc -l) +BLOCKS_TESTED=0 + + +function statistics() +{ + PREFIX_LEN=$BLOCKS_TESTED + wc -l $RESULTS + cat $RESULTS | tail -n $PREFIX_LEN + + SUMOK=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f8 | paste -s -d+ - | bc) + SUMFAIL=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f9 | paste -s -d+ - | bc) + SUMTOTAL=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f10 | paste -s -d+ - | bc) + + ZEROES=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "0") + ONES=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "1") + TWOS=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "2") + THREES=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "3") + FOURS=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "4") + FIVES=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "5") + SIXES=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "6") + SEVENS=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "7") + EIGHTS=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "8") + NINES=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "9") + TIMEOUTS=$(cat $RESULTS | tail -n $PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "134") + + printf "\n\nStatistics\n" | tee -a $RESULTS + echo "---------------------------------------------------------------------------------------" | tee -a $RESULTS + echo "Total blocks: " $BLOCKS_TESTED | tee -a $RESULTS + echo "Total transactions: " $SUMTOTAL | tee -a $RESULTS + echo "Transactions without prefetched JUMPDEST table: "$SUMFAIL | tee -a $RESULTS + echo "Failure rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMFAIL * 100 / $SUMTOTAL))%") | tee -a $RESULTS + echo "Success rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMOK * 100 / $SUMTOTAL))%") | tee -a $RESULTS + echo "Zeroes: " $ZEROES | tee -a $RESULTS + echo "Ones: " $ONES | tee -a $RESULTS + echo "Twos: " $TWOS | tee -a $RESULTS + echo "Threes: " $THREES | tee -a $RESULTS + echo "Fours: " $FOURS | tee -a $RESULTS + echo "Fives: " $FIVES | tee -a $RESULTS + echo "Sixes: " $SIXES | tee -a $RESULTS + echo "Sevens: " $SEVENS | tee -a $RESULTS + echo "Eights: " $EIGHTS | tee -a $RESULTS + echo "Nines: " $NINES | tee -a $RESULTS + echo "Timeouts: " $TIMEOUTS | tee -a $RESULTS + echo "End of statistics" | tee -a $RESULTS + exit 0 +} +trap statistics INT EXIT # QUIT # HUP TERM + +# Must match the values in prove_stdio.sh or build is dirty. +#export RAYON_NUM_THREADS=1 +#export TOKIO_WORKER_THREADS=1 +export RUST_BACKTRACE=full +#export RUST_LOG=info +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=33554432 + +TIP=`cast block-number --rpc-url $RPC` +STATICTIP=6555 + +REPO_ROOT=$(git rev-parse --show-toplevel) + +GITHASH=`git rev-parse --short HEAD` + + +nice -19 cargo build --release --bin rpc +nice -19 cargo build --release --bin leader + + +echo "Testing against jerigon testnet 2, current revision: $GITHASH." + +FAILING_BLOCKS1=" +678 +679 +680 +681 +690 +692 +697 +737 +1178 +1913 +3010 +3114 +3115 +3205 +3206 +3215 +3265 +3915 +4076 +4284 +4285 +4286 +5282 +5661 +6086 +6237 +6321 +6494 +6495 +" + +FAILING_BLOCKS2=" +678 +679 +680 +681 +690 +692 +697 +737 +3010 +" + + +#BLOCKS="$(seq $STATICTIP)" +BLOCKS="$(seq 18 6555)" +#BLOCKS=$FAILING_BLOCKS1 +#BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` + +echo "Testing: $BLOCKS" + + + +printf "\n\nReport started: $(date)" | tee -a $RESULTS +printf "\n\nTable of exit codes\n" | tee -a $RESULTS +echo "---------------------------------------------------------------------------------------" | tee -a $RESULTS +echo "0 is success" | tee -a $RESULTS +echo "1 [unexpected] is other errors" | tee -a $RESULTS +echo "2 [unexpected] is undecided" | tee -a $RESULTS +echo "4 [expected] is Attempted to collapse an extension node" | tee -a $RESULTS +echo "5 [unexpected] is non-matching jumpdest tables" | tee -a $RESULTS +echo "6 [expected] is empty witness. Possibly due to Error: Failed to get proof for account" | tee -a $RESULTS +echo "7 [expected] is Found a Hash node during an insert in a PartialTrie" | tee -a $RESULTS +echo "8 [expected] is Attempted to delete a value that ended up inside a hash node" | tee -a $RESULTS +echo "9 [expected] is Memory allocation failed. Increase RAM" | tee -a $RESULTS +echo "134 [undecided] is timeout. Try increasing the proving timeout." | tee -a $RESULTS + +printf "\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a $RESULTS +echo "---------------------------------------------------------------------------------------" | tee -a $RESULTS + +for BLOCK in $BLOCKS; do + TOTALTIME=0 + GITHASH=`git rev-parse --short HEAD` + WITNESS="witnesses/$BLOCK.jerigon2.$GITHASH.witness.json" + echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace + SECONDS=0 + nice -19 -- "${REPO_ROOT}/target/release/rpc" --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs --timeout 120 fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` + DURATION_RPC=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` + TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` + TXOK=`echo -n $(($TXALL - $TXNONE))` + echo "Now testing block $BLOCK .." + export RUST_LOG=info + SECONDS=0 + #timeout 600s + nice -19 -- ./prove_stdio.sh $WITNESS test_only $BLOCK + EXITCODE=$? + TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` + DURATION_PRV=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + TOTALTIME=`date -u -d @"$TOTALTIME" +'%-Hh%-Mm%-Ss'` + if [ $EXITCODE -eq 0 ] + then + VERDICT="success" + else + VERDICT="failure" + fi + printf "%s %10i %s %3i %8s %8s %8s %3i %3i %3i \n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION_RPC $DURATION_PRV $TOTALTIME $TXOK $TXNONE $TXALL | tee -a $RESULTS + ((BLOCKS_TESTED+=1)) + + ### Clean up except when unknown error or undecided + TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" + if [ $EXITCODE -ne 1 ] && [ $EXITCODE -ne 2 ]; then + #rm $TEST_OUT_PATH + #rm $WITNESS + echo + fi + +done + From 99aba90b1d1ef38d2b61f0cbec9251e7ef57c4c8 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 14:00:27 +0100 Subject: [PATCH 099/107] . --- .../src/generation/prover_input.rs | 71 ++++++++++--------- scripts/test_new_chain.sh | 4 +- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index a2f8ff86b..bd339c62f 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -830,41 +830,42 @@ impl GenerationState { info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; - let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - //.ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; - - let (simp, ref simw): (Option, Option) = - sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - - info!("Generating JUMPDEST tables: finished"); - - if rpcw.is_some() && simw != &rpcw { - if let Some(s) = simw { - info!("SIMW {}", s); - } - if let Some(r) = rpcw.as_ref() { - info!("RPCW {}", r); - } - info!("SIMW == RPCW ? {}", simw == &rpcw); - info!("tx: {:?}", self.inputs.txn_hashes); - // let is_equal = simw == &rpcw; - // let block_num = self.inputs.block_metadata.block_number; - // tracing::error!( - // block_num, - // tables_match = is_equal, - // tx = self.inputs.txn_hashes - // ) - // panic!(); - // info!("SIMP {:?}", &simp); - // info!("RPCP {:?}", &rpcp); - // info!("SIMP == RPCP ? {}", &simp == &rpcp); - } else { - info!("JUMPDEST tables are equal."); - } - - // self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - - Ok(simp.unwrap()) + let sims = simulate_cpu_and_get_user_jumps("terminate_common", self) + .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; + + // let (simp, ref simw): (Option, + // Option) = sims.map_or_else(|| (None, None), + // |(sim, simw)| (Some(sim), Some(simw))); + + // info!("Generating JUMPDEST tables: finished"); + + // if rpcw.is_some() && simw != &rpcw { + // if let Some(s) = simw { + // info!("SIMW {}", s); + // } + // if let Some(r) = rpcw.as_ref() { + // info!("RPCW {}", r); + // } + // info!("SIMW == RPCW ? {}", simw == &rpcw); + // info!("tx: {:?}", self.inputs.txn_hashes); + // // let is_equal = simw == &rpcw; + // // let block_num = self.inputs.block_metadata.block_number; + // // tracing::error!( + // // block_num, + // // tables_match = is_equal, + // // tx = self.inputs.txn_hashes + // // ) + // // panic!(); + // // info!("SIMP {:?}", &simp); + // // info!("RPCP {:?}", &rpcp); + // // info!("SIMP == RPCP ? {}", &simp == &rpcp); + // } else { + // info!("JUMPDEST tables are equal."); + // } + + // // self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; + + Ok(sims.0) } /// Given a HashMap containing the contexts and the jumpdest addresses, diff --git a/scripts/test_new_chain.sh b/scripts/test_new_chain.sh index ac43c17b2..17bb650ef 100755 --- a/scripts/test_new_chain.sh +++ b/scripts/test_new_chain.sh @@ -65,7 +65,7 @@ function statistics() echo "End of statistics" | tee -a $RESULTS exit 0 } -trap statistics INT EXIT # QUIT # HUP TERM +trap statistics EXIT # INT QUIT # HUP TERM # Must match the values in prove_stdio.sh or build is dirty. #export RAYON_NUM_THREADS=1 @@ -135,7 +135,7 @@ FAILING_BLOCKS2=" #BLOCKS="$(seq $STATICTIP)" -BLOCKS="$(seq 18 6555)" +BLOCKS="$(seq 105 6555)" #BLOCKS=$FAILING_BLOCKS1 #BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` From b2aa93392775bbb9f4091f05df025a643543e615 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 14:16:42 +0100 Subject: [PATCH 100/107] . --- .../src/generation/prover_input.rs | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index bd339c62f..cb3daf559 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -830,42 +830,46 @@ impl GenerationState { info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; - let sims = simulate_cpu_and_get_user_jumps("terminate_common", self) - .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; - - // let (simp, ref simw): (Option, - // Option) = sims.map_or_else(|| (None, None), - // |(sim, simw)| (Some(sim), Some(simw))); - - // info!("Generating JUMPDEST tables: finished"); - - // if rpcw.is_some() && simw != &rpcw { - // if let Some(s) = simw { - // info!("SIMW {}", s); - // } - // if let Some(r) = rpcw.as_ref() { - // info!("RPCW {}", r); - // } - // info!("SIMW == RPCW ? {}", simw == &rpcw); - // info!("tx: {:?}", self.inputs.txn_hashes); - // // let is_equal = simw == &rpcw; - // // let block_num = self.inputs.block_metadata.block_number; - // // tracing::error!( - // // block_num, - // // tables_match = is_equal, - // // tx = self.inputs.txn_hashes - // // ) - // // panic!(); - // // info!("SIMP {:?}", &simp); - // // info!("RPCP {:?}", &rpcp); - // // info!("SIMP == RPCP ? {}", &simp == &rpcp); - // } else { - // info!("JUMPDEST tables are equal."); - // } - - // // self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - - Ok(sims.0) + let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); + // .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; + + let (simp, ref simw): (Option, Option) = sims + .clone() + .map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + + info!("Generating JUMPDEST tables: finished"); + + if rpcw.is_some() && simw != &rpcw { + if let Some(s) = simw { + info!("SIMW {}", s); + } + if let Some(r) = rpcw.as_ref() { + info!("RPCW {}", r); + } + info!("SIMW == RPCW ? {}", simw == &rpcw); + info!("tx: {:?}", self.inputs.txn_hashes); + // let is_equal = simw == &rpcw; + // let block_num = self.inputs.block_metadata.block_number; + // tracing::error!( + // block_num, + // tables_match = is_equal, + // tx = self.inputs.txn_hashes + // ) + // panic!(); + // info!("SIMP {:?}", &simp); + // info!("RPCP {:?}", &rpcp); + // info!("SIMP == RPCP ? {}", &simp == &rpcp); + } else { + info!("JUMPDEST tables are equal."); + } + + // self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; + + if sims.as_ref().is_none() { + return Ok(Default::default()); + } + + Ok(sims.unwrap().0) } /// Given a HashMap containing the contexts and the jumpdest addresses, From 97f5684cbd094a410aa9d97f5c5ef43814c52cfc Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 14:37:48 +0100 Subject: [PATCH 101/107] . --- .../src/cpu/kernel/interpreter.rs | 24 +------------------ .../src/generation/jumpdest.rs | 6 +---- .../src/generation/prover_input.rs | 6 +---- scripts/test_new_chain.sh | 2 +- 4 files changed, 4 insertions(+), 34 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index e947f2f49..6c9933f51 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -21,7 +21,7 @@ use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; -use crate::generation::jumpdest::{Context, JumpDestTableProcessed, JumpDestTableWitness}; +use crate::generation::jumpdest::{JumpDestTableProcessed, JumpDestTableWitness}; use crate::generation::linked_list::LinkedListsPtrs; use crate::generation::mpt::{load_linked_lists_and_txn_and_receipt_mpts, TrieRootPtrs}; use crate::generation::prover_input::get_proofs_and_jumpdests; @@ -200,28 +200,6 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc_progressive( JumpDestTableProcessed::new(ctx_proof) } -/// Orchestrates the proving of all contexts in a specific bytecode. -/// -/// # Arguments -/// -/// - `code`: The bytecode for the context `ctx`. -/// - `ctx`: Map from `ctx` to its list of `JUMPDEST` offsets. -/// -/// # Outputs -/// -/// Returns a [`HashMap`] from `ctx` to [`Vec`] of proofs. Each proofs ia a -/// pair. -fn prove_context_jumpdests(code: &[u8], ctx: &Context) -> HashMap> { - ctx.iter() - .map(|(&ctx, jumpdests)| { - let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { - get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) - }); - (ctx, proofs) - }) - .collect() -} - impl Interpreter { /// Returns an instance of `Interpreter` given `GenerationInputs`, and /// assuming we are initializing with the `KERNEL` code. diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 30f974525..bcdbfdddb 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -65,7 +65,7 @@ impl JumpDestTableProcessed { pub fn merge<'a>(jdts: impl IntoIterator) -> Self { jdts.into_iter().fold(Default::default(), |acc, next| { let mut inner = acc.0.clone(); - let b = next.iter().map(|(a,b)| (a.clone(), b.clone())); + let b = next.iter().map(|(a, b)| (a.clone(), b.clone())); inner.extend(b); JumpDestTableProcessed(inner) }) @@ -106,10 +106,6 @@ impl JumpDestTableWitness { acc.extend(next, cnt) }) } - - pub(crate) fn get_all_contexts(&self) -> Vec { - todo!() - } } // The following Display instances are added to make it easier to read diffs. diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index cb3daf559..0df366738 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -364,10 +364,6 @@ impl GenerationState { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; // TODO(einar-polygon) - // get_code from self.memory - let code = self.get_code(context)?; - let current_code = self.get_current_code()?; - if self.jumpdest_table.is_none() { self.jumpdest_table = Some(JumpDestTableProcessed::default()); } @@ -869,7 +865,7 @@ impl GenerationState { return Ok(Default::default()); } - Ok(sims.unwrap().0) + Ok(simp.unwrap()) } /// Given a HashMap containing the contexts and the jumpdest addresses, diff --git a/scripts/test_new_chain.sh b/scripts/test_new_chain.sh index 17bb650ef..529a55b70 100755 --- a/scripts/test_new_chain.sh +++ b/scripts/test_new_chain.sh @@ -135,7 +135,7 @@ FAILING_BLOCKS2=" #BLOCKS="$(seq $STATICTIP)" -BLOCKS="$(seq 105 6555)" +BLOCKS="$(seq 105 106)" #BLOCKS=$FAILING_BLOCKS1 #BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` From 54f50e9e1640043a8b304a8dbdae81e2934ee206 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 21:58:00 +0100 Subject: [PATCH 102/107] . --- scripts/test_new_chain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_new_chain.sh b/scripts/test_new_chain.sh index 529a55b70..d3401c299 100755 --- a/scripts/test_new_chain.sh +++ b/scripts/test_new_chain.sh @@ -135,7 +135,7 @@ FAILING_BLOCKS2=" #BLOCKS="$(seq $STATICTIP)" -BLOCKS="$(seq 105 106)" +BLOCKS="$(seq 105 110)" #BLOCKS=$FAILING_BLOCKS1 #BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` From 6a097ee0c87510b836b53e9c41606eadca136c06 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 22:37:08 +0100 Subject: [PATCH 103/107] . --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 6c9933f51..d25fb3a33 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -107,7 +107,13 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); + + if let Some(cc) = interpreter.generation_state.jumpdest_table { + interpreter.generation_state.jumpdest_table = + Some(JumpDestTableProcessed::merge([&cc, &jdtp])); + } else { + interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); + } Some((jdtp, jdtw)) } } From 75d6d80fb62daa53b3f2413a2ca8f39668edf7d5 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 22:50:47 +0100 Subject: [PATCH 104/107] . --- scripts/prove_stdio.sh | 2 +- scripts/test_new_chain.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 1e432c8f7..c586dbca0 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -37,7 +37,7 @@ export TOKIO_WORKER_THREADS=$num_procs #export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=info +export RUST_LOG=debug # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' diff --git a/scripts/test_new_chain.sh b/scripts/test_new_chain.sh index d3401c299..3ca997f07 100755 --- a/scripts/test_new_chain.sh +++ b/scripts/test_new_chain.sh @@ -135,7 +135,7 @@ FAILING_BLOCKS2=" #BLOCKS="$(seq $STATICTIP)" -BLOCKS="$(seq 105 110)" +BLOCKS="$(seq 106 107)" #BLOCKS=$FAILING_BLOCKS1 #BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` From 7b3a9afafc4848c6faf357a1260504fdc54662c9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 23:02:52 +0100 Subject: [PATCH 105/107] . --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index d25fb3a33..3acbe9835 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -108,12 +108,12 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( clock ); - if let Some(cc) = interpreter.generation_state.jumpdest_table { - interpreter.generation_state.jumpdest_table = - Some(JumpDestTableProcessed::merge([&cc, &jdtp])); - } else { - interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); - } + // if let Some(cc) = interpreter.generation_state.jumpdest_table { + // interpreter.generation_state.jumpdest_table = + // Some(JumpDestTableProcessed::merge([&cc, &jdtp])); + // } else { + // interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); + // } Some((jdtp, jdtw)) } } From 551aa3eed46279962fac487ba800a7b32aadf98f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 3 Nov 2024 23:22:53 +0100 Subject: [PATCH 106/107] . --- evm_arithmetization/src/generation/prover_input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 0df366738..69ff871f3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -362,8 +362,8 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { // QUESTION: Is there a reason we do not use `self.registers.context` here? let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + assert_eq!(context, self.registers.context); - // TODO(einar-polygon) if self.jumpdest_table.is_none() { self.jumpdest_table = Some(JumpDestTableProcessed::default()); } From 0572f3d470fe869d58b80e65b5fc232a39d8c435 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 4 Nov 2024 12:38:34 +0100 Subject: [PATCH 107/107] . --- scripts/prove_stdio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index c586dbca0..1ef3ee90b 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -37,7 +37,7 @@ export TOKIO_WORKER_THREADS=$num_procs #export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=debug +export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld'