Skip to content

Commit

Permalink
cleaner
Browse files Browse the repository at this point in the history
  • Loading branch information
@0xaatif
0xaatif committed Jun 12, 2024
1 parent 8ccd0c2 commit f27e05a
Show file tree
Hide file tree
Showing 6 changed files with 73 additions and 249 deletions.
18 changes: 1 addition & 17 deletions trace_decoder/src/compact/compact_prestate_processing.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,29 +365,12 @@ impl Header {
}
}

// #[derive(Debug)]
// pub struct CompactWitnessDecodingOutput {
// pub tries: PartialTriePreImages,
// pub code: Option<HashMap<CodeHash, Vec<u8>>>,
// }

#[derive(Debug)]
struct ParserState {
entries: WitnessEntries,
}

impl ParserState {
fn create_and_extract_header(
witness_bytes_raw: Vec<u8>,
) -> CompactParsingResult<(Header, Self)> {
let witness_bytes = WitnessBytes::<CompactCursorFast>::new(witness_bytes_raw);
let (header, entries) = witness_bytes.process_into_instructions_and_header()?;

let p_state = Self { entries };

Ok((header, p_state))
}

fn apply_rules_to_witness_entries(
&mut self,
entry_buf: &mut Vec<WitnessEntry>,
Expand Down Expand Up @@ -1223,6 +1206,7 @@ pub struct ProcessedCompactOutput {
pub witness_out: StateTrieExtractionOutput,
}

#[cfg(test)]
pub fn testme(bytes: &[u8]) -> (Vec<Instruction>, NodeEntry, StateTrieExtractionOutput) {
let witness_bytes = WitnessBytes::<DebugCompactCursor>::new(bytes.to_vec());
let (_header, entries) = witness_bytes
Expand Down
24 changes: 20 additions & 4 deletions trace_decoder/src/decoding.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,6 @@ pub enum TraceParsingErrorReason {
#[error("Missing account storage trie in base trie when constructing subset partial trie for txn (account: {0:x})")]
MissingAccountStorageTrie(HashedAccountAddr),

/// Failure due to trying to access a non-existent key in the trie.
#[error("Tried accessing a non-existent key ({1:x}) in the {0} trie (root hash: {2:x})")]
NonExistentTrieEntry(TrieType, Nibbles, TrieRootHash),

/// Failure due to missing keys when creating a sub-partial trie.
#[error("Missing key {0:x} when creating sub-partial tries (Trie type: {1})")]
MissingKeysCreatingSubPartialTrie(Nibbles, TrieType),
Expand Down Expand Up @@ -529,6 +525,26 @@ impl ProcessedBlockTrace {
/// allow the proof generation process to finish. Specifically, we need
/// at least two entries to generate an agg proof, and we need an agg
/// proof to generate a block proof. These entries do not mutate state.
///
/// The IR is used to generate root proofs, then aggregation proofs and
/// finally block proofs. Because aggregation proofs require at least
/// two entries, we pad the vector of IRs thanks to additional dummy
/// payload intermediary representations whenever necessary.
///
/// ### [Withdrawals](https://ethereum.org/staking/withdrawals) and Padding
///
/// Withdrawals are all proven together in a dummy payload. A dummy payload
/// corresponds to the IR of a proof with no transaction. They must,
/// however, be proven last. The padding is therefore carried out as
/// follows: If there are no transactions in the block, we add two dummy
/// transactions. The withdrawals -- if any -- are added to the second
/// dummy transaction. If there is only one transaction in the block, we
/// add one dummy transaction. If there are withdrawals, the dummy
/// transaction is at the end. Otherwise, it is added at the start. If
/// there are two or more transactions:
/// - if there are no withdrawals, no dummy transactions are added
/// - if there are withdrawals, one dummy transaction is added at the end,
/// with all the withdrawals in it.
fn pad_gen_inputs_with_dummy_inputs_if_needed(
gen_inputs: &mut Vec<GenerationInputs>,
other_data: &OtherBlockData,
Expand Down
179 changes: 37 additions & 142 deletions trace_decoder/src/lib.rs
View file
Original file line number Diff line number Diff line change
@@ -1,120 +1,27 @@
//! This library generates an Intermediary Representation (IR) of
//! a block's transactions, given a [BlockTrace] and some additional
//! data represented by [OtherBlockData].
//! The trace protocol for sending proof information to a prover scheduler.
//!
//! A [BlockTrace] is defined as follows:
//! ```ignore
//! pub struct BlockTrace {
//! /// The state and storage trie pre-images (i.e. the tries before
//! /// the execution of the current block) in multiple possible formats.
//! pub trie_pre_images: BlockTraceTriePreImages,
//! /// Traces and other info per transaction. The index of the transaction
//! /// within the block corresponds to the slot in this vec.
//! pub txn_info: Vec<TxnInfo>,
//! }
//! ```
//! The trie preimages are the hashed partial tries at the
//! start of the block. A [TxnInfo] contains all the transaction data
//! necessary to generate an IR.
//! Because parsing performance has a very negligible impact on overall proof
//! generation latency & throughput, the overall priority of this protocol is
//! ease of implementation for clients. The flexibility comes from giving
//! multiple ways to the client to provide the data for the protocol, where the
//! implementors can pick whichever way is the most convenient for them.
//!
//! # Usage
//! It might not be obvious why we need traces for each txn in order to generate
//! proofs. While it's true that we could just run all the txns of a block in an
//! EVM to generate the traces ourselves, there are a few major downsides:
//! - The client is likely a full node and already has to run the txns in an EVM
//! anyways.
//! - We want this protocol to be as agnostic as possible to the underlying
//! chain that we're generating proofs for, and running our own EVM would
//! likely cause us to loose this genericness.
//!
//! [The zero-bin prover](https://github.com/topos-protocol/zero-bin/blob/main/prover/src/lib.rs)
//! provides a use case for this library:
//! ```ignore
//! pub async fn prove(
//! // In this example, [self] is a [ProverInput] storing a [BlockTrace] and
//! // [OtherBlockData].
//! self,
//! runtime: &Runtime,
//! previous: Option<PlonkyProofIntern>,
//! ) -> Result<GeneratedBlockProof> {
//! let block_number = self.get_block_number();
//! info!("Proving block {block_number}");
//!
//! let other_data = self.other_data;
//! // The method calls [into_txn_proof_gen_ir] (see below) to
//! // generate an IR for each block transaction.
//! let txs = self.block_trace.into_txn_proof_gen_ir(
//! &ProcessingMeta::new(resolve_code_hash_fn),
//! other_data.clone(),
//! )?;
//!
//! // The block IRs are provided to the prover to generate an
//! // aggregation proof.
//! let agg_proof = IndexedStream::from(txs)
//! .map(&TxProof)
//! .fold(&AggProof)
//! .run(runtime)
//! .await?;
//!
//!
//! if let AggregatableProof::Agg(proof) = agg_proof {
//! let prev = previous.map(|p| GeneratedBlockProof {
//! b_height: block_number.as_u64() - 1,
//! intern: p,
//! });
//!
//! // The final aggregation proof is then used to prove the
//! // current block.
//! let block_proof = Literal(proof)
//! .map(&BlockProof { prev })
//! .run(runtime)
//! .await?;
//!
//! info!("Successfully proved block {block_number}");
//! Ok(block_proof.0)
//! } else {
//! bail!("AggProof is is not GeneratedAggProof")
//! }
//! }
//! ```
//!
//! As we see in the example, to turn a [BlockTrace] into a
//! vector of IRs, one must call the method
//! [into_txn_proof_gen_ir](BlockTrace::into_txn_proof_gen_ir):
//! ```ignore
//! pub fn into_txn_proof_gen_ir<F>(
//! self,
//! // Specifies the way code hashes should be dealt with.
//! p_meta: &ProcessingMeta<F>,
//! // Extra data needed for proof generation.
//! other_data: OtherBlockData,
//! ) -> TraceParsingResult<Vec<GenerationInputs>>
//! ```
//!
//! It first preprocesses the [BlockTrace] to provide transaction,
//! withdrawals and tries data that can be directly used to generate an IR.
//! For each transaction,
//! [into_txn_proof_gen_ir](BlockTrace::into_txn_proof_gen_ir) extracts the
//! necessary data from the processed transaction information to
//! return the IR.
//!
//! The IR is used to generate root proofs, then aggregation proofs and finally
//! block proofs. Because aggregation proofs require at least two entries, we
//! pad the vector of IRs thanks to additional dummy payload intermediary
//! representations whenever necessary.
//!
//! ### [Withdrawals](https://ethereum.org/staking/withdrawals) and Padding
//!
//! Withdrawals are all proven together in a dummy payload. A dummy payload
//! corresponds to the IR of a proof with no transaction. They must, however, be
//! proven last. The padding is therefore carried out as follows: If there are
//! no transactions in the block, we add two dummy transactions. The withdrawals
//! -- if any -- are added to the second dummy transaction. If there is only one
//! transaction in the block, we add one dummy transaction. If
//! there are withdrawals, the dummy transaction is at the end. Otherwise, it is
//! added at the start. If there are two or more transactions:
//! - if there are no withdrawals, no dummy transactions are added
//! - if there are withdrawals, one dummy transaction is added at the end, with
//! all the withdrawals in it.
//! While it's also true that we run our own zk-EVM (plonky2) to generate
//! proofs, it's critical that we are able to generate txn proofs in parallel.
//! Since generating proofs with plonky2 is very slow, this would force us to
//! sequentialize the entire proof generation process. So in the end, it's ideal
//! if we can get this information sent to us instead.
#![feature(linked_list_cursors)]
#![feature(trait_alias)]
#![feature(iter_array_chunks)]

pub use decoding::TraceParsingError;
pub use processed_block_trace::ProcessingMeta;

/// Provides debugging tools and a compact representation of state and storage
/// tries, used in tests.
Expand All @@ -129,29 +36,6 @@ mod processed_block_trace;
mod types;
/// Defines useful functions necessary to the other modules.
mod utils;

// ! The trace protocol for sending proof information to a prover scheduler.
// !
// ! Because parsing performance has a very negligible impact on overall proof
// ! generation latency & throughput, the overall priority of this protocol is
// ! ease of implementation for clients. The flexibility comes from giving
// ! multiple ways to the client to provide the data for the protocol, where the
// ! implementors can pick whichever way is the most convenient for them.
// !
// ! It might not be obvious why we need traces for each txn in order to
// generate ! proofs. While it's true that we could just run all the txns of a
// block in an ! EVM to generate the traces ourselves, there are a few major
// downsides: ! - The client is likely a full node and already has to run the
// txns in an EVM ! anyways.
// ! - We want this protocol to be as agnostic as possible to the underlying
// ! chain that we're generating proofs for, and running our own EVM would
// ! likely cause us to loose this genericness.
// !
// ! While it's also true that we run our own zk-EVM (plonky2) to generate
// ! proofs, it's critical that we are able to generate txn proofs in parallel.
// ! Since generating proofs with plonky2 is very slow, this would force us to
// ! sequentialize the entire proof generation process. So in the end, it's
// ideal ! if we can get this information sent to us instead.
use std::collections::HashMap;

use ethereum_types::Address;
Expand All @@ -169,13 +53,18 @@ use crate::{
/// Core payload needed to generate a proof for a block. Note that the scheduler
/// may need to request some additional data from the client along with this in
/// order to generate a proof.
///
/// The trie preimages are the hashed partial tries at the
/// start of the block. A [TxnInfo] contains all the transaction data
/// necessary to generate an IR.
#[derive(Debug, Deserialize, Serialize)]
pub struct BlockTrace {
/// The trie pre-images (state & storage) in multiple possible formats.
/// The state and storage trie pre-images (i.e. the tries before
/// the execution of the current block) in multiple possible formats.
pub trie_pre_images: BlockTraceTriePreImages,

/// Traces and other info per txn. The index of the txn corresponds to the
/// slot in this vec.
/// Traces and other info per transaction. The index of the transaction
/// within the block corresponds to the slot in this vec.
pub txn_info: Vec<TxnInfo>,
}

Expand Down Expand Up @@ -353,12 +242,15 @@ pub struct BlockLevelData {
pub withdrawals: Vec<(ethereum_types::Address, ethereum_types::U256)>,
}

pub fn legacy(
pub fn old(
block_trace: BlockTrace,
other_block_data: OtherBlockData,
resolver: impl Fn(&ethereum_types::H256) -> Vec<u8>,
) -> Result<Vec<evm_arithmetization_type_1::GenerationInputs>, Box<TraceParsingError>> {
block_trace.into_txn_proof_gen_ir(&ProcessingMeta::new(resolver), other_block_data)
) -> anyhow::Result<Vec<evm_arithmetization_type_1::GenerationInputs>> {
Ok(block_trace.into_txn_proof_gen_ir(
&processed_block_trace::ProcessingMeta::new(resolver),
other_block_data,
)?)
}

pub use type1::type1 as new;
Expand Down Expand Up @@ -495,7 +387,7 @@ mod type1 {
.enumerate()
{
println!("case {}", ix);
let (their_instructions, their_execution, their_reshaped) =
let (their_instructions, their_execution, _their_reshaped) =
crate::compact::compact_prestate_processing::testme(&case.bytes);

let instructions = wire::parse(&case.bytes).unwrap();
Expand Down Expand Up @@ -541,6 +433,7 @@ mod type1 {
}
}

#[cfg(test)]
fn instruction2instruction(
ours: wire::Instruction,
) -> crate::compact::compact_prestate_processing::Instruction {
Expand Down Expand Up @@ -581,6 +474,7 @@ mod type1 {
theirs
}

#[cfg(test)]
fn execution2node(
ours: execution::Execution,
) -> crate::compact::compact_prestate_processing::NodeEntry {
Expand All @@ -593,6 +487,7 @@ mod type1 {
})
}

#[cfg(test)]
fn node2node(ours: execution::Node) -> crate::compact::compact_prestate_processing::NodeEntry {
use either::Either;
use execution::*;
Expand Down
14 changes: 7 additions & 7 deletions trace_decoder/src/processed_block_trace.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use crate::compact::compact_prestate_processing::{
};
use crate::decoding::{TraceParsingError, TraceParsingResult};
use crate::types::{
CodeHash, CodeHashResolveFunc, HashedAccountAddr, HashedNodeAddr, HashedStorageAddrNibbles,
TrieRootHash, EMPTY_CODE_HASH, EMPTY_TRIE_HASH,
CodeHash, HashedAccountAddr, HashedNodeAddr, HashedStorageAddrNibbles, TrieRootHash,
EMPTY_CODE_HASH, EMPTY_TRIE_HASH,
};
use crate::utils::{
hash, print_value_and_hash_nodes_of_storage_trie, print_value_and_hash_nodes_of_trie,
Expand All @@ -41,7 +41,7 @@ impl BlockTrace {
other_data: OtherBlockData,
) -> TraceParsingResult<Vec<GenerationInputs>>
where
F: CodeHashResolveFunc,
F: Fn(&CodeHash) -> Vec<u8>,
{
let withdrawals = other_data.b_data.withdrawals.clone();

Expand Down Expand Up @@ -153,14 +153,14 @@ fn process_block_trace_trie_pre_images(
#[derive(Debug)]
pub struct ProcessingMeta<F>
where
F: CodeHashResolveFunc,
F: Fn(&CodeHash) -> Vec<u8>,
{
resolve_code_hash_fn: F,
}

impl<F> ProcessingMeta<F>
where
F: CodeHashResolveFunc,
F: Fn(&CodeHash) -> Vec<u8>,
{
/// Returns a `ProcessingMeta` given the provided code hash resolving
/// function.
Expand Down Expand Up @@ -190,7 +190,7 @@ pub(crate) struct CodeHashResolving<F> {
pub extra_code_hash_mappings: HashMap<CodeHash, Vec<u8>>,
}

impl<F: CodeHashResolveFunc> CodeHashResolving<F> {
impl<F: Fn(&CodeHash) -> Vec<u8>> CodeHashResolving<F> {
fn resolve(&mut self, c_hash: &CodeHash) -> Vec<u8> {
match self.extra_code_hash_mappings.get(c_hash) {
Some(code) => code.clone(),
Expand All @@ -204,7 +204,7 @@ impl<F: CodeHashResolveFunc> CodeHashResolving<F> {
}

impl TxnInfo {
pub(crate) fn into_processed_txn_info<F: CodeHashResolveFunc>(
pub(crate) fn into_processed_txn_info<F: Fn(&CodeHash) -> Vec<u8>>(
self,
all_accounts_in_pre_image: &[(HashedAccountAddr, AccountRlp)],
extra_state_accesses: &[HashedAccountAddr],
Expand Down
Loading

0 comments on commit f27e05a

Please sign in to comment.