From 4d917382dc322fcf1b029471bf48a7cb18fadeaf Mon Sep 17 00:00:00 2001 From: Vladimir Trifonov Date: Wed, 24 Apr 2024 20:07:23 +0300 Subject: [PATCH] feat: provide IR for debugging upon failure (#48) * feat: provide IR for debugging upon failure * fix: fix clippy issues * fix: fix pr comments * fix: make evm_arithmetization non optional for ops * fix: fix pr comments * fix: fix clippy issues * fix: fix clippy issue * fix: fix pr comment * fix: fix clippy issue * fix: fix cargo lock * fix: fix pr comments * fix: fix format issues * fix: fix save input on error for test-only * fix: fix pr comments * fix: fix * fix: fix clippy issue * fix: fmt fix --------- Co-authored-by: Vladimir Trifonov --- Cargo.lock | 2 + README.md | 5 ++ common/Cargo.toml | 2 + common/src/debug_utils.rs | 102 ++++++++++++++++++++++++++++++++++++ common/src/lib.rs | 1 + leader/src/cli.rs | 9 ++++ leader/src/http.rs | 16 ++++-- leader/src/jerigon.rs | 5 +- leader/src/main.rs | 17 ++++-- leader/src/stdio.rs | 3 +- ops/src/lib.rs | 107 +++++++++++++++++++++++++++++++++----- prover/src/lib.rs | 19 +++++-- 12 files changed, 263 insertions(+), 25 deletions(-) create mode 100644 common/src/debug_utils.rs diff --git a/Cargo.lock b/Cargo.lock index d9777c37..ba777c30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -641,6 +641,8 @@ dependencies = [ "plonky2", "proof_gen", "seahash", + "serde", + "serde_json", "thiserror", "trace_decoder", "tracing", diff --git a/README.md b/README.md index 6b410242..d826ab96 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ A composition of [`paladin`](https://github.com/0xPolygonZero/paladin) and [`plo - [RPC Usage](#rpc-usage) - [Docker](#docker) - [Development Branches](#development-branches) + - [Testing Blocks](#testing-blocks) + - [Proving Blocks](#proving-blocks) + - [Generating Witnesses Only](#generating-witnesses-only) - [License](#license) - [Contribution](#contribution) @@ -217,6 +220,8 @@ Options: If provided, write the generated proof to this file instead of stdout -h, --help Print help + -s, --save-inputs-on-error + If provided, save the public inputs to disk on error ``` Prove a block. diff --git a/common/Cargo.toml b/common/Cargo.toml index 2eab391d..9a08bbd1 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -17,4 +17,6 @@ evm_arithmetization = { workspace = true } clap = { workspace = true } anyhow = { workspace = true } trace_decoder = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } seahash = "4.1.0" diff --git a/common/src/debug_utils.rs b/common/src/debug_utils.rs new file mode 100644 index 00000000..f8cb53dd --- /dev/null +++ b/common/src/debug_utils.rs @@ -0,0 +1,102 @@ +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use serde::Serialize; +use serde_json::Error as SerdeError; +use thiserror::Error; + +const DEBUG_FOLDER: &str = "./debug"; + +/// Ensures that the specified directory exists on the filesystem. +/// +/// This function checks if the directory at `folder_path` exists. If not, it +/// attempts to create the directory. It returns an error if the path is not a +/// directory or if there are issues accessing or creating the directory. +/// +/// # Parameters +/// * `folder_path` - A reference to a `Path` that specifies the directory to +/// check or create. +/// +/// # Returns +/// * `Ok(())` - The directory exists or was successfully created. +/// * `Err(io::Error)` - The path is not a directory, or there was a problem +/// accessing or creating the directory. +fn ensure_directory_exists(folder_path: &Path) -> io::Result<()> { + match fs::metadata(folder_path) { + Ok(metadata) => { + if metadata.is_dir() { + Ok(()) // The directory already exists + } else { + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "The path exists but is not a directory", + )) + } + } + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + // Directory does not exist, try to create it + fs::create_dir(folder_path) + } else { + // Re-throw the error if it's not a 'NotFound' error + Err(e) + } + } + } +} + +/// An error type for save debug input information. +#[derive(Error, Debug)] +pub enum SaveInputError { + #[error("failed to create directory '{0}'")] + CreateDirectoryError(PathBuf, #[source] io::Error), + + #[error("failed to create file '{0}'")] + CreateFileError(PathBuf, #[source] io::Error), + + #[error("failed to serialize inputs")] + SerializationError(#[source] SerdeError), + + #[error("failed to write to file '{0}'")] + WriteToFileError(PathBuf, #[source] io::Error), +} + +/// Serializes a collection of inputs to a pretty-printed JSON format and saves +/// them to a file. +/// +/// # Arguments +/// +/// * `file_name` - The name of the file (including the extension) where the +/// serialized data will be saved. +/// * `inputs` - A collection of items to be serialized. Each item in the +/// collection must implement the `Serialize` trait. +/// +/// # Returns +/// +/// This function returns a `Result<(), std::io::Error>` indicating the +/// operation's success or failure. +pub fn save_inputs_to_disk( + file_name: String, + inputs: T, +) -> Result<(), SaveInputError> { + let debug_folder = Path::new(DEBUG_FOLDER); + let input_file_path = debug_folder.join(file_name); + + // Ensure the DEBUG_FOLDER exists + ensure_directory_exists(debug_folder) + .map_err(|e| SaveInputError::CreateDirectoryError(debug_folder.to_path_buf(), e))?; + + let mut file = File::create(&input_file_path) + .map_err(|e| SaveInputError::CreateFileError(input_file_path.clone(), e))?; + + // Serialize the entire collection to a pretty JSON string + let all_inputs_str = + serde_json::to_string_pretty(&inputs).map_err(SaveInputError::SerializationError)?; + + // Write the serialized data to the file + file.write_all(all_inputs_str.as_bytes()) + .map_err(|e| SaveInputError::WriteToFileError(input_file_path, e))?; + + Ok(()) +} diff --git a/common/src/lib.rs b/common/src/lib.rs index 234099cf..e9e7b504 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,2 +1,3 @@ +pub mod debug_utils; pub mod parsing; pub mod prover_state; diff --git a/leader/src/cli.rs b/leader/src/cli.rs index 6ab9d8fb..10316f6a 100644 --- a/leader/src/cli.rs +++ b/leader/src/cli.rs @@ -25,6 +25,9 @@ pub(crate) enum Command { /// The previous proof output. #[arg(long, short = 'f', value_hint = ValueHint::FilePath)] previous_proof: Option, + /// If true, save the public inputs to disk on error. + #[arg(short, long, default_value_t = false)] + save_inputs_on_error: bool, }, /// Reads input from a Jerigon node and writes output to stdout. Jerigon { @@ -44,6 +47,9 @@ pub(crate) enum Command { /// stdout. #[arg(long, short = 'o', value_hint = ValueHint::FilePath)] proof_output_path: Option, + /// If true, save the public inputs to disk on error. + #[arg(short, long, default_value_t = false)] + save_inputs_on_error: bool, }, /// Reads input from HTTP and writes output to a directory. Http { @@ -53,5 +59,8 @@ pub(crate) enum Command { /// The directory to which output should be written. #[arg(short, long, value_hint = ValueHint::DirPath)] output_dir: PathBuf, + /// If true, save the public inputs to disk on error. + #[arg(short, long, default_value_t = false)] + save_inputs_on_error: bool, }, } diff --git a/leader/src/http.rs b/leader/src/http.rs index 84187794..80111ce4 100644 --- a/leader/src/http.rs +++ b/leader/src/http.rs @@ -11,7 +11,12 @@ use serde_json::to_writer; use tracing::{debug, error, info}; /// The main function for the HTTP mode. -pub(crate) async fn http_main(runtime: Runtime, port: u16, output_dir: PathBuf) -> Result<()> { +pub(crate) async fn http_main( + runtime: Runtime, + port: u16, + output_dir: PathBuf, + save_inputs_on_error: bool, +) -> Result<()> { let addr = SocketAddr::from(([0, 0, 0, 0], port)); debug!("listening on {}", addr); @@ -20,7 +25,7 @@ pub(crate) async fn http_main(runtime: Runtime, port: u16, output_dir: PathBuf) "/prove", post({ let runtime = runtime.clone(); - move |body| prove(body, runtime, output_dir.clone()) + move |body| prove(body, runtime, output_dir.clone(), save_inputs_on_error) }), ); let listener = tokio::net::TcpListener::bind(&addr).await?; @@ -60,12 +65,17 @@ async fn prove( Json(payload): Json, runtime: Arc, output_dir: PathBuf, + save_inputs_on_error: bool, ) -> StatusCode { debug!("Received payload: {:#?}", payload); let block_number = payload.prover_input.get_block_number(); - match payload.prover_input.prove(&runtime, payload.previous).await { + match payload + .prover_input + .prove(&runtime, payload.previous, save_inputs_on_error) + .await + { Ok(b_proof) => match write_to_file(output_dir, block_number, &b_proof) { Ok(file) => { info!("Successfully wrote proof to {}", file.display()); diff --git a/leader/src/jerigon.rs b/leader/src/jerigon.rs index 899ea20e..6719112a 100644 --- a/leader/src/jerigon.rs +++ b/leader/src/jerigon.rs @@ -16,6 +16,7 @@ pub(crate) async fn jerigon_main( checkpoint_block_number: u64, previous: Option, proof_output_path_opt: Option, + save_inputs_on_error: bool, ) -> Result<()> { let prover_input = rpc::fetch_prover_input(rpc::FetchProverInputRequest { rpc_url, @@ -24,7 +25,9 @@ pub(crate) async fn jerigon_main( }) .await?; - let proof = prover_input.prove(&runtime, previous).await; + let proof = prover_input + .prove(&runtime, previous, save_inputs_on_error) + .await; runtime.close().await?; let proof = serde_json::to_vec(&proof?.intern)?; diff --git a/leader/src/main.rs b/leader/src/main.rs index 36d2f1e3..8c054ea4 100644 --- a/leader/src/main.rs +++ b/leader/src/main.rs @@ -47,11 +47,18 @@ async fn main() -> Result<()> { let runtime = Runtime::from_config(&args.paladin, register()).await?; match args.command { - Command::Stdio { previous_proof } => { + Command::Stdio { + previous_proof, + save_inputs_on_error, + } => { let previous_proof = get_previous_proof(previous_proof)?; - stdio::stdio_main(runtime, previous_proof).await?; + stdio::stdio_main(runtime, previous_proof, save_inputs_on_error).await?; } - Command::Http { port, output_dir } => { + Command::Http { + port, + output_dir, + save_inputs_on_error, + } => { // check if output_dir exists, is a directory, and is writable let output_dir_metadata = std::fs::metadata(&output_dir); if output_dir_metadata.is_err() { @@ -61,7 +68,7 @@ async fn main() -> Result<()> { panic!("output-dir is not a writable directory"); } - http::http_main(runtime, port, output_dir).await?; + http::http_main(runtime, port, output_dir, save_inputs_on_error).await?; } Command::Jerigon { rpc_url, @@ -69,6 +76,7 @@ async fn main() -> Result<()> { checkpoint_block_number, previous_proof, proof_output_path, + save_inputs_on_error, } => { let previous_proof = get_previous_proof(previous_proof)?; @@ -79,6 +87,7 @@ async fn main() -> Result<()> { checkpoint_block_number, previous_proof, proof_output_path, + save_inputs_on_error, ) .await?; } diff --git a/leader/src/stdio.rs b/leader/src/stdio.rs index e3b6ec5a..7f1e6e3f 100644 --- a/leader/src/stdio.rs +++ b/leader/src/stdio.rs @@ -9,13 +9,14 @@ use prover::ProverInput; pub(crate) async fn stdio_main( runtime: Runtime, previous: Option, + save_inputs_on_error: bool, ) -> Result<()> { let mut buffer = String::new(); std::io::stdin().read_to_string(&mut buffer)?; let des = &mut serde_json::Deserializer::from_str(&buffer); let input: ProverInput = serde_path_to_error::deserialize(des)?; - let proof = input.prove(&runtime, previous).await; + let proof = input.prove(&runtime, previous, save_inputs_on_error).await; runtime.close().await?; let proof = proof?; diff --git a/ops/src/lib.rs b/ops/src/lib.rs index 1200047c..5b6caf69 100644 --- a/ops/src/lib.rs +++ b/ops/src/lib.rs @@ -1,7 +1,7 @@ use std::time::Instant; -use common::prover_state::p_state; -use evm_arithmetization::GenerationInputs; +use common::{debug_utils::save_inputs_to_disk, prover_state::p_state}; +use evm_arithmetization::{proof::PublicValues, GenerationInputs}; use keccak_hash::keccak; use paladin::{ operation::{FatalError, FatalStrategy, Monoid, Operation, Result}, @@ -12,12 +12,14 @@ use proof_gen::{ proof_types::{AggregatableProof, GeneratedAggProof, GeneratedBlockProof}, }; use serde::{Deserialize, Serialize}; -use tracing::{event, info_span, Level}; +use tracing::{error, event, info_span, Level}; registry!(); #[derive(Deserialize, Serialize, RemoteExecute)] -pub struct TxProof; +pub struct TxProof { + pub save_inputs_on_error: bool, +} #[cfg(not(feature = "test_only"))] impl Operation for TxProof { @@ -26,9 +28,27 @@ impl Operation for TxProof { fn execute(&self, input: Self::Input) -> Result { let _span = TxProofSpan::new(&input); - let proof = common::prover_state::p_manager() - .generate_txn_proof(input) - .map_err(|err| FatalError::from_anyhow(err, FatalStrategy::Terminate))?; + let proof = if self.save_inputs_on_error { + common::prover_state::p_manager() + .generate_txn_proof(input.clone()) + .map_err(|err| { + if let Err(write_err) = save_inputs_to_disk( + format!( + "b{}_txn_{}_input.log", + input.block_metadata.block_number, input.txn_number_before + ), + input, + ) { + error!("Failed to save txn proof input to disk: {:?}", write_err); + } + + FatalError::from_anyhow(err, FatalStrategy::Terminate) + })? + } else { + common::prover_state::p_manager() + .generate_txn_proof(input) + .map_err(|err| FatalError::from_anyhow(err, FatalStrategy::Terminate))? + }; Ok(proof.into()) } @@ -41,8 +61,30 @@ impl Operation for TxProof { fn execute(&self, input: Self::Input) -> Result { let _span = TxProofSpan::new(&input); - evm_arithmetization::prover::testing::simulate_execution::(input) + + if self.save_inputs_on_error { + evm_arithmetization::prover::testing::simulate_execution::( + input.clone(), + ) + .map_err(|err| { + if let Err(write_err) = save_inputs_to_disk( + format!( + "b{}_txn_{}_input.log", + input.block_metadata.block_number, input.txn_number_before + ), + input, + ) { + error!("Failed to save txn proof input to disk: {:?}", write_err); + } + + FatalError::from_anyhow(err, FatalStrategy::Terminate) + })?; + } else { + evm_arithmetization::prover::testing::simulate_execution::( + input, + ) .map_err(|err| FatalError::from_anyhow(err, FatalStrategy::Terminate))?; + } Ok(()) } @@ -106,13 +148,40 @@ impl Drop for TxProofSpan { } #[derive(Deserialize, Serialize, RemoteExecute)] -pub struct AggProof; +pub struct AggProof { + pub save_inputs_on_error: bool, +} + +fn get_agg_proof_public_values(elem: AggregatableProof) -> PublicValues { + match elem { + AggregatableProof::Txn(info) => info.p_vals, + AggregatableProof::Agg(info) => info.p_vals, + } +} impl Monoid for AggProof { type Elem = AggregatableProof; fn combine(&self, a: Self::Elem, b: Self::Elem) -> Result { - let result = generate_agg_proof(p_state(), &a, &b).map_err(FatalError::from)?; + let result = generate_agg_proof(p_state(), &a, &b).map_err(|e| { + if self.save_inputs_on_error { + let pv = vec![ + get_agg_proof_public_values(a), + get_agg_proof_public_values(b), + ]; + if let Err(write_err) = save_inputs_to_disk( + format!( + "b{}_agg_lhs_rhs_inputs.log", + pv[0].block_metadata.block_number + ), + pv, + ) { + error!("Failed to save agg proof inputs to disk: {:?}", write_err); + } + } + + FatalError::from(e) + })?; Ok(result.into()) } @@ -126,6 +195,7 @@ impl Monoid for AggProof { #[derive(Deserialize, Serialize, RemoteExecute)] pub struct BlockProof { pub prev: Option, + pub save_inputs_on_error: bool, } impl Operation for BlockProof { @@ -134,8 +204,21 @@ impl Operation for BlockProof { fn execute(&self, input: Self::Input) -> Result { Ok( - generate_block_proof(p_state(), self.prev.as_ref(), &input) - .map_err(FatalError::from)?, + generate_block_proof(p_state(), self.prev.as_ref(), &input).map_err(|e| { + if self.save_inputs_on_error { + if let Err(write_err) = save_inputs_to_disk( + format!( + "b{}_block_input.log", + input.p_vals.block_metadata.block_number + ), + input.p_vals, + ) { + error!("Failed to save block proof input to disk: {:?}", write_err); + } + } + + FatalError::from(e) + })?, ) } } diff --git a/prover/src/lib.rs b/prover/src/lib.rs index abf424a9..8cef4303 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -35,6 +35,7 @@ impl ProverInput { self, runtime: &Runtime, previous: Option, + save_inputs_on_error: bool, ) -> Result { let block_number = self.get_block_number(); info!("Proving block {block_number}"); @@ -46,8 +47,12 @@ impl ProverInput { )?; let agg_proof = IndexedStream::from(txs) - .map(&TxProof) - .fold(&ops::AggProof) + .map(&TxProof { + save_inputs_on_error, + }) + .fold(&ops::AggProof { + save_inputs_on_error, + }) .run(runtime) .await?; @@ -58,7 +63,10 @@ impl ProverInput { }); let block_proof = paladin::directive::Literal(proof) - .map(&ops::BlockProof { prev }) + .map(&ops::BlockProof { + prev, + save_inputs_on_error, + }) .run(runtime) .await?; @@ -74,6 +82,7 @@ impl ProverInput { self, runtime: &Runtime, _previous: Option, + save_inputs_on_error: bool, ) -> Result { let block_number = self.get_block_number(); info!("Testing witness generation for block {block_number}."); @@ -85,7 +94,9 @@ impl ProverInput { )?; IndexedStream::from(txs) - .map(&TxProof) + .map(&TxProof { + save_inputs_on_error, + }) .run(runtime) .await? .try_collect::>()