RISCV executor

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "asm_utils",
     "airgen",
     "type_check",
+    "riscv_executor",
 ]
 
 [patch."https://github.com/privacy-scaling-explorations/halo2.git"]

analysis/Cargo.toml

@@ -9,10 +9,10 @@ edition = "2021"
 ast = { path = "../ast" }
 itertools = "^0.10"
 log = "0.4.18"
-number = { version = "0.1.0", path = "../number" }
+number = { path = "../number" }
 parser = { path = "../parser" }
 asm_to_pil = { path = "../asm_to_pil" }
-type_check = { version = "0.1.0", path = "../type_check" }
+type_check = { path = "../type_check" }
 
 [dev-dependencies]
 parser = { path = "../parser" }

analysis/src/lib.rs

@@ -10,9 +10,18 @@ pub use macro_expansion::MacroExpander;
 use ast::{asm_analysis::AnalysisASMFile, parsed::asm::ASMProgram, DiffMonitor};
 use number::FieldElement;
 
-pub fn analyze<T: FieldElement>(file: ASMProgram<T>) -> Result<AnalysisASMFile<T>, Vec<String>> {
+pub fn convert_asm_to_pil<T: FieldElement>(
+    file: ASMProgram<T>,
+) -> Result<AnalysisASMFile<T>, Vec<String>> {
     let mut monitor = DiffMonitor::default();
+    let file = analyze(file, &mut monitor)?;
+    Ok(convert_analyzed_to_pil_constraints(file, &mut monitor))
+}
 
+pub fn analyze<T: FieldElement>(
+    file: ASMProgram<T>,
+    monitor: &mut DiffMonitor,
+) -> Result<AnalysisASMFile<T>, Vec<String>> {
     // expand macros
     log::debug!("Run expand analysis step");
     let file = macro_expansion::expand(file);
@@ -23,15 +32,27 @@ pub fn analyze<T: FieldElement>(file: ASMProgram<T>) -> Result<AnalysisASMFile<T
 
     // run analysis on virtual machines, reducing them to constrained machines
     log::debug!("Start asm analysis");
-    let file = vm::analyze(file, &mut monitor)?;
+    let file = vm::analyze(file, monitor)?;
     log::debug!("End asm analysis");
 
+    Ok(file)
+}
+
+pub fn convert_analyzed_to_pil_constraints<T: FieldElement>(
+    file: AnalysisASMFile<T>,
+    monitor: &mut DiffMonitor,
+) -> AnalysisASMFile<T> {
+    // remove all asm (except external instructions)
+    log::debug!("Run asm_to_pil");
+    let file = asm_to_pil::compile(file);
+    monitor.push(&file);
+
     // enforce blocks using `operation_id` and `latch`
     log::debug!("Run enforce_block analysis step");
     let file = block_enforcer::enforce(file);
     monitor.push(&file);
 
-    Ok(file)
+    file
 }
 
 pub mod utils {

analysis/src/vm/mod.rs

@@ -19,10 +19,6 @@ pub fn analyze<T: FieldElement>(
     log::debug!("Run batch analysis step");
     let file = batcher::batch(file);
     monitor.push(&file);
-    // remove all asm (except external instructions)
-    log::debug!("Run asm_to_pil analysis step");
-    let file = asm_to_pil::compile(file);
-    monitor.push(&file);
 
     Ok(file)
 }

ast/src/asm_analysis/mod.rs

@@ -87,7 +87,7 @@ pub struct FunctionStatements<T> {
 }
 
 pub struct BatchRef<'a, T> {
-    statements: &'a [FunctionStatement<T>],
+    pub statements: &'a [FunctionStatement<T>],
     reason: &'a Option<IncompatibleSet>,
 }
 
@@ -151,7 +151,7 @@ impl<T> FunctionStatements<T> {
     }
 
     /// iterate over the batches by reference
-    fn iter_batches(&self) -> impl Iterator<Item = BatchRef<T>> {
+    pub fn iter_batches(&self) -> impl Iterator<Item = BatchRef<T>> {
         match &self.batches {
             Some(batches) => Either::Left(batches.iter()),
             None => Either::Right(
@@ -246,7 +246,7 @@ pub struct CallableSymbolDefinition<T> {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Default)]
-pub struct CallableSymbolDefinitions<T>(BTreeMap<String, CallableSymbol<T>>);
+pub struct CallableSymbolDefinitions<T>(pub BTreeMap<String, CallableSymbol<T>>);
 
 impl<T> IntoIterator for CallableSymbolDefinitions<T> {
     type Item = CallableSymbolDefinition<T>;

ast/src/parsed/asm.rs

@@ -325,6 +325,7 @@ pub enum FunctionStatement<T> {
 pub enum DebugDirective {
     File(usize, String, String),
     Loc(usize, usize, usize),
+    OriginalInstruction(String),
 }
 
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]

ast/src/parsed/display.rs

@@ -238,6 +238,9 @@ impl Display for DebugDirective {
             DebugDirective::Loc(file, line, col) => {
                 write!(f, "debug loc {file} {line} {col};")
             }
+            DebugDirective::OriginalInstruction(insn) => {
+                write!(f, "debug insn \"{insn}\";")
+            }
         }
     }
 }

backend/Cargo.toml

@@ -10,8 +10,8 @@ halo2 = ["dep:halo2"]
 halo2 = { path = "../halo2", optional = true }
 pil_analyzer = { path = "../pil_analyzer" }
 number = { path = "../number" }
+ast = { path = "../ast" }
 strum = { version = "0.24.1", features = ["derive"] }
-ast = { version = "0.1.0", path = "../ast" }
 log = "0.4.17"
 serde_json = "1.0"
 thiserror = "1.0.43"

compiler/Cargo.toml

@@ -19,10 +19,10 @@ pilopt = { path = "../pilopt" }
 asm_to_pil = { path = "../asm_to_pil" }
 pil_analyzer = { path = "../pil_analyzer" }
 halo2 = { path = "../halo2", optional = true }
-ast = { version = "0.1.0", path = "../ast" }
-analysis = { version = "0.1.0", path = "../analysis" }
-linker = { version = "0.1.0", path = "../linker" }
-airgen = { version = "0.1.0", path = "../airgen" }
+ast = { path = "../ast" }
+analysis = { path = "../analysis" }
+linker = { path = "../linker" }
+airgen = { path = "../airgen" }
 importer = { path = "../importer" }
 
 [dev-dependencies]

compiler/benches/executor_benchmark.rs

@@ -1,5 +1,4 @@
 use ::compiler::inputs_to_query_callback;
-use analysis::analyze;
 use ast::analyzed::Analyzed;
 use criterion::{criterion_group, criterion_main, Criterion};
 
@@ -19,15 +18,15 @@ fn get_pil() -> Analyzed<T> {
     let contents = compiler::compile(riscv_asm_files, &CoProcessors::base());
     let parsed = parser::parse_asm::<T>(None, &contents).unwrap();
     let resolved = importer::resolve(None, parsed).unwrap();
-    let analyzed = analyze(resolved).unwrap();
+    let analyzed = analysis::convert_asm_to_pil(resolved).unwrap();
     let graph = airgen::compile(analyzed);
     let pil = linker::link(graph).unwrap();
     let analyzed = pil_analyzer::analyze_string(&format!("{pil}"));
     pilopt::optimize(analyzed)
 }
 
 fn run_witgen<T: FieldElement>(analyzed: &Analyzed<T>, input: Vec<T>) {
-    let query_callback = inputs_to_query_callback(input);
+    let query_callback = inputs_to_query_callback(&input);
     let constants = constant_evaluator::generate(analyzed);
     executor::witgen::WitnessGenerator::new(analyzed, &constants, query_callback).generate();
 }

compiler/src/lib.rs

@@ -8,12 +8,15 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::time::Instant;
 
+use analysis::analyze;
+use analysis::convert_analyzed_to_pil_constraints;
 use ast::analyzed::Analyzed;
+use ast::DiffMonitor;
 
 pub mod util;
 mod verify;
 
-use analysis::analyze;
+use ast::asm_analysis::AnalysisASMFile;
 pub use backend::{BackendType, Proof};
 use executor::witgen::QueryCallback;
 pub use verify::{
@@ -122,6 +125,7 @@ pub fn compile_asm<T: FieldElement>(
         file_name,
         &contents,
         inputs,
+        None,
         output_dir,
         force_overwrite,
         prove_with,
@@ -130,20 +134,12 @@ pub fn compile_asm<T: FieldElement>(
     .1)
 }
 
-/// Compiles the contents of a .asm file, outputs the PIL on stdout and tries to generate
-/// fixed and witness columns.
-///
-/// Returns the relative pil file name and the compilation result if any compilation was done.
 #[allow(clippy::print_stderr)]
-pub fn compile_asm_string<T: FieldElement>(
+pub fn compile_asm_string_to_analyzed_ast<T: FieldElement>(
     file_name: &str,
     contents: &str,
-    inputs: Vec<T>,
-    output_dir: &Path,
-    force_overwrite: bool,
-    prove_with: Option<BackendType>,
-    external_witness_values: Vec<(&str, Vec<T>)>,
-) -> Result<(PathBuf, Option<CompilationResult<T>>), Vec<String>> {
+    monitor: &mut DiffMonitor,
+) -> Result<AnalysisASMFile<T>, Vec<String>> {
     let parsed = parser::parse_asm(Some(file_name), contents).unwrap_or_else(|err| {
         eprintln!("Error parsing .asm file:");
         err.output_to_stderr();
@@ -153,11 +149,27 @@ pub fn compile_asm_string<T: FieldElement>(
     let resolved =
         importer::resolve(Some(PathBuf::from(file_name)), parsed).map_err(|e| vec![e])?;
     log::debug!("Run analysis");
-    let analysed = analyze(resolved).unwrap();
+    let analyzed = analyze(resolved, monitor)?;
     log::debug!("Analysis done");
-    log::trace!("{analysed}");
+    log::trace!("{analyzed}");
+
+    Ok(analyzed)
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn convert_analyzed_to_pil<T: FieldElement>(
+    file_name: &str,
+    monitor: &mut DiffMonitor,
+    analyzed: AnalysisASMFile<T>,
+    inputs: Vec<T>,
+    output_dir: &Path,
+    force_overwrite: bool,
+    prove_with: Option<BackendType>,
+    external_witness_values: Vec<(&str, Vec<T>)>,
+) -> Result<(PathBuf, Option<CompilationResult<T>>), Vec<String>> {
+    let constraints = convert_analyzed_to_pil_constraints(analyzed, monitor);
     log::debug!("Run airgen");
-    let graph = airgen::compile(analysed);
+    let graph = airgen::compile(constraints);
     log::debug!("Airgen done");
     log::trace!("{graph}");
     log::debug!("Run linker");
@@ -179,7 +191,7 @@ pub fn compile_asm_string<T: FieldElement>(
         return Ok((pil_file_path, None));
     }
 
-    fs::write(pil_file_path.clone(), format!("{pil}")).unwrap();
+    fs::write(&pil_file_path, format!("{pil}")).unwrap();
 
     let pil_file_name = pil_file_path.file_name().unwrap();
     Ok((
@@ -195,6 +207,40 @@ pub fn compile_asm_string<T: FieldElement>(
     ))
 }
 
+pub type AnalyzedASTHook<'a, T> = &'a mut dyn FnMut(&AnalysisASMFile<T>);
+
+/// Compiles the contents of a .asm file, outputs the PIL on stdout and tries to generate
+/// fixed and witness columns.
+///
+/// Returns the relative pil file name and the compilation result if any compilation was done.
+#[allow(clippy::too_many_arguments)]
+pub fn compile_asm_string<T: FieldElement>(
+    file_name: &str,
+    contents: &str,
+    inputs: Vec<T>,
+    analyzed_hook: Option<AnalyzedASTHook<T>>,
+    output_dir: &Path,
+    force_overwrite: bool,
+    prove_with: Option<BackendType>,
+    external_witness_values: Vec<(&str, Vec<T>)>,
+) -> Result<(PathBuf, Option<CompilationResult<T>>), Vec<String>> {
+    let mut monitor = DiffMonitor::default();
+    let analyzed = compile_asm_string_to_analyzed_ast(file_name, contents, &mut monitor)?;
+    if let Some(hook) = analyzed_hook {
+        hook(&analyzed);
+    };
+    convert_analyzed_to_pil(
+        file_name,
+        &mut monitor,
+        analyzed,
+        inputs,
+        output_dir,
+        force_overwrite,
+        prove_with,
+        external_witness_values,
+    )
+}
+
 pub struct CompilationResult<T: FieldElement> {
     /// Constant columns, potentially incomplete (if success is false)
     pub constants: Vec<(String, Vec<T>)>,

compiler/src/verify.rs

@@ -16,6 +16,7 @@ pub fn verify_asm_string<T: FieldElement>(file_name: &str, contents: &str, input
         file_name,
         contents,
         inputs,
+        None,
         &temp_dir,
         true,
         Some(BackendType::PilStarkCli),

executor/Cargo.toml

@@ -6,13 +6,13 @@ edition = "2021"
 [dependencies]
 itertools = "^0.10"
 log = { version = "0.4.17", features = ["release_max_level_debug"] }
+ast = { path = "../ast" }
 number = { path = "../number" }
 parser_util = { path = "../parser_util" }
 pil_analyzer = { path = "../pil_analyzer" }
 rayon = "1.7.0"
 bit-vec = "0.6.3"
 num-traits = "0.2.15"
-ast = { version = "0.1.0", path = "../ast" }
 lazy_static = "1.4.0"
 
 [dev-dependencies]

halo2/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 number = { path = "../number" }
 pil_analyzer = { path = "../pil_analyzer" }
+ast = { path = "../ast" }
 polyexen = { git = "https://github.com/Dhole/polyexen", branch = "feature/shuffles" }
 halo2_proofs = "0.2"
 halo2_curves = { git = "https://github.com/privacy-scaling-explorations/halo2curves", tag = "0.3.2", package = "halo2curves" }
@@ -16,7 +17,6 @@ itertools = "^0.10"
 num-bigint = "^0.4"
 log = "0.4.17"
 rand = "0.8.5"
-ast = { version = "0.1.0", path = "../ast" }
 
 [dev-dependencies]
 importer = { path = "../importer" }

halo2/src/mock_prover.rs

@@ -35,7 +35,7 @@ pub fn mock_prove<T: FieldElement>(
 mod test {
     use std::{fs, path::PathBuf};
 
-    use analysis::analyze;
+    use analysis::convert_asm_to_pil;
     use number::Bn254Field;
     use parser::parse_asm;
     use test_log::test;
@@ -54,7 +54,7 @@ mod test {
         let contents = fs::read_to_string(&location).unwrap();
         let parsed = parse_asm::<Bn254Field>(Some(&location), &contents).unwrap();
         let resolved = importer::resolve(Some(PathBuf::from(location)), parsed).unwrap();
-        let analysed = analyze(resolved).unwrap();
+        let analysed = convert_asm_to_pil(resolved).unwrap();
         let graph = airgen::compile(analysed);
         let pil = linker::link(graph).unwrap();
 

importer/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ast = { version = "0.1.0", path = "../ast" }
-number = { version = "0.1.0", path = "../number" }
-pretty_assertions = "1.4.0"
+ast = { path = "../ast" }
+number = { path = "../number" }
 parser = { path = "../parser" }
+pretty_assertions = "1.4.0"

linker/Cargo.toml

@@ -6,10 +6,10 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ast = { version = "0.1.0", path = "../ast" }
-number = { version = "0.1.0", path = "../number" }
-pretty_assertions = "1.3.0"
+ast = { path = "../ast" }
+number = { path = "../number" }
 analysis = { path = "../analysis" }
+pretty_assertions = "1.3.0"
 
 [dev-dependencies]
 parser = { path = "../parser" }