Merge remote-tracking branch 'origin/master' into express-lane-timeboost

Roll back express_lane_service_test.go use of CalldataUnits to mean an
errored tx and go back to using nil. Check for nil before dereferencing
the tx when generating an error message now.

.dockerignore

@@ -36,6 +36,7 @@ arbitrator/tools/wasmer/target/
 arbitrator/tools/wasm-tools/
 arbitrator/tools/pricers/
 arbitrator/tools/module_roots/
+arbitrator/tools/stylus_benchmark
 arbitrator/langs/rust/target/
 arbitrator/langs/bf/target/
 

.github/workflows/arbitrator-ci.yml

@@ -171,6 +171,9 @@ jobs:
       - name: Rustfmt - langs/rust
         run: cargo fmt --all --manifest-path arbitrator/langs/rust/Cargo.toml -- --check
 
+      - name: Rustfmt - tools/stylus_benchmark
+        run: cargo fmt --all --manifest-path arbitrator/tools/stylus_benchmark/Cargo.toml -- --check
+
       - name: Make proofs from test cases
         run: make -j test-gen-proofs
 

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       redis:
         image: redis
         ports:
-            - 6379:6379
+          - 6379:6379
 
     strategy:
       fail-fast: false
@@ -192,7 +192,7 @@ jobs:
 
       - name: run challenge tests
         if: matrix.test-mode == 'challenge'
-        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover
+        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --timeout 60m --cover
 
       - name: run stylus tests
         if: matrix.test-mode == 'stylus'

.github/workflows/submodule-pin-check.yml

@@ -25,9 +25,9 @@ jobs:
         run: |
           status_state="pending"
           declare -Ar exceptions=(
-            [contracts]=origin/develop
+            [contracts]=origin/pre-bold
             [nitro-testnode]=origin/master
-            
+
             #TODO Rachel to check these are the intended branches.
             [arbitrator/langs/c]=origin/vm-storage-cache
             [arbitrator/tools/wasmer]=origin/adopt-v4.2.8
@@ -38,7 +38,7 @@ jobs:
             if [[ -v exceptions[$mod] ]]; then
               branch=${exceptions[$mod]}
             fi
-            
+
             if ! git -C $mod merge-base --is-ancestor HEAD $branch; then
               echo $mod diverges from $branch
               divergent=1

.gitmodules

@@ -23,6 +23,9 @@
 [submodule "nitro-testnode"]
 	path = nitro-testnode
 	url = https://github.com/OffchainLabs/nitro-testnode.git
+[submodule "bold"]
+	path = bold
+	url = https://github.com/OffchainLabs/bold.git
 [submodule "arbitrator/langs/rust"]
 	path = arbitrator/langs/rust
 	url = https://github.com/OffchainLabs/stylus-sdk-rs.git

Dockerfile

@@ -234,6 +234,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
 COPY go.mod go.sum ./
 COPY go-ethereum/go.mod go-ethereum/go.sum go-ethereum/
 COPY fastcache/go.mod fastcache/go.sum fastcache/
+COPY bold/go.mod bold/go.sum bold/
 RUN go mod download
 COPY . ./
 COPY --from=contracts-builder workspace/contracts/build/ contracts/build/

LICENSE.md

@@ -19,7 +19,7 @@ Additional Use Grant: You may use the Licensed Work in a production environment
                       validating the correctness of the posted chain state, or to deploy
                       and operate (x) a blockchain that settles to a Covered Arbitrum Chain
                       or (y) a blockchain in accordance with, and subject to, the [Arbitrum
-                      Expansion Program Term of Use](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf). For purposes of this
+                      Expansion Program Term of Use](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf). For purposes of this
                       Additional Use Grant, the "Covered Arbitrum Chains" are 
                       (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170), 
                       Arbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro

README.md

@@ -47,7 +47,7 @@ Nitro is currently licensed under a [Business Source License](./LICENSE.md), sim
 
 The Additional Use Grant also permits the deployment of the Nitro software, in a permissionless fashion and without cost, as a new blockchain provided that the chain settles to either Arbitrum One or Arbitrum Nova.
 
-For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
+For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
 
 ## Contact
 

arbitrator/Cargo.toml

@@ -12,6 +12,7 @@ members = [
 exclude = [
         "stylus/tests/",
         "tools/wasmer/",
+        "tools/stylus_benchmark",
 ]
 resolver = "2"
 

arbitrator/arbutil/src/benchmark.rs

@@ -0,0 +1,14 @@
+// Copyright 2024, Offchain Labs, Inc.
+// For license information, see https://github.com/nitro/blob/master/LICENSE
+
+use crate::evm::api::Ink;
+use std::time::{Duration, Instant};
+
+// Benchmark is used to track the performance of blocks of code in stylus
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Benchmark {
+    pub timer: Option<Instant>,
+    pub elapsed_total: Duration,
+    pub ink_start: Option<Ink>,
+    pub ink_total: Ink,
+}

arbitrator/arbutil/src/lib.rs

@@ -1,6 +1,7 @@
 // Copyright 2022-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
 
+pub mod benchmark;
 /// cbindgen:ignore
 pub mod color;
 pub mod crypto;

arbitrator/jit/src/lib.rs

@@ -0,0 +1,51 @@
+// Copyright 2021-2024, Offchain Labs, Inc.
+// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
+
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+mod arbcompress;
+mod caller_env;
+pub mod machine;
+mod prepare;
+pub mod program;
+mod socket;
+pub mod stylus_backend;
+mod test;
+mod wasip1_stub;
+mod wavmio;
+
+#[derive(StructOpt)]
+#[structopt(name = "jit-prover")]
+pub struct Opts {
+    #[structopt(short, long)]
+    binary: PathBuf,
+    #[structopt(long, default_value = "0")]
+    inbox_position: u64,
+    #[structopt(long, default_value = "0")]
+    delayed_inbox_position: u64,
+    #[structopt(long, default_value = "0")]
+    position_within_message: u64,
+    #[structopt(long)]
+    last_block_hash: Option<String>,
+    #[structopt(long)]
+    last_send_root: Option<String>,
+    #[structopt(long)]
+    inbox: Vec<PathBuf>,
+    #[structopt(long)]
+    delayed_inbox: Vec<PathBuf>,
+    #[structopt(long)]
+    preimages: Option<PathBuf>,
+    #[structopt(long)]
+    cranelift: bool,
+    #[structopt(long)]
+    forks: bool,
+    #[structopt(long)]
+    pub debug: bool,
+    #[structopt(long)]
+    pub require_success: bool,
+    // JSON inputs supercede any of the command-line inputs which could
+    // be specified in the JSON file.
+    #[structopt(long)]
+    json_inputs: Option<PathBuf>,
+}

arbitrator/jit/src/main.rs

@@ -1,58 +1,13 @@
 // Copyright 2022-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/nitro/blob/master/LICENSE
 
-use crate::machine::{Escape, WasmEnv};
 use arbutil::{color, Color};
 use eyre::Result;
-use std::path::PathBuf;
+use jit::machine;
+use jit::machine::{Escape, WasmEnv};
+use jit::Opts;
 use structopt::StructOpt;
 
-mod arbcompress;
-mod caller_env;
-mod machine;
-mod prepare;
-mod program;
-mod socket;
-mod stylus_backend;
-mod test;
-mod wasip1_stub;
-mod wavmio;
-
-#[derive(StructOpt)]
-#[structopt(name = "jit-prover")]
-pub struct Opts {
-    #[structopt(short, long)]
-    binary: PathBuf,
-    #[structopt(long, default_value = "0")]
-    inbox_position: u64,
-    #[structopt(long, default_value = "0")]
-    delayed_inbox_position: u64,
-    #[structopt(long, default_value = "0")]
-    position_within_message: u64,
-    #[structopt(long)]
-    last_block_hash: Option<String>,
-    #[structopt(long)]
-    last_send_root: Option<String>,
-    #[structopt(long)]
-    inbox: Vec<PathBuf>,
-    #[structopt(long)]
-    delayed_inbox: Vec<PathBuf>,
-    #[structopt(long)]
-    preimages: Option<PathBuf>,
-    #[structopt(long)]
-    cranelift: bool,
-    #[structopt(long)]
-    forks: bool,
-    #[structopt(long)]
-    debug: bool,
-    #[structopt(long)]
-    require_success: bool,
-    // JSON inputs supercede any of the command-line inputs which could
-    // be specified in the JSON file.
-    #[structopt(long)]
-    json_inputs: Option<PathBuf>,
-}
-
 fn main() -> Result<()> {
     let opts = Opts::from_args();
     let env = match WasmEnv::cli(&opts) {

arbitrator/jit/src/prepare.rs

@@ -1,7 +1,7 @@
 // Copyright 2022-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/nitro/blob/master/LICENSE
 
-use crate::WasmEnv;
+use crate::machine::WasmEnv;
 use arbutil::{Bytes32, PreimageType};
 use eyre::Ok;
 use prover::parse_input::FileData;

arbitrator/jit/src/program.rs

@@ -4,8 +4,8 @@
 #![allow(clippy::too_many_arguments)]
 
 use crate::caller_env::JitEnv;
-use crate::machine::{Escape, MaybeEscape, WasmEnvMut};
-use crate::stylus_backend::exec_wasm;
+use crate::machine::{Escape, MaybeEscape, WasmEnv, WasmEnvMut};
+use crate::stylus_backend::{exec_wasm, MessageFromCothread};
 use arbutil::evm::api::Gas;
 use arbutil::Bytes32;
 use arbutil::{evm::EvmData, format::DebugBytes, heapify};
@@ -16,6 +16,7 @@ use prover::{
     machine::Module,
     programs::{config::PricingParams, prelude::*},
 };
+use std::sync::Arc;
 
 const DEFAULT_STYLUS_ARBOS_VERSION: u64 = 31;
 
@@ -130,10 +131,6 @@ pub fn new_program(
     let evm_data: EvmData = unsafe { *Box::from_raw(evm_data_handler as *mut EvmData) };
     let config: JitConfig = unsafe { *Box::from_raw(stylus_config_handler as *mut JitConfig) };
 
-    // buy ink
-    let pricing = config.stylus.pricing;
-    let ink = pricing.gas_to_ink(Gas(gas));
-
     let Some(module) = exec.module_asms.get(&compiled_hash).cloned() else {
         return Err(Escape::Failure(format!(
             "module hash {:?} not found in {:?}",
@@ -142,6 +139,21 @@ pub fn new_program(
         )));
     };
 
+    exec_program(exec, module, calldata, config, evm_data, gas)
+}
+
+pub fn exec_program(
+    exec: &mut WasmEnv,
+    module: Arc<[u8]>,
+    calldata: Vec<u8>,
+    config: JitConfig,
+    evm_data: EvmData,
+    gas: u64,
+) -> Result<u32, Escape> {
+    // buy ink
+    let pricing = config.stylus.pricing;
+    let ink = pricing.gas_to_ink(Gas(gas));
+
     let cothread = exec_wasm(
         module,
         calldata,
@@ -162,7 +174,10 @@ pub fn new_program(
 /// returns request_id for the first request from the program
 pub fn start_program(mut env: WasmEnvMut, module: u32) -> Result<u32, Escape> {
     let (_, exec) = env.jit_env();
+    start_program_with_wasm_env(exec, module)
+}
 
+pub fn start_program_with_wasm_env(exec: &mut WasmEnv, module: u32) -> Result<u32, Escape> {
     if exec.threads.len() as u32 != module || module == 0 {
         return Escape::hostio(format!(
             "got request for thread {module} but len is {}",
@@ -179,26 +194,27 @@ pub fn start_program(mut env: WasmEnvMut, module: u32) -> Result<u32, Escape> {
 /// request_id MUST be last request id returned from start_program or send_response
 pub fn get_request(mut env: WasmEnvMut, id: u32, len_ptr: GuestPtr) -> Result<u32, Escape> {
     let (mut mem, exec) = env.jit_env();
+    let msg = get_last_msg(exec, id)?;
+    mem.write_u32(len_ptr, msg.req_data.len() as u32);
+    Ok(msg.req_type)
+}
+
+pub fn get_last_msg(exec: &mut WasmEnv, id: u32) -> Result<MessageFromCothread, Escape> {
     let thread = exec.threads.last_mut().unwrap();
     let msg = thread.last_message()?;
     if msg.1 != id {
         return Escape::hostio("get_request id doesn't match");
     };
-    mem.write_u32(len_ptr, msg.0.req_data.len() as u32);
-    Ok(msg.0.req_type)
+    Ok(msg.0)
 }
 
 // gets data associated with last request.
 // request_id MUST be last request receieved
 // data_ptr MUST point to a buffer of at least the length returned by get_request
 pub fn get_request_data(mut env: WasmEnvMut, id: u32, data_ptr: GuestPtr) -> MaybeEscape {
     let (mut mem, exec) = env.jit_env();
-    let thread = exec.threads.last_mut().unwrap();
-    let msg = thread.last_message()?;
-    if msg.1 != id {
-        return Escape::hostio("get_request id doesn't match");
-    };
-    mem.write_slice(data_ptr, &msg.0.req_data);
+    let msg = get_last_msg(exec, id)?;
+    mem.write_slice(data_ptr, &msg.req_data);
     Ok(())
 }
 
@@ -217,11 +233,21 @@ pub fn set_response(
     let result = mem.read_slice(result_ptr, result_len as usize);
     let raw_data = mem.read_slice(raw_data_ptr, raw_data_len as usize);
 
+    set_response_with_wasm_env(exec, id, gas, result, raw_data)
+}
+
+pub fn set_response_with_wasm_env(
+    exec: &mut WasmEnv,
+    id: u32,
+    gas: u64,
+    result: Vec<u8>,
+    raw_data: Vec<u8>,
+) -> MaybeEscape {
     let thread = exec.threads.last_mut().unwrap();
     thread.set_response(id, result, raw_data, Gas(gas))
 }
 
-/// sends previos response
+/// sends previous response
 /// MUST be called right after set_response to the same id
 /// returns request_id for the next request
 pub fn send_response(mut env: WasmEnvMut, req_id: u32) -> Result<u32, Escape> {
@@ -239,16 +265,19 @@ pub fn send_response(mut env: WasmEnvMut, req_id: u32) -> Result<u32, Escape> {
 /// removes the last created program
 pub fn pop(mut env: WasmEnvMut) -> MaybeEscape {
     let (_, exec) = env.jit_env();
+    pop_with_wasm_env(exec)
+}
 
+pub fn pop_with_wasm_env(exec: &mut WasmEnv) -> MaybeEscape {
     match exec.threads.pop() {
         None => Err(Escape::Child(eyre!("no child"))),
         Some(mut thread) => thread.wait_done(),
     }
 }
 
 pub struct JitConfig {
-    stylus: StylusConfig,
-    compile: CompileConfig,
+    pub stylus: StylusConfig,
+    pub compile: CompileConfig,
 }
 
 /// Creates a `StylusConfig` from its component parts.