diff --git a/Cargo.lock b/Cargo.lock index e4cfe189f..cc58d04da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -949,6 +949,26 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.77", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -1094,6 +1114,17 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "c-kzg" version = "1.0.3" @@ -1126,6 +1157,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -1159,6 +1199,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.16" @@ -2348,6 +2399,7 @@ dependencies = [ "os_pipe", "reqwest", "revm", + "rocksdb", "serde", "serde_json", "tokio", @@ -2430,18 +2482,59 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets", +] + [[package]] name = "libm" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "librocksdb-sys" +version = "0.16.0+8.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked_list_allocator" version = "0.10.5" @@ -2530,6 +2623,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -2600,6 +2699,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3546,6 +3655,16 @@ dependencies = [ "rustc-hex", ] +[[package]] +name = "rocksdb" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "ruint" version = "1.12.3" @@ -3582,6 +3701,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hex" version = "2.1.0" diff --git a/Cargo.toml b/Cargo.toml index 888d21d76..340fa85d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,9 @@ revm = { version = "14.0", default-features = false } superchain = "0.3" superchain-primitives = { version = "0.3", default-features = false } +# K/V database +rocksdb = { version = "0.22", default-features = false, features = ["snappy"] } + # Alloy alloy-rlp = { version = "0.3", default-features = false } alloy-trie = { version = "0.5", default-features = false } diff --git a/bin/client/testdata/granite-op-sepolia-16491249-witness.tar.zst b/bin/client/testdata/granite-op-sepolia-16491249-witness.tar.zst index 031ee0831..df13e3ec5 100644 Binary files a/bin/client/testdata/granite-op-sepolia-16491249-witness.tar.zst and b/bin/client/testdata/granite-op-sepolia-16491249-witness.tar.zst differ diff --git a/bin/host/Cargo.toml b/bin/host/Cargo.toml index c80c43882..fe4bf6a4e 100644 --- a/bin/host/Cargo.toml +++ b/bin/host/Cargo.toml @@ -41,3 +41,4 @@ serde_json.workspace = true tracing-subscriber.workspace = true command-fds.workspace = true os_pipe.workspace = true +rocksdb.workspace = true diff --git a/bin/host/src/fetcher/mod.rs b/bin/host/src/fetcher/mod.rs index 81c554635..203170722 100644 --- a/bin/host/src/fetcher/mod.rs +++ b/bin/host/src/fetcher/mod.rs @@ -125,7 +125,7 @@ where kv_lock.set( PreimageKey::new(*hash, PreimageKeyType::Keccak256).into(), raw_header.into(), - ); + )?; } HintType::L1Transactions => { // Validate the hint data length. @@ -207,7 +207,7 @@ where kv_write_lock.set( PreimageKey::new(*hash, PreimageKeyType::Sha256).into(), sidecar.kzg_commitment.to_vec(), - ); + )?; // Write all the field elements to the key-value store. There should be 4096. // The preimage oracle key for each field element is the keccak256 hash of @@ -221,11 +221,11 @@ where kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), blob_key.into(), - ); + )?; kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::Blob).into(), sidecar.blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), - ); + )?; } // Write the KZG Proof as the 4096th element. @@ -235,11 +235,11 @@ where kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), blob_key.into(), - ); + )?; kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::Blob).into(), sidecar.kzg_proof.to_vec(), - ); + )?; } HintType::L1Precompile => { // Validate the hint data length. @@ -270,9 +270,11 @@ where kv_lock.set( PreimageKey::new(*input_hash, PreimageKeyType::Keccak256).into(), hint_data.into(), - ); - kv_lock - .set(PreimageKey::new(*input_hash, PreimageKeyType::Precompile).into(), result); + )?; + kv_lock.set( + PreimageKey::new(*input_hash, PreimageKeyType::Precompile).into(), + result, + )?; } HintType::L2BlockHeader => { // Validate the hint data length. @@ -297,7 +299,7 @@ where kv_lock.set( PreimageKey::new(*hash, PreimageKeyType::Keccak256).into(), raw_header.into(), - ); + )?; } HintType::L2Transactions => { // Validate the hint data length. @@ -371,7 +373,7 @@ where let mut kv_write_lock = self.kv_store.write().await; kv_write_lock - .set(PreimageKey::new(*hash, PreimageKeyType::Keccak256).into(), code.into()); + .set(PreimageKey::new(*hash, PreimageKeyType::Keccak256).into(), code.into())?; } HintType::StartingL2Output => { const OUTPUT_ROOT_VERSION: u8 = 0; @@ -415,7 +417,7 @@ where kv_write_lock.set( PreimageKey::new(*output_root, PreimageKeyType::Keccak256).into(), raw_output.into(), - ); + )?; } HintType::L2StateNode => { if hint_data.len() != 32 { @@ -439,7 +441,7 @@ where kv_write_lock.set( PreimageKey::new(*hash, PreimageKeyType::Keccak256).into(), preimage.into(), - ); + )?; } HintType::L2AccountProof => { if hint_data.len() != 8 + 20 { @@ -463,11 +465,12 @@ where let mut kv_write_lock = self.kv_store.write().await; // Write the account proof nodes to the key-value store. - proof_response.account_proof.into_iter().for_each(|node| { + proof_response.account_proof.into_iter().try_for_each(|node| { let node_hash = keccak256(node.as_ref()); let key = PreimageKey::new(*node_hash, PreimageKeyType::Keccak256); - kv_write_lock.set(key.into(), node.into()); - }); + kv_write_lock.set(key.into(), node.into())?; + Ok(()) + })?; } HintType::L2AccountStorageProof => { if hint_data.len() != 8 + 20 + 32 { @@ -492,19 +495,21 @@ where let mut kv_write_lock = self.kv_store.write().await; // Write the account proof nodes to the key-value store. - proof_response.account_proof.into_iter().for_each(|node| { + proof_response.account_proof.into_iter().try_for_each(|node| { let node_hash = keccak256(node.as_ref()); let key = PreimageKey::new(*node_hash, PreimageKeyType::Keccak256); - kv_write_lock.set(key.into(), node.into()); - }); + kv_write_lock.set(key.into(), node.into())?; + Ok(()) + })?; // Write the storage proof nodes to the key-value store. let storage_proof = proof_response.storage_proof.remove(0); - storage_proof.proof.into_iter().for_each(|node| { + storage_proof.proof.into_iter().try_for_each(|node| { let node_hash = keccak256(node.as_ref()); let key = PreimageKey::new(*node_hash, PreimageKeyType::Keccak256); - kv_write_lock.set(key.into(), node.into()); - }); + kv_write_lock.set(key.into(), node.into())?; + Ok(()) + })?; } } @@ -544,8 +549,7 @@ where // `ProofRetainer` in the event that there are no leaves inserted. if nodes.is_empty() { let empty_key = PreimageKey::new(*EMPTY_ROOT_HASH, PreimageKeyType::Keccak256); - kv_write_lock.set(empty_key.into(), [EMPTY_STRING_CODE].into()); - return Ok(()) + return kv_write_lock.set(empty_key.into(), [EMPTY_STRING_CODE].into()) } let mut hb = kona_mpt::ordered_trie_with_encoder(nodes, |node, buf| { @@ -558,7 +562,7 @@ where let value_hash = keccak256(value.as_ref()); let key = PreimageKey::new(*value_hash, PreimageKeyType::Keccak256); - kv_write_lock.set(key.into(), value.into()); + kv_write_lock.set(key.into(), value.into())?; } Ok(()) diff --git a/bin/host/src/kv/disk.rs b/bin/host/src/kv/disk.rs index 9357bdb5b..ee76e0aba 100644 --- a/bin/host/src/kv/disk.rs +++ b/bin/host/src/kv/disk.rs @@ -1,35 +1,48 @@ -//! Contains a concrete implementation of the [KeyValueStore] trait that stores data on disk. -//! -//! Data is stored in a directory, with a separate file for each key. The key is the filename, and -//! the value is the raw contents of the file. +//! Contains a concrete implementation of the [KeyValueStore] trait that stores data on disk +//! using [rocksdb]. use super::KeyValueStore; -use alloy_primitives::hex; -use std::{fs, path::PathBuf}; +use anyhow::{anyhow, Result}; +use rocksdb::{Options, DB}; +use std::path::PathBuf; /// A simple, synchronous key-value store that stores data on disk. -#[derive(Default, Clone, Debug, Eq, PartialEq)] +#[derive(Debug)] pub struct DiskKeyValueStore { data_directory: PathBuf, + db: DB, } impl DiskKeyValueStore { /// Create a new [DiskKeyValueStore] with the given data directory. pub fn new(data_directory: PathBuf) -> Self { - Self { data_directory } + let db = DB::open(&Self::get_db_options(), data_directory.as_path()) + .unwrap_or_else(|e| panic!("Failed to open database at {data_directory:?}: {e}")); + + Self { data_directory, db } + } + + /// Gets the [Options] for the underlying RocksDB instance. + fn get_db_options() -> Options { + let mut options = Options::default(); + options.set_compression_type(rocksdb::DBCompressionType::Snappy); + options.create_if_missing(true); + options } } impl KeyValueStore for DiskKeyValueStore { fn get(&self, key: alloy_primitives::B256) -> Option> { - let path = self.data_directory.join(format!("{}.bin", hex::encode(key))); - fs::create_dir_all(&self.data_directory).ok()?; - fs::read(path).ok() + self.db.get(*key).ok()? } - fn set(&mut self, key: alloy_primitives::B256, value: Vec) { - let path = self.data_directory.join(format!("{}.bin", hex::encode(key))); - fs::create_dir_all(&self.data_directory).expect("Failed to create directory"); - fs::write(path, value.as_slice()).expect("Failed to write data to disk"); + fn set(&mut self, key: alloy_primitives::B256, value: Vec) -> Result<()> { + self.db.put(*key, value).map_err(|e| anyhow!("Failed to set key-value pair: {}", e)) + } +} + +impl Drop for DiskKeyValueStore { + fn drop(&mut self) { + let _ = DB::destroy(&Self::get_db_options(), self.data_directory.as_path()); } } diff --git a/bin/host/src/kv/local.rs b/bin/host/src/kv/local.rs index 12bd92c64..7df49336b 100644 --- a/bin/host/src/kv/local.rs +++ b/bin/host/src/kv/local.rs @@ -3,6 +3,7 @@ use super::KeyValueStore; use crate::cli::HostCli; use alloy_primitives::B256; +use anyhow::Result; use kona_client::boot::{ L1_HEAD_KEY, L2_CHAIN_ID_KEY, L2_CLAIM_BLOCK_NUMBER_KEY, L2_CLAIM_KEY, L2_OUTPUT_ROOT_KEY, L2_ROLLUP_CONFIG_KEY, @@ -40,7 +41,7 @@ impl KeyValueStore for LocalKeyValueStore { } } - fn set(&mut self, _: B256, _: Vec) { + fn set(&mut self, _: B256, _: Vec) -> Result<()> { unreachable!("LocalKeyValueStore is read-only") } } diff --git a/bin/host/src/kv/mem.rs b/bin/host/src/kv/mem.rs index 6459ad715..59542be11 100644 --- a/bin/host/src/kv/mem.rs +++ b/bin/host/src/kv/mem.rs @@ -2,6 +2,7 @@ use super::KeyValueStore; use alloy_primitives::B256; +use anyhow::Result; use std::collections::HashMap; /// A simple, synchronous key-value store that stores data in memory. This is useful for testing and @@ -23,7 +24,8 @@ impl KeyValueStore for MemoryKeyValueStore { self.store.get(&key).cloned() } - fn set(&mut self, key: B256, value: Vec) { + fn set(&mut self, key: B256, value: Vec) -> Result<()> { self.store.insert(key, value); + Ok(()) } } diff --git a/bin/host/src/kv/mod.rs b/bin/host/src/kv/mod.rs index 8b996d517..78ef8c4fa 100644 --- a/bin/host/src/kv/mod.rs +++ b/bin/host/src/kv/mod.rs @@ -1,6 +1,7 @@ //! This module contains the [KeyValueStore] trait and concrete implementations of it. use alloy_primitives::B256; +use anyhow::Result; use std::sync::Arc; use tokio::sync::RwLock; @@ -25,5 +26,5 @@ pub trait KeyValueStore { fn get(&self, key: B256) -> Option>; /// Set the value associated with the given key. - fn set(&mut self, key: B256, value: Vec); + fn set(&mut self, key: B256, value: Vec) -> Result<()>; } diff --git a/bin/host/src/kv/split.rs b/bin/host/src/kv/split.rs index 2cd2e81f5..2ec1f215b 100644 --- a/bin/host/src/kv/split.rs +++ b/bin/host/src/kv/split.rs @@ -1,11 +1,11 @@ //! Contains a concrete implementation of the [KeyValueStore] trait that splits between two separate //! [KeyValueStore]s depending on [PreimageKeyType]. +use super::KeyValueStore; use alloy_primitives::B256; +use anyhow::Result; use kona_preimage::PreimageKeyType; -use super::KeyValueStore; - /// A split implementation of the [KeyValueStore] trait that splits between two separate /// [KeyValueStore]s. #[derive(Clone, Debug)] @@ -41,7 +41,7 @@ where } } - fn set(&mut self, key: B256, value: Vec) { - self.remote_store.set(key, value); + fn set(&mut self, key: B256, value: Vec) -> Result<()> { + self.remote_store.set(key, value) } }