From 2adfadbd8dc8d7bd229ab0b9bd31b7fdba4a4891 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 21:42:06 +0000 Subject: [PATCH 01/29] can read eigenda v1 cert --- Cargo.lock | 2 + bin/client/justfile | 4 +- bin/host/Cargo.toml | 1 + bin/host/src/eigenda_fetcher/mod.rs | 9 ++++ crates/eigenda/Cargo.toml | 1 + crates/eigenda/src/certificate.rs | 76 +++++++++++++++++++++++++++++ crates/eigenda/src/lib.rs | 4 ++ 7 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 crates/eigenda/src/certificate.rs diff --git a/Cargo.lock b/Cargo.lock index 171dfe4..80614e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1723,6 +1723,7 @@ name = "hokulea-eigenda" version = "0.1.0" dependencies = [ "alloy-primitives", + "alloy-rlp", "async-trait", "kona-derive", "op-alloy-protocol", @@ -1740,6 +1741,7 @@ dependencies = [ "async-trait", "clap", "hokulea-client", + "hokulea-eigenda", "hokulea-proof", "kona-host", "kona-preimage", diff --git a/bin/client/justfile b/bin/client/justfile index 9ac832d..de993ed 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="../../../optimism/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="/home/ubuntu/op-main-repo/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) @@ -120,7 +120,7 @@ run-client-native block_number l1_rpc l1_beacon_rpc l2_rpc rollup_node_rpc rollu cd $(git rev-parse --show-toplevel) echo "Running host program with native client program..." - cargo r --bin hokulea-host --release -- \ + cargo r --bin hokulea-host -- \ --l1-head $L1_HEAD \ --agreed-l2-head-hash $AGREED_L2_HEAD_HASH \ --claimed-l2-output-root $CLAIMED_L2_OUTPUT_ROOT \ diff --git a/bin/host/Cargo.toml b/bin/host/Cargo.toml index e286988..69820d0 100644 --- a/bin/host/Cargo.toml +++ b/bin/host/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" # Workspace hokulea-proof.workspace = true hokulea-client.workspace = true +hokulea-eigenda.workspace = true # Kona kona-preimage = { workspace = true, features = ["std"] } diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 643785d..976e78c 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -12,6 +12,8 @@ use kona_preimage::{PreimageKey, PreimageKeyType}; use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; +use hokulea_eigenda::BlobInfo; +use alloy_rlp::{Decodable}; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -136,6 +138,13 @@ where trace!(target: "fetcher_with_eigenda_support", "Fetching hint: {hint_type} {hint_data}"); if hint_type == ExtendedHintType::EigenDACommitment { + + let item_slice= hint_data.as_ref(); + + // the fourth because 0x01010000 in the beginnin is metadata + let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; + info!("cert_blob_info {:?}", cert_blob_info); + let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. diff --git a/crates/eigenda/Cargo.toml b/crates/eigenda/Cargo.toml index 258342d..baa1f5f 100644 --- a/crates/eigenda/Cargo.toml +++ b/crates/eigenda/Cargo.toml @@ -9,6 +9,7 @@ kona-derive.workspace = true # Op Alloy op-alloy-protocol.workspace = true alloy-primitives.workspace = true +alloy-rlp.workspace = true tracing.workspace = true async-trait.workspace = true diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs new file mode 100644 index 0000000..1ecdcaa --- /dev/null +++ b/crates/eigenda/src/certificate.rs @@ -0,0 +1,76 @@ +// data struct copied from https://github.com/Layr-Labs/eigenda-client-rs/blob/3ac1f62ae3d99aedf3de7a2fe827fab17db7b874/src/blob_info.rs +use core::fmt; + +use alloy_primitives::Bytes; +use alloy_rlp::{RlpDecodable, RlpEncodable}; + +use alloc::vec::Vec; + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct G1Commitment { + pub x: [u8; 32], + pub y: [u8; 32], +} + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobQuorumParam { + pub quorum_number: u32, + pub adversary_threshold_percentage: u32, + pub confirmation_threshold_percentage: u32, + pub chunk_length: u32, +} + +impl BlobQuorumParam { + pub fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend(&self.quorum_number.to_be_bytes()); + bytes.extend(&self.adversary_threshold_percentage.to_be_bytes()); + bytes.extend(&self.confirmation_threshold_percentage.to_be_bytes()); + bytes.extend(&self.chunk_length.to_be_bytes()); + + bytes + } +} + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobHeader { + pub commitment: G1Commitment, + pub data_length: u32, + pub blob_quorum_params: Vec, +} + + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BatchHeader { + pub batch_root: Bytes, + pub quorum_numbers: Bytes, + pub quorum_signed_percentages: Bytes, + pub reference_block_number: u32, +} + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BatchMetadata { + pub batch_header: BatchHeader, + pub signatory_record_hash: Bytes, + pub fee: Bytes, + pub confirmation_block_number: u32, + pub batch_header_hash: Bytes, +} + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobVerificationProof { + pub batch_id: u32, + pub blob_index: u32, + pub batch_medatada: BatchMetadata, + pub inclusion_proof: Bytes, + pub quorum_indexes: Bytes, +} + +#[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] +pub struct BlobInfo { + pub blob_header: BlobHeader, + pub blob_verification_proof: BlobVerificationProof, +} diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index b62f3f5..e457bbf 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -26,3 +26,7 @@ pub use eigenda_blobs::EigenDABlobSource; mod eigenda_data; pub use eigenda_data::EigenDABlobData; + +mod certificate; +pub use certificate::BlobInfo; + From 082ab7fd02d0bbcc6a20f62fc8613d0f0ea8c1a6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 21:45:35 +0000 Subject: [PATCH 02/29] small fix --- README.md | 18 +++++++++++++++++- bin/client/justfile | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 822851f..d340b8f 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,20 @@ cd bin/client just run-client-native-against-devnet ``` -![](./hokulea.jpeg) +To use eigenda proxy within optimism devnet, modify ops-bedrock/docker-compose.yaml: +``` +da-server: + image: ghcr.io/layr-labs/eigenda-proxy:v1.6.1 + environment: + EIGENDA_PROXY_ADDR: 0.0.0.0 + EIGENDA_PROXY_PORT: 3100 + EIGENDA_PROXY_METRICS_ENABLED: true + EIGENDA_PROXY_METRICS_PORT: 7300 + EIGENDA_PROXY_MEMSTORE_ENABLED: true + EIGENDA_PROXY_MEMSTORE_EXPIRATION: 45m + EIGENDA_PROXY_MEMSTORE_PUT_LATENCY: 0s + EIGENDA_PROXY_MEMSTORE_GET_LATENCY: 0s + EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED: true +``` + +![](./hokulea.jpeg) \ No newline at end of file diff --git a/bin/client/justfile b/bin/client/justfile index de993ed..fb05bd2 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="/home/ubuntu/op-main-repo/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="../../../optimism/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) From 6f9aef790398b9e0ded1bc8c702f65b9cdb16075 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:03:08 +0000 Subject: [PATCH 03/29] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 4 ++-- crates/eigenda/src/certificate.rs | 22 +++++----------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 976e78c..c3a735a 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; use hokulea_eigenda::BlobInfo; -use alloy_rlp::{Decodable}; +use alloy_rlp::Decodable; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -139,7 +139,7 @@ where if hint_type == ExtendedHintType::EigenDACommitment { - let item_slice= hint_data.as_ref(); + let item_slice = hint_data.as_ref(); // the fourth because 0x01010000 in the beginnin is metadata let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs index 1ecdcaa..eedfd65 100644 --- a/crates/eigenda/src/certificate.rs +++ b/crates/eigenda/src/certificate.rs @@ -1,6 +1,3 @@ -// data struct copied from https://github.com/Layr-Labs/eigenda-client-rs/blob/3ac1f62ae3d99aedf3de7a2fe827fab17db7b874/src/blob_info.rs -use core::fmt; - use alloy_primitives::Bytes; use alloy_rlp::{RlpDecodable, RlpEncodable}; @@ -13,7 +10,6 @@ pub struct G1Commitment { pub y: [u8; 32], } - #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobQuorumParam { pub quorum_number: u32, @@ -22,19 +18,7 @@ pub struct BlobQuorumParam { pub chunk_length: u32, } -impl BlobQuorumParam { - pub fn to_bytes(&self) -> Vec { - let mut bytes = Vec::new(); - bytes.extend(&self.quorum_number.to_be_bytes()); - bytes.extend(&self.adversary_threshold_percentage.to_be_bytes()); - bytes.extend(&self.confirmation_threshold_percentage.to_be_bytes()); - bytes.extend(&self.chunk_length.to_be_bytes()); - - bytes - } -} - - +/// eigenda v1 blob header #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobHeader { pub commitment: G1Commitment, @@ -60,6 +44,7 @@ pub struct BatchMetadata { pub batch_header_hash: Bytes, } +/// eigenda v1 blob verification proof #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobVerificationProof { pub batch_id: u32, @@ -69,8 +54,11 @@ pub struct BlobVerificationProof { pub quorum_indexes: Bytes, } +/// eigenda v1 certificate #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BlobInfo { + /// v1 blob header pub blob_header: BlobHeader, + /// v1 blob verification proof with merkle tree pub blob_verification_proof: BlobVerificationProof, } From aee0d7b2cc71cbe928f14cb31281cfb1e8fdad96 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:08:03 +0000 Subject: [PATCH 04/29] fix link for real --- bin/host/src/eigenda_fetcher/mod.rs | 7 +++---- crates/eigenda/src/certificate.rs | 2 -- crates/eigenda/src/lib.rs | 1 - 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index c3a735a..e5b36c7 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -4,16 +4,16 @@ use crate::eigenda_blobs::OnlineEigenDABlobProvider; use alloy_primitives::{keccak256, B256}; use alloy_provider::ReqwestProvider; +use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; +use hokulea_eigenda::BlobInfo; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; use kona_preimage::{PreimageKey, PreimageKeyType}; use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; -use hokulea_eigenda::BlobInfo; -use alloy_rlp::Decodable; /// The [FetcherWithEigenDASupport] struct wraps and extends kona's [Fetcher] struct with the ability /// to fetch preimages from EigenDA. @@ -138,13 +138,12 @@ where trace!(target: "fetcher_with_eigenda_support", "Fetching hint: {hint_type} {hint_data}"); if hint_type == ExtendedHintType::EigenDACommitment { - let item_slice = hint_data.as_ref(); // the fourth because 0x01010000 in the beginnin is metadata let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; info!("cert_blob_info {:?}", cert_blob_info); - + let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs index eedfd65..afcb198 100644 --- a/crates/eigenda/src/certificate.rs +++ b/crates/eigenda/src/certificate.rs @@ -3,7 +3,6 @@ use alloy_rlp::{RlpDecodable, RlpEncodable}; use alloc::vec::Vec; - #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct G1Commitment { pub x: [u8; 32], @@ -26,7 +25,6 @@ pub struct BlobHeader { pub blob_quorum_params: Vec, } - #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct BatchHeader { pub batch_root: Bytes, diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index e457bbf..68e0d5d 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -29,4 +29,3 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; - From 2d17537ce3862fbed29e8fd653bde9de1d92db14 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:23:44 +0000 Subject: [PATCH 05/29] make it non eigenda-proxy compatible --- bin/host/src/eigenda_fetcher/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index e5b36c7..84671c1 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -141,8 +141,11 @@ where let item_slice = hint_data.as_ref(); // the fourth because 0x01010000 in the beginnin is metadata - let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..])?; - info!("cert_blob_info {:?}", cert_blob_info); + match BlobInfo::decode(&mut &item_slice[4..]) { + Ok( cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), + Err(e) => info!("cannot decode cert_blob_info {:?}", e), + } + let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); From 40e86894de9ec719cdc5f626f522f50a5d6edad1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 22:25:25 +0000 Subject: [PATCH 06/29] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 84671c1..74b9e9c 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -142,10 +142,9 @@ where // the fourth because 0x01010000 in the beginnin is metadata match BlobInfo::decode(&mut &item_slice[4..]) { - Ok( cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), + Ok(cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), Err(e) => info!("cannot decode cert_blob_info {:?}", e), } - let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); From 6c0c7275d09cad3bd6037e42df6986c7c3ab41c7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 20 Dec 2024 02:09:48 +0000 Subject: [PATCH 07/29] wip --- bin/host/src/eigenda_fetcher/mod.rs | 47 +++++++++++++++++++++++++++- crates/proof/src/eigenda_provider.rs | 43 ++++++++++++++++++++----- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 74b9e9c..f89da31 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -159,11 +159,56 @@ where // Acquire a lock on the key-value store and set the preimages. let mut kv_write_lock = self.kv_store.write().await; - // Set the preimage for the blob commitment. + // ToDo - remove it once cert is actually correct kv_write_lock.set( PreimageKey::new(*keccak256(cert), PreimageKeyType::GlobalGeneric).into(), eigenda_blob.to_vec(), )?; + + // fake a commitment + let t1 = cert.clone(); + let mut kzg_commitment = [0u8; 32]; + let mut a = 32; + if a > t1.len() { + a = t1.len() + } + kzg_commitment[..a].copy_from_slice(t1.as_ref()); + let blob_length = (eigenda_blob.len() + 32 - 1) / 32; // in term of field element + let kzg_proof = cert.clone(); + // end of fake + + // Write all the field elements to the key-value store. + // The preimage oracle key for each field element is the keccak256 hash of + // `abi.encodePacked(cert.KZGCommitment, uint256(i))` + let mut blob_key = [0u8; 80]; + blob_key[..32].copy_from_slice(kzg_commitment.as_ref()); + for i in 0..blob_length { + blob_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + let blob_key_hash = keccak256(blob_key.as_ref()); + + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), + blob_key.into(), + )?; + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), + eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), + )?; + } + + // Write the KZG Proof as the last element. + blob_key[72..].copy_from_slice((blob_length).to_be_bytes().as_ref()); + let blob_key_hash = keccak256(blob_key.as_ref()); + + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), + blob_key.into(), + )?; + kv_write_lock.set( + PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), + kzg_proof.to_vec(), + )?; + } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); } diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 10e01e3..601693c 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -32,14 +32,43 @@ impl EigenDABlobProvider for OracleEigenDAProvider .write(&ExtendedHintType::EigenDACommitment.encode_with(&[cert])) .await .map_err(OracleProviderError::Preimage)?; + + // hack - remove later, when cert actually contain length let data = self - .oracle - .get(PreimageKey::new( - *keccak256(cert), - PreimageKeyType::GlobalGeneric, - )) - .await - .map_err(OracleProviderError::Preimage)?; + .oracle + .get(PreimageKey::new( + *keccak256(cert), + PreimageKeyType::GlobalGeneric, + )) + .await + .map_err(OracleProviderError::Preimage)?; + let blob_size = data.len(); + // + + let mut blob = vec![0, blob_size]; + let mut field_element_key = [0u8; 80]; + + + + field_element_key[..48].copy_from_slice(commitment.as_ref()); + for i in 0..FIELD_ELEMENTS_PER_BLOB { + field_element_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + + let mut field_element = [0u8; 32]; + self.oracle + .get_exact( + PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::Blob), + &mut field_element, + ) + .await + .map_err(OracleProviderError::Preimage)?; + blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); + } + + tracing::info!(target: "client_oracle", "Retrieved blob {blob_hash:?} from the oracle."); + + + Ok(data.into()) } From 1ada330010e7a24bf0ba53e76f485eaf39782919 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 07:27:22 +0000 Subject: [PATCH 08/29] add partial decoding logic to be add to read raw blob --- Cargo.lock | 500 +++++++++++++++++++++++++-- Cargo.toml | 2 + bin/client/justfile | 2 +- bin/host/src/eigenda_blobs.rs | 2 +- bin/host/src/eigenda_fetcher/mod.rs | 44 ++- crates/eigenda/Cargo.toml | 3 + crates/eigenda/src/eigenda.rs | 43 ++- crates/eigenda/src/errors.rs | 7 + crates/eigenda/src/lib.rs | 3 + crates/proof/Cargo.toml | 4 + crates/proof/src/eigenda_provider.rs | 54 +-- 11 files changed, 584 insertions(+), 80 deletions(-) create mode 100644 crates/eigenda/src/errors.rs diff --git a/Cargo.lock b/Cargo.lock index 80614e6..189cdb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,7 +167,7 @@ dependencies = [ "alloy-sol-types", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -193,7 +193,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -267,7 +267,7 @@ dependencies = [ "schnellru", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", "tokio", "tracing", "url", @@ -354,7 +354,7 @@ dependencies = [ "alloy-rpc-types-engine", "serde", "serde_with", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -425,7 +425,7 @@ dependencies = [ "auto_impl", "elliptic-curve", "k256", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -498,7 +498,7 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror", + "thiserror 2.0.7", "tokio", "tower", "tracing", @@ -592,6 +592,39 @@ version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +[[package]] +name = "ark-bn254" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69eab57e8d2663efa5c63135b2af4f396d66424f88954c21104125ab6b3e6bc" +dependencies = [ + "ark-ec", + "ark-ff 0.5.0", + "ark-std 0.5.0", +] + +[[package]] +name = "ark-ec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d68f2d516162846c1238e755a7c4d131b892b70cc70c471a8e3ca3ed818fce" +dependencies = [ + "ahash", + "ark-ff 0.5.0", + "ark-poly", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "educe", + "fnv", + "hashbrown 0.15.2", + "itertools 0.13.0", + "num-bigint", + "num-integer", + "num-traits", + "rayon", + "zeroize", +] + [[package]] name = "ark-ff" version = "0.3.0" @@ -630,6 +663,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ark-ff" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" +dependencies = [ + "ark-ff-asm 0.5.0", + "ark-ff-macros 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "educe", + "itertools 0.13.0", + "num-bigint", + "num-traits", + "paste", + "rayon", + "zeroize", +] + [[package]] name = "ark-ff-asm" version = "0.3.0" @@ -650,6 +704,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ark-ff-asm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" +dependencies = [ + "quote", + "syn 2.0.90", +] + [[package]] name = "ark-ff-macros" version = "0.3.0" @@ -675,6 +739,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ark-ff-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "ark-poly" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579305839da207f02b89cd1679e50e67b4331e2f9294a57693e5051b7703fe27" +dependencies = [ + "ahash", + "ark-ff 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "educe", + "fnv", + "hashbrown 0.15.2", + "rayon", +] + [[package]] name = "ark-serialize" version = "0.3.0" @@ -696,6 +789,31 @@ dependencies = [ "num-bigint", ] +[[package]] +name = "ark-serialize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" +dependencies = [ + "ark-serialize-derive", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "num-bigint", + "rayon", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "ark-std" version = "0.3.0" @@ -716,6 +834,17 @@ dependencies = [ "rand", ] +[[package]] +name = "ark-std" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" +dependencies = [ + "num-traits", + "rand", + "rayon", +] + [[package]] name = "arrayvec" version = "0.7.6" @@ -815,7 +944,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1130,6 +1259,43 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1285,6 +1451,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1322,6 +1509,18 @@ dependencies = [ "spki", ] +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "either" version = "1.13.0" @@ -1356,6 +1555,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-ordinalize" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "enumn" version = "0.1.14" @@ -1443,6 +1662,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1725,8 +1954,11 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "async-trait", + "bytes", "kona-derive", "op-alloy-protocol", + "rust-kzg-bn254", + "thiserror 2.0.7", "tracing", ] @@ -1756,6 +1988,7 @@ name = "hokulea-proof" version = "0.1.0" dependencies = [ "alloy-primitives", + "alloy-rlp", "async-trait", "hokulea-eigenda", "kona-derive", @@ -1765,6 +1998,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "op-alloy-rpc-types-engine", + "tracing", ] [[package]] @@ -2180,7 +2414,7 @@ dependencies = [ "serde", "serde_json", "spin", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2199,7 +2433,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "op-alloy-rpc-types-engine", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2217,7 +2451,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "op-alloy-rpc-types-engine", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2235,7 +2469,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-rpc-types-engine", "revm", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2284,7 +2518,7 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "alloy-trie", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2295,7 +2529,7 @@ dependencies = [ "alloy-primitives", "async-channel", "async-trait", - "thiserror", + "thiserror 2.0.7", "tracing", ] @@ -2323,7 +2557,7 @@ dependencies = [ "serde", "serde_json", "spin", - "thiserror", + "thiserror 2.0.7", "tokio", "tracing", ] @@ -2337,7 +2571,7 @@ dependencies = [ "cfg-if", "kona-preimage", "linked_list_allocator", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2381,7 +2615,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", ] [[package]] @@ -2678,7 +2922,7 @@ dependencies = [ "alloy-serde", "derive_more", "serde", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2693,7 +2937,7 @@ dependencies = [ "alloy-sol-types", "serde", "serde_repr", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2715,7 +2959,7 @@ dependencies = [ "op-alloy-consensus", "op-alloy-genesis", "serde", - "thiserror", + "thiserror 2.0.7", "tracing", "unsigned-varint", ] @@ -2748,7 +2992,7 @@ dependencies = [ "op-alloy-genesis", "op-alloy-protocol", "serde", - "thiserror", + "thiserror 2.0.7", ] [[package]] @@ -2795,6 +3039,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "overload" version = "0.1.1" @@ -2865,7 +3115,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2887,7 +3137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror", + "thiserror 2.0.7", "ucd-trie", ] @@ -3095,6 +3345,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.8" @@ -3104,6 +3374,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "regex" version = "1.11.1" @@ -3325,6 +3606,32 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" +[[package]] +name = "rust-kzg-bn254" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdae4058a9f604acf7023d99d931d6f30261fff93787bcfd1f1ccfc725b701c" +dependencies = [ + "ark-bn254", + "ark-ec", + "ark-ff 0.5.0", + "ark-poly", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "byteorder", + "crossbeam-channel", + "directories", + "hex-literal", + "num-bigint", + "num-traits", + "num_cpus", + "rand", + "rayon", + "sha2", + "sys-info", + "ureq", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3386,7 +3693,9 @@ version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3867,6 +4176,16 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -3907,13 +4226,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.7", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -4218,6 +4557,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots", +] + [[package]] name = "url" version = "2.5.4" @@ -4380,6 +4735,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4410,7 +4774,7 @@ checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ "windows-result", "windows-strings", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4419,7 +4783,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4429,7 +4793,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ "windows-result", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", ] [[package]] @@ -4438,7 +4811,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -4447,7 +4820,22 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -4456,28 +4844,46 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4490,24 +4896,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 130a138..f793a27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,12 +62,14 @@ cfg-if = "1.0.0" reqwest = "0.12.9" async-trait = "0.1.83" linked_list_allocator = "0.10.5" +bytes = "1.9.0" # General sha2 = { version = "0.10.8", default-features = false } c-kzg = { version = "2.0.0", default-features = false } anyhow = { version = "1.0.93", default-features = false } thiserror = { version = "2.0.4", default-features = false } +rust-kzg-bn254 = { version = "0.2.1", default-features = false } # Tracing tracing-loki = "0.2.5" diff --git a/bin/client/justfile b/bin/client/justfile index fb05bd2..2773ded 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="../../../optimism/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="~/op-main-repo/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) diff --git a/bin/host/src/eigenda_blobs.rs b/bin/host/src/eigenda_blobs.rs index f9c675b..36ce0f4 100644 --- a/bin/host/src/eigenda_blobs.rs +++ b/bin/host/src/eigenda_blobs.rs @@ -30,7 +30,7 @@ impl OnlineEigenDABlobProvider { ) -> Result { let url = format!("{}/{}/{}", self.base, GET_METHOD, cert.slice(1..)); - let raw_response = self.inner.get(url).send().await?; + let raw_response = self.inner.get(url).header("raw", "true").send().await?; raw_response.bytes().await } diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index f89da31..5f303ad 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -161,29 +161,35 @@ where // ToDo - remove it once cert is actually correct kv_write_lock.set( - PreimageKey::new(*keccak256(cert), PreimageKeyType::GlobalGeneric).into(), + PreimageKey::new(*keccak256(cert.clone()), PreimageKeyType::GlobalGeneric).into(), eigenda_blob.to_vec(), )?; - // fake a commitment - let t1 = cert.clone(); - let mut kzg_commitment = [0u8; 32]; - let mut a = 32; - if a > t1.len() { - a = t1.len() - } - kzg_commitment[..a].copy_from_slice(t1.as_ref()); - let blob_length = (eigenda_blob.len() + 32 - 1) / 32; // in term of field element - let kzg_proof = cert.clone(); - // end of fake + // data + let item_slice = cert.as_ref(); + let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); + info!("cert_blob_info {:?}", cert_blob_info); // Write all the field elements to the key-value store. // The preimage oracle key for each field element is the keccak256 hash of // `abi.encodePacked(cert.KZGCommitment, uint256(i))` - let mut blob_key = [0u8; 80]; - blob_key[..32].copy_from_slice(kzg_commitment.as_ref()); + + let mut blob_key = [0u8; 96]; + blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); + blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); + + // Todo ensure data_length is always power of 2. Proxy made mistake + let data_size = cert_blob_info.blob_header.data_length as u64; + let blob_length: u64 = data_size / 32; + + // proxy could just return the original blob + let mut padded_eigenda_blob = vec![0u8; data_size as usize]; + padded_eigenda_blob[..eigenda_blob.len()].copy_from_slice(eigenda_blob.as_ref()); + + info!("cert_blob_info blob_length {:?}", blob_length); + for i in 0..blob_length { - blob_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + blob_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); let blob_key_hash = keccak256(blob_key.as_ref()); kv_write_lock.set( @@ -192,22 +198,24 @@ where )?; kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), + padded_eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), )?; } // Write the KZG Proof as the last element. - blob_key[72..].copy_from_slice((blob_length).to_be_bytes().as_ref()); + blob_key[88..].copy_from_slice((blob_length).to_be_bytes().as_ref()); let blob_key_hash = keccak256(blob_key.as_ref()); kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), blob_key.into(), )?; + // proof to be done kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - kzg_proof.to_vec(), + [1,2,3].to_vec(), )?; + } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); diff --git a/crates/eigenda/Cargo.toml b/crates/eigenda/Cargo.toml index baa1f5f..c6ba39e 100644 --- a/crates/eigenda/Cargo.toml +++ b/crates/eigenda/Cargo.toml @@ -12,6 +12,9 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true tracing.workspace = true async-trait.workspace = true +thiserror.workspace = true +bytes.workspace = true +rust-kzg-bn254.workspace = true [features] default = [] diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 69e5d32..df23eeb 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -1,17 +1,22 @@ //! Contains the [EigenDADataSource], which is a concrete implementation of the //! [DataAvailabilityProvider] trait for the EigenDA protocol. + use crate::eigenda_blobs::EigenDABlobSource; use crate::traits::EigenDABlobProvider; +use crate::errors::CodecError; use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; +use bytes::buf::Buf; use async_trait::async_trait; use kona_derive::{ sources::EthereumDataSource, traits::{BlobProvider, ChainProvider, DataAvailabilityProvider}, types::PipelineResult, + errors::{PipelineErrorKind, PipelineError, PipelineEncodingError}, }; use op_alloy_protocol::BlockInfo; +use rust_kzg_bn254::helpers::remove_empty_byte_from_padded_bytes_unchecked; /// A factory for creating an Ethereum data source provider. #[derive(Debug, Clone)] @@ -61,9 +66,13 @@ where // just dump all the data out info!(target: "eth-datasource", "next item {:?}", item); - let eigenda_source_result = self.eigenda_source.next(&item).await; - info!(target: "eigenda-datasource", "eigenda_source_result {:?}", eigenda_source_result); - eigenda_source_result + let padded_eigenda_blob = self.eigenda_source.next(&item).await?; + info!(target: "eigenda-datasource", "eigenda_source_result {:?}", padded_eigenda_blob); + + // get the actual blob as encoded inside blob + let eigenda_blob = self.default_decode_blob(padded_eigenda_blob)?; + + Ok(eigenda_blob) } fn clear(&mut self) { @@ -71,3 +80,31 @@ where self.ethereum_source.clear(); } } + +impl EigenDADataSource +where + C: ChainProvider + Send + Sync + Clone + Debug, + B: BlobProvider + Send + Sync + Clone + Debug, + A: EigenDABlobProvider + Send + Sync + Clone + Debug, +{ + // https://github.com/Layr-Labs/eigenda/blob/1345e77c8a91fed8e5e33f02c3e32c9ed9921670/api/clients/codecs/default_blob_codec.go#L44 + fn default_decode_blob(&self, padded_eigenda_blob: Bytes) -> PipelineResult { + if padded_eigenda_blob.len() < 32 { + // ToDo format error better + //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); + unimplemented!() + } + + info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", padded_eigenda_blob); + + let content_size = padded_eigenda_blob.slice(2..6).get_u32(); + info!(target: "eigenda-datasource", "content_size {:?}", content_size); + let codec_data = padded_eigenda_blob.slice(32..); + + let blob_content = remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content: Bytes = blob_content.into(); + + Ok(blob_content.slice(..content_size as usize)) + } + +} \ No newline at end of file diff --git a/crates/eigenda/src/errors.rs b/crates/eigenda/src/errors.rs new file mode 100644 index 0000000..c9c2867 --- /dev/null +++ b/crates/eigenda/src/errors.rs @@ -0,0 +1,7 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CodecError { + #[error("blob does not contain 32 header bytes, meaning it is malformed")] + BlobTooShort, +} diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 68e0d5d..231a598 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -29,3 +29,6 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; + +mod errors; +pub use errors::CodecError; \ No newline at end of file diff --git a/crates/proof/Cargo.toml b/crates/proof/Cargo.toml index b03c361..4d6fc6b 100644 --- a/crates/proof/Cargo.toml +++ b/crates/proof/Cargo.toml @@ -19,6 +19,10 @@ alloy-primitives.workspace = true op-alloy-protocol.workspace = true op-alloy-rpc-types-engine.workspace = true op-alloy-genesis = { workspace = true, features = ["serde"] } +alloy-rlp.workspace = true + + +tracing.workspace = true # General async-trait.workspace = true diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 601693c..f335216 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -2,12 +2,14 @@ use alloc::boxed::Box; use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; -use hokulea_eigenda::EigenDABlobProvider; +use hokulea_eigenda::{EigenDABlobProvider, BlobInfo}; use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; use crate::hint::ExtendedHintType; +use alloy_rlp::Decodable; +use tracing::info; /// The oracle-backed EigenDA provider for the client program. #[derive(Debug, Clone)] @@ -33,43 +35,51 @@ impl EigenDABlobProvider for OracleEigenDAProvider .await .map_err(OracleProviderError::Preimage)?; + // the fourth because 0x01010000 in the beginnin is metadata + let item_slice = cert.as_ref(); + let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); + info!("cert_blob_info {:?}", cert_blob_info); + // hack - remove later, when cert actually contain length let data = self - .oracle - .get(PreimageKey::new( - *keccak256(cert), - PreimageKeyType::GlobalGeneric, - )) - .await - .map_err(OracleProviderError::Preimage)?; - let blob_size = data.len(); - // - - let mut blob = vec![0, blob_size]; - let mut field_element_key = [0u8; 80]; + .oracle + .get(PreimageKey::new( + *keccak256(cert), + PreimageKeyType::GlobalGeneric, + )) + .await + .map_err(OracleProviderError::Preimage)?; + + let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; + // 96 because our g1 commitment has 64 bytes in v1 + let mut field_element_key = [0u8; 96]; - field_element_key[..48].copy_from_slice(commitment.as_ref()); - for i in 0..FIELD_ELEMENTS_PER_BLOB { - field_element_key[72..].copy_from_slice(i.to_be_bytes().as_ref()); + // ToDo data_length should be power of 2, proxy should have returned it with dividing 32 + let data_length = cert_blob_info.blob_header.data_length as u64 / 32; + + info!("cert_blob_info.blob_header.data_length {:?}", data_length); + + field_element_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); + field_element_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); + for i in 0..data_length { + field_element_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); let mut field_element = [0u8; 32]; self.oracle .get_exact( - PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::Blob), + PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::GlobalGeneric), &mut field_element, ) .await .map_err(OracleProviderError::Preimage)?; blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); } + + info!("cert_blob_info blob {:?}", blob); - tracing::info!(target: "client_oracle", "Retrieved blob {blob_hash:?} from the oracle."); - - - - Ok(data.into()) + Ok(blob.into()) } async fn get_element(&mut self, cert: &Bytes, element: &Bytes) -> Result { From 10959c3ffbe9bbf671b438671ad88f6c53d8eb2e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 17:41:07 +0000 Subject: [PATCH 09/29] move decode logic --- .gitignore | 1 + bin/host/src/eigenda_fetcher/mod.rs | 8 ++---- crates/eigenda/src/eigenda.rs | 41 ++-------------------------- crates/eigenda/src/eigenda_data.rs | 28 +++++++++++++++++-- crates/eigenda/src/lib.rs | 2 +- crates/proof/src/eigenda_provider.rs | 14 ++++++---- 6 files changed, 41 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 5d1f033..b9dd3f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ data/ +optimism/ diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 5f303ad..736f070 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -178,14 +178,14 @@ where blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); - // Todo ensure data_length is always power of 2. Proxy made mistake + // Todo ensure data_length is always power of 2. Proxy made mistake let data_size = cert_blob_info.blob_header.data_length as u64; let blob_length: u64 = data_size / 32; // proxy could just return the original blob let mut padded_eigenda_blob = vec![0u8; data_size as usize]; padded_eigenda_blob[..eigenda_blob.len()].copy_from_slice(eigenda_blob.as_ref()); - + info!("cert_blob_info blob_length {:?}", blob_length); for i in 0..blob_length { @@ -213,10 +213,8 @@ where // proof to be done kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - [1,2,3].to_vec(), + [1, 2, 3].to_vec(), )?; - - } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); } diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index df23eeb..9653c36 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -2,21 +2,19 @@ //! [DataAvailabilityProvider] trait for the EigenDA protocol. use crate::eigenda_blobs::EigenDABlobSource; -use crate::traits::EigenDABlobProvider; use crate::errors::CodecError; +use crate::traits::EigenDABlobProvider; use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; -use bytes::buf::Buf; use async_trait::async_trait; use kona_derive::{ + errors::{PipelineEncodingError, PipelineError, PipelineErrorKind}, sources::EthereumDataSource, traits::{BlobProvider, ChainProvider, DataAvailabilityProvider}, types::PipelineResult, - errors::{PipelineErrorKind, PipelineError, PipelineEncodingError}, }; use op_alloy_protocol::BlockInfo; -use rust_kzg_bn254::helpers::remove_empty_byte_from_padded_bytes_unchecked; /// A factory for creating an Ethereum data source provider. #[derive(Debug, Clone)] @@ -66,12 +64,7 @@ where // just dump all the data out info!(target: "eth-datasource", "next item {:?}", item); - let padded_eigenda_blob = self.eigenda_source.next(&item).await?; - info!(target: "eigenda-datasource", "eigenda_source_result {:?}", padded_eigenda_blob); - - // get the actual blob as encoded inside blob - let eigenda_blob = self.default_decode_blob(padded_eigenda_blob)?; - + let eigenda_blob = self.eigenda_source.next(&item).await?; Ok(eigenda_blob) } @@ -80,31 +73,3 @@ where self.ethereum_source.clear(); } } - -impl EigenDADataSource -where - C: ChainProvider + Send + Sync + Clone + Debug, - B: BlobProvider + Send + Sync + Clone + Debug, - A: EigenDABlobProvider + Send + Sync + Clone + Debug, -{ - // https://github.com/Layr-Labs/eigenda/blob/1345e77c8a91fed8e5e33f02c3e32c9ed9921670/api/clients/codecs/default_blob_codec.go#L44 - fn default_decode_blob(&self, padded_eigenda_blob: Bytes) -> PipelineResult { - if padded_eigenda_blob.len() < 32 { - // ToDo format error better - //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); - unimplemented!() - } - - info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", padded_eigenda_blob); - - let content_size = padded_eigenda_blob.slice(2..6).get_u32(); - info!(target: "eigenda-datasource", "content_size {:?}", content_size); - let codec_data = padded_eigenda_blob.slice(32..); - - let blob_content = remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); - let blob_content: Bytes = blob_content.into(); - - Ok(blob_content.slice(..content_size as usize)) - } - -} \ No newline at end of file diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 7a60e9d..80f895e 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -1,7 +1,10 @@ use alloy_primitives::Bytes; +use bytes::buf::Buf; use kona_derive::errors::BlobDecodingError; +use rust_kzg_bn254::helpers; + #[derive(Default, Clone, Debug)] /// Represents the data structure for EigenDA Blob. pub struct EigenDABlobData { @@ -13,9 +16,28 @@ impl EigenDABlobData { /// Decodes the blob into raw byte data. /// Returns a [BlobDecodingError] if the blob is invalid. pub(crate) fn decode(&self) -> Result { - // where we can implement zero bytes etc. - info!(target: "eigenda-blobdata", "decode {} {:?}", self.blob.len(), self.blob.clone()); - Ok(self.blob.clone()) + if self.blob.len() < 32 { + // ToDo format error better + //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); + unimplemented!() + } + + info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", self.blob); + + // see https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L44 + let content_size = self.blob.slice(2..6).get_u32(); + info!(target: "eigenda-datasource", "content_size {:?}", content_size); + + // the first 32 Bytes are reserved as the header field element + let codec_data = self.blob.slice(32..); + + // rust kzg bn254 impl already + let blob_content = + helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); + let blob_content: Bytes = blob_content.into(); + + // take data + Ok(blob_content.slice(..content_size as usize)) } } diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 231a598..bfc282a 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -31,4 +31,4 @@ mod certificate; pub use certificate::BlobInfo; mod errors; -pub use errors::CodecError; \ No newline at end of file +pub use errors::CodecError; diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index f335216..704aa88 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -2,7 +2,7 @@ use alloc::boxed::Box; use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; -use hokulea_eigenda::{EigenDABlobProvider, BlobInfo}; +use hokulea_eigenda::{BlobInfo, EigenDABlobProvider}; use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; @@ -50,9 +50,8 @@ impl EigenDABlobProvider for OracleEigenDAProvider .await .map_err(OracleProviderError::Preimage)?; - let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; - + // 96 because our g1 commitment has 64 bytes in v1 let mut field_element_key = [0u8; 96]; @@ -60,7 +59,7 @@ impl EigenDABlobProvider for OracleEigenDAProvider let data_length = cert_blob_info.blob_header.data_length as u64 / 32; info!("cert_blob_info.blob_header.data_length {:?}", data_length); - + field_element_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); field_element_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); for i in 0..data_length { @@ -69,14 +68,17 @@ impl EigenDABlobProvider for OracleEigenDAProvider let mut field_element = [0u8; 32]; self.oracle .get_exact( - PreimageKey::new(*keccak256(field_element_key), PreimageKeyType::GlobalGeneric), + PreimageKey::new( + *keccak256(field_element_key), + PreimageKeyType::GlobalGeneric, + ), &mut field_element, ) .await .map_err(OracleProviderError::Preimage)?; blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); } - + info!("cert_blob_info blob {:?}", blob); Ok(blob.into()) From 854ec241d86c745f5aeadd848ca0f7d90bf30ba1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 17:42:20 +0000 Subject: [PATCH 10/29] remove debug log --- crates/eigenda/src/eigenda_blobs.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/crates/eigenda/src/eigenda_blobs.rs b/crates/eigenda/src/eigenda_blobs.rs index 1fb3957..d2994a6 100644 --- a/crates/eigenda/src/eigenda_blobs.rs +++ b/crates/eigenda/src/eigenda_blobs.rs @@ -39,21 +39,15 @@ where /// Fetches the next blob from the source. pub async fn next(&mut self, altda_commitment: &Bytes) -> PipelineResult { - info!(target: "eigenda-blobsource", "next"); self.load_blobs(altda_commitment).await?; - info!(target: "eigenda-blobsource", "next 1"); let next_data = match self.next_data() { Ok(d) => d, Err(e) => return e, }; - info!(target: "eigenda-blobsource", "next 2"); // Decode the blob data to raw bytes. // Otherwise, ignore blob and recurse next. match next_data.decode() { - Ok(d) => { - info!(target: "eigenda-blobsource", "next 3"); - Ok(d) - } + Ok(d) => Ok(d), Err(_) => { warn!(target: "blob-source", "Failed to decode blob data, skipping"); panic!() From 5e3a6953f1f0b3892b9abb528e220adeb16d0d12 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 17:45:30 +0000 Subject: [PATCH 11/29] change altda to eigenda --- bin/host/src/eigenda_fetcher/mod.rs | 4 ++-- crates/eigenda/src/eigenda_blobs.rs | 19 ++++++++----------- crates/proof/src/hint.rs | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 736f070..6c02b31 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -96,7 +96,7 @@ where /// Fetch the preimage for the given key. The requested is routed to the appropriate fetcher /// based on the last hint that was received (see hint() above). - /// FetcherWithEigenDASupport -> get_preimage_altda -> prefetch that only understands altda hints + /// FetcherWithEigenDASupport -> get_preimage_eigenda -> prefetch that only understands eigenda hints /// \-> Fetcher -> get_preimage -> prefetch that understands all other hints pub async fn get_preimage(&self, key: B256) -> Result> { match self.last_eigenda_hint.as_ref() { @@ -147,7 +147,7 @@ where } let cert = hint_data; - info!(target: "fetcher_with_eigenda_support", "Fetching AltDACommitment cert: {:?}", cert); + info!(target: "fetcher_with_eigenda_support", "Fetching eigenda commitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. let eigenda_blob = self .eigenda_blob_provider diff --git a/crates/eigenda/src/eigenda_blobs.rs b/crates/eigenda/src/eigenda_blobs.rs index d2994a6..741176c 100644 --- a/crates/eigenda/src/eigenda_blobs.rs +++ b/crates/eigenda/src/eigenda_blobs.rs @@ -17,7 +17,7 @@ where B: EigenDABlobProvider + Send, { /// Fetches blobs. - pub altda_fetcher: B, + pub eigenda_fetcher: B, /// EigenDA blobs. pub data: Vec, /// Whether the source is open. @@ -29,17 +29,17 @@ where B: EigenDABlobProvider + Send, { /// Creates a new blob source. - pub const fn new(altda_fetcher: B) -> Self { + pub const fn new(eigenda_fetcher: B) -> Self { Self { - altda_fetcher, + eigenda_fetcher, data: Vec::new(), open: false, } } /// Fetches the next blob from the source. - pub async fn next(&mut self, altda_commitment: &Bytes) -> PipelineResult { - self.load_blobs(altda_commitment).await?; + pub async fn next(&mut self, eigenda_commitment: &Bytes) -> PipelineResult { + self.load_blobs(eigenda_commitment).await?; let next_data = match self.next_data() { Ok(d) => d, Err(e) => return e, @@ -51,8 +51,6 @@ where Err(_) => { warn!(target: "blob-source", "Failed to decode blob data, skipping"); panic!() - // todo need to add recursion - // self.next(altda_commitment).await } } } @@ -64,14 +62,13 @@ where } /// Loads blob data into the source if it is not open. - async fn load_blobs(&mut self, altda_commitment: &Bytes) -> Result<(), BlobProviderError> { + async fn load_blobs(&mut self, eigenda_commitment: &Bytes) -> Result<(), BlobProviderError> { if self.open { return Ok(()); } - info!(target: "eigenda-blobsource", "going to fetch through altda fetcher"); - // it should use self.altda_fetcher to get the blob - let data = self.altda_fetcher.get_blob(altda_commitment).await; + info!(target: "eigenda-blobsource", "going to fetch through eigenda fetcher"); + let data = self.eigenda_fetcher.get_blob(eigenda_commitment).await; match data { Ok(data) => { self.open = true; diff --git a/crates/proof/src/hint.rs b/crates/proof/src/hint.rs index 0e388ad..af453b3 100644 --- a/crates/proof/src/hint.rs +++ b/crates/proof/src/hint.rs @@ -1,4 +1,4 @@ -//! This module contains the [ExtendedHintType], which adds an AltDACommitment case to kona's [HintType] enum. +//! This module contains the [ExtendedHintType], which adds an EigenDACommitment case to kona's [HintType] enum. use alloc::{ string::{String, ToString}, From d0782e745fb0d69d70fe890d4e0d88aa1625f592 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 19:12:24 +0000 Subject: [PATCH 12/29] add comments and add staleness check and empty bytes check --- crates/eigenda/src/eigenda.rs | 26 ++++++++++-- crates/eigenda/src/eigenda_data.rs | 2 + crates/proof/src/eigenda_provider.rs | 59 ++++++++++++++++++---------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 9653c36..5168275 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -4,6 +4,8 @@ use crate::eigenda_blobs::EigenDABlobSource; use crate::errors::CodecError; use crate::traits::EigenDABlobProvider; +use crate::BlobInfo; +use alloy_rlp::Decodable; use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; @@ -59,12 +61,28 @@ where async fn next(&mut self, block_ref: &BlockInfo) -> PipelineResult { // then acutally use ethereum da to fetch. items are Bytes - let item = self.ethereum_source.next(block_ref).await?; + let cert = self.ethereum_source.next(block_ref).await?; - // just dump all the data out - info!(target: "eth-datasource", "next item {:?}", item); + // verify if cert is too stale + let cert_blob_info = BlobInfo::decode(&mut &cert.as_ref()[4..]).unwrap(); + info!("cert_blob_info {:?}", cert_blob_info); + let rbn = cert_blob_info + .blob_verification_proof + .batch_medatada + .batch_header + .reference_block_number as u64; + let l1_block_number = block_ref.number; - let eigenda_blob = self.eigenda_source.next(&item).await?; + // ToDo make it part of rollup config + let stale_gap = 100 as u64; + + // check staleness + if rbn + stale_gap < l1_block_number { + // return error + unimplemented!() + } + + let eigenda_blob = self.eigenda_source.next(&cert).await?; Ok(eigenda_blob) } diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 80f895e..3f12c30 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -36,6 +36,8 @@ impl EigenDABlobData { helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); let blob_content: Bytes = blob_content.into(); + // might insert a FFT here, + // take data Ok(blob_content.slice(..content_size as usize)) } diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 704aa88..c72458f 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -3,7 +3,7 @@ use alloc::sync::Arc; use alloy_primitives::{keccak256, Bytes}; use async_trait::async_trait; use hokulea_eigenda::{BlobInfo, EigenDABlobProvider}; -use kona_preimage::{CommsClient, PreimageKey, PreimageKeyType}; +use kona_preimage::{errors::PreimageOracleError, CommsClient, PreimageKey, PreimageKeyType}; use kona_proof::errors::OracleProviderError; @@ -37,45 +37,62 @@ impl EigenDABlobProvider for OracleEigenDAProvider // the fourth because 0x01010000 in the beginnin is metadata let item_slice = cert.as_ref(); + + // cert should at least contain 32 bytes for header + 4 bytes for commitment type metadata + if item_slice.len() <= 32 + 4 { + return Err(OracleProviderError::Preimage(PreimageOracleError::Other( + "does not contain header".into(), + ))); + } + + // the first four bytes are metadata, like cert version, OP generic commitement + // see https://github.com/Layr-Labs/eigenda-proxy/blob/main/commitments/mode.go#L39 + // the first byte my guess is the OP let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); info!("cert_blob_info {:?}", cert_blob_info); - // hack - remove later, when cert actually contain length - let data = self - .oracle - .get(PreimageKey::new( - *keccak256(cert), - PreimageKeyType::GlobalGeneric, - )) - .await - .map_err(OracleProviderError::Preimage)?; - let mut blob: Vec = vec![0; cert_blob_info.blob_header.data_length as usize]; // 96 because our g1 commitment has 64 bytes in v1 - let mut field_element_key = [0u8; 96]; - - // ToDo data_length should be power of 2, proxy should have returned it with dividing 32 + // why 96, the original 4844 has bytes length of 80 (it has 48 bytes for commitment) + // even then, it is not that the entire 80 bytes are used. Some bytes are empty + // for solidity optimization, I remember. + // + // ToDo investigate later to decide a right size + let mut blob_key = [0u8; 96]; + + // In eigenDA terminology, length describes the number of field element, size describes + // number of bytes. In eigenda proxy memstore mode, the datalength is wronly assigned to + // be the bytes lenght. We need to resolve it later. + // For now, we internally divides 32. ToDo let data_length = cert_blob_info.blob_header.data_length as u64 / 32; info!("cert_blob_info.blob_header.data_length {:?}", data_length); - field_element_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); - field_element_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); + // the common key + blob_key[..32].copy_from_slice(&cert_blob_info.blob_header.commitment.x); + blob_key[32..64].copy_from_slice(&cert_blob_info.blob_header.commitment.y); + for i in 0..data_length { - field_element_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); + blob_key[88..].copy_from_slice(i.to_be_bytes().as_ref()); let mut field_element = [0u8; 32]; self.oracle .get_exact( - PreimageKey::new( - *keccak256(field_element_key), - PreimageKeyType::GlobalGeneric, - ), + PreimageKey::new(*keccak256(blob_key), PreimageKeyType::GlobalGeneric), &mut field_element, ) .await .map_err(OracleProviderError::Preimage)?; + + // if field element is 0, it means the host has identified that the data + // has breached eigenda invariant, i.e cert is valid + if field_element.len() == 0 { + return Err(OracleProviderError::Preimage(PreimageOracleError::Other( + "field elememnt is empty, breached eigenda invariant".into(), + ))); + } + blob[(i as usize) << 5..(i as usize + 1) << 5].copy_from_slice(field_element.as_ref()); } From 83280f9027c13da6a6702e65c1140d7f86c236df Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 21 Dec 2024 21:38:52 +0000 Subject: [PATCH 13/29] add staleness comment --- crates/eigenda/src/eigenda.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 5168275..dc08eb9 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -77,6 +77,9 @@ where let stale_gap = 100 as u64; // check staleness + // ToDo this would require the op-rollup to follow the same pattern + // but passing blockId to proxy which implement the logic, + // see https://github.com/ethereum-optimism/optimism/blob/0bb2ff57c8133f1e3983820c0bf238001eca119b/op-alt-da/damgr.go#L211 if rbn + stale_gap < l1_block_number { // return error unimplemented!() From b57d10bf4f3af9c102c2b10babc45c9e003ee1c8 Mon Sep 17 00:00:00 2001 From: bx3 <71761807+scale-chain-sub@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:08:09 -0800 Subject: [PATCH 14/29] client host comm with original eigenda proxy --- Cargo.lock | 1 + bin/client/justfile | 2 +- bin/host/Cargo.toml | 3 ++ bin/host/src/eigenda_blobs.rs | 2 +- bin/host/src/eigenda_fetcher/mod.rs | 56 +++++++++++++++-------------- crates/eigenda/src/constant.rs | 3 ++ crates/eigenda/src/lib.rs | 3 ++ 7 files changed, 41 insertions(+), 29 deletions(-) create mode 100644 crates/eigenda/src/constant.rs diff --git a/Cargo.lock b/Cargo.lock index 189cdb6..c62a849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1979,6 +1979,7 @@ dependencies = [ "kona-preimage", "proptest", "reqwest", + "rust-kzg-bn254", "tokio", "tracing", ] diff --git a/bin/client/justfile b/bin/client/justfile index 2773ded..80a4c76 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="~/op-main-repo/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="/Users/bowenxue/Documents/eigenda-integration/optimism/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) diff --git a/bin/host/Cargo.toml b/bin/host/Cargo.toml index 69820d0..73cedb7 100644 --- a/bin/host/Cargo.toml +++ b/bin/host/Cargo.toml @@ -26,5 +26,8 @@ async-trait.workspace = true tokio = { workspace = true, features = ["full"] } clap = { workspace = true, features = ["derive", "env"] } +# Cryptography +rust-kzg-bn254.workspace = true + [dev-dependencies] proptest.workspace = true diff --git a/bin/host/src/eigenda_blobs.rs b/bin/host/src/eigenda_blobs.rs index 36ce0f4..f9c675b 100644 --- a/bin/host/src/eigenda_blobs.rs +++ b/bin/host/src/eigenda_blobs.rs @@ -30,7 +30,7 @@ impl OnlineEigenDABlobProvider { ) -> Result { let url = format!("{}/{}/{}", self.base, GET_METHOD, cert.slice(1..)); - let raw_response = self.inner.get(url).header("raw", "true").send().await?; + let raw_response = self.inner.get(url).send().await?; raw_response.bytes().await } diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 6c02b31..e75ffde 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -8,9 +8,11 @@ use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; use hokulea_eigenda::BlobInfo; +use hokulea_eigenda::BLOB_ENCODING_VERSION; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; use kona_preimage::{PreimageKey, PreimageKeyType}; +use rust_kzg_bn254::helpers; use std::sync::Arc; use tokio::sync::RwLock; use tracing::{error, info, trace, warn}; @@ -138,37 +140,45 @@ where trace!(target: "fetcher_with_eigenda_support", "Fetching hint: {hint_type} {hint_data}"); if hint_type == ExtendedHintType::EigenDACommitment { - let item_slice = hint_data.as_ref(); - - // the fourth because 0x01010000 in the beginnin is metadata - match BlobInfo::decode(&mut &item_slice[4..]) { - Ok(cert_blob_info) => info!("cert_blob_info {:?}", cert_blob_info), - Err(e) => info!("cannot decode cert_blob_info {:?}", e), - } - let cert = hint_data; info!(target: "fetcher_with_eigenda_support", "Fetching eigenda commitment cert: {:?}", cert); // Fetch the blob sidecar from the blob provider. - let eigenda_blob = self + let rollup_data = self .eigenda_blob_provider .fetch_eigenda_blob(&cert) .await .map_err(|e| anyhow!("Failed to fetch eigenda blob: {e}"))?; - info!(target: "fetcher_with_eigenda_support", "eigenda_blob len {}", eigenda_blob.len()); // Acquire a lock on the key-value store and set the preimages. let mut kv_write_lock = self.kv_store.write().await; - // ToDo - remove it once cert is actually correct - kv_write_lock.set( - PreimageKey::new(*keccak256(cert.clone()), PreimageKeyType::GlobalGeneric).into(), - eigenda_blob.to_vec(), - )?; - - // data + // the fourth because 0x01010000 in the beginnin is metadata + let rollup_data_len = rollup_data.len() as u32; let item_slice = cert.as_ref(); let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); - info!("cert_blob_info {:?}", cert_blob_info); + + // Todo ensure data_length is always power of 2. Proxy made mistake + let data_size = cert_blob_info.blob_header.data_length as u64; + let blob_length: u64 = data_size / 32; + + // encode to become raw blob + let codec_rollup_data = helpers::convert_by_padding_empty_byte(rollup_data.as_ref()); + let codec_rollup_data_len = codec_rollup_data.len() as u32; + + let mut raw_blob = vec![0u8; data_size as usize]; + + if 32 + codec_rollup_data_len as u64 > data_size { + return Err(anyhow!("data size is less than reconstructed data codec_rollup_data_len {} data_size {}", codec_rollup_data_len, data_size)); + } + + // blob header + // https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L25 + // raw blob the immediate data just before taking IFFT + raw_blob[1] = BLOB_ENCODING_VERSION; + raw_blob[2..6].copy_from_slice(&rollup_data_len.to_be_bytes()); + + // encode length as uint32 + raw_blob[32..(32 + codec_rollup_data_len as usize)].copy_from_slice(&codec_rollup_data); // Write all the field elements to the key-value store. // The preimage oracle key for each field element is the keccak256 hash of @@ -178,14 +188,6 @@ where blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); - // Todo ensure data_length is always power of 2. Proxy made mistake - let data_size = cert_blob_info.blob_header.data_length as u64; - let blob_length: u64 = data_size / 32; - - // proxy could just return the original blob - let mut padded_eigenda_blob = vec![0u8; data_size as usize]; - padded_eigenda_blob[..eigenda_blob.len()].copy_from_slice(eigenda_blob.as_ref()); - info!("cert_blob_info blob_length {:?}", blob_length); for i in 0..blob_length { @@ -198,7 +200,7 @@ where )?; kv_write_lock.set( PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - padded_eigenda_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), + raw_blob[(i as usize) << 5..(i as usize + 1) << 5].to_vec(), )?; } diff --git a/crates/eigenda/src/constant.rs b/crates/eigenda/src/constant.rs new file mode 100644 index 0000000..d2d8d8e --- /dev/null +++ b/crates/eigenda/src/constant.rs @@ -0,0 +1,3 @@ +/// This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] +/// followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] +pub const BLOB_ENCODING_VERSION: u8 = 0x0; diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index bfc282a..f4ace61 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -32,3 +32,6 @@ pub use certificate::BlobInfo; mod errors; pub use errors::CodecError; + +mod constant; +pub use constant::BLOB_ENCODING_VERSION; From 212aebd40514b531414fd431ebf90fea108b535b Mon Sep 17 00:00:00 2001 From: bx3 <71761807+scale-chain-sub@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:21:57 -0800 Subject: [PATCH 15/29] fix error --- crates/eigenda/src/eigenda.rs | 7 +++---- crates/eigenda/src/eigenda_data.rs | 4 +--- crates/eigenda/src/errors.rs | 7 ------- crates/eigenda/src/lib.rs | 3 --- 4 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 crates/eigenda/src/errors.rs diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index dc08eb9..d35a0cb 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -2,7 +2,6 @@ //! [DataAvailabilityProvider] trait for the EigenDA protocol. use crate::eigenda_blobs::EigenDABlobSource; -use crate::errors::CodecError; use crate::traits::EigenDABlobProvider; use crate::BlobInfo; use alloy_rlp::Decodable; @@ -11,7 +10,7 @@ use alloc::{boxed::Box, fmt::Debug}; use alloy_primitives::Bytes; use async_trait::async_trait; use kona_derive::{ - errors::{PipelineEncodingError, PipelineError, PipelineErrorKind}, + errors::{PipelineError, PipelineErrorKind}, sources::EthereumDataSource, traits::{BlobProvider, ChainProvider, DataAvailabilityProvider}, types::PipelineResult, @@ -81,8 +80,8 @@ where // but passing blockId to proxy which implement the logic, // see https://github.com/ethereum-optimism/optimism/blob/0bb2ff57c8133f1e3983820c0bf238001eca119b/op-alt-da/damgr.go#L211 if rbn + stale_gap < l1_block_number { - // return error - unimplemented!() + // ToDo double check + return Err(PipelineErrorKind::Temporary(PipelineError::EndOfSource)); } let eigenda_blob = self.eigenda_source.next(&cert).await?; diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 3f12c30..cdd8951 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -17,9 +17,7 @@ impl EigenDABlobData { /// Returns a [BlobDecodingError] if the blob is invalid. pub(crate) fn decode(&self) -> Result { if self.blob.len() < 32 { - // ToDo format error better - //return Err(PipelineErrorKind::Temporary(PipelineError::BadEncoding(PipelineEncodingError::SpanBatchError(())))); - unimplemented!() + return Err(BlobDecodingError::InvalidLength); } info!(target: "eigenda-datasource", "padded_eigenda_blob {:?}", self.blob); diff --git a/crates/eigenda/src/errors.rs b/crates/eigenda/src/errors.rs deleted file mode 100644 index c9c2867..0000000 --- a/crates/eigenda/src/errors.rs +++ /dev/null @@ -1,7 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum CodecError { - #[error("blob does not contain 32 header bytes, meaning it is malformed")] - BlobTooShort, -} diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index f4ace61..501fb6c 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -30,8 +30,5 @@ pub use eigenda_data::EigenDABlobData; mod certificate; pub use certificate::BlobInfo; -mod errors; -pub use errors::CodecError; - mod constant; pub use constant::BLOB_ENCODING_VERSION; From 571efb09f5dee202c4465ff660625c723dd3afa7 Mon Sep 17 00:00:00 2001 From: bx3 <71761807+scale-chain-sub@users.noreply.github.com> Date: Sat, 21 Dec 2024 19:57:18 -0800 Subject: [PATCH 16/29] fix lint --- bin/host/src/eigenda_fetcher/mod.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index e75ffde..b2f3663 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -204,19 +204,19 @@ where )?; } - // Write the KZG Proof as the last element. - blob_key[88..].copy_from_slice((blob_length).to_be_bytes().as_ref()); - let blob_key_hash = keccak256(blob_key.as_ref()); - - kv_write_lock.set( - PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), - blob_key.into(), - )?; + // Write the KZG Proof as the last element, needed for ZK + //blob_key[88..].copy_from_slice((blob_length).to_be_bytes().as_ref()); + //let blob_key_hash = keccak256(blob_key.as_ref()); + + //kv_write_lock.set( + // PreimageKey::new(*blob_key_hash, PreimageKeyType::Keccak256).into(), + // blob_key.into(), + //)?; // proof to be done - kv_write_lock.set( - PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), - [1, 2, 3].to_vec(), - )?; + //kv_write_lock.set( + // PreimageKey::new(*blob_key_hash, PreimageKeyType::GlobalGeneric).into(), + // [1, 2, 3].to_vec(), + //)?; } else { panic!("Invalid hint type: {hint_type}. FetcherWithEigenDASupport.prefetch only supports EigenDACommitment hints."); } From 12a0b47af0a4ba8ab87ea422d9fc6ddcc1f94e98 Mon Sep 17 00:00:00 2001 From: Bowen <71761807+scale-chain-sub@users.noreply.github.com> Date: Thu, 2 Jan 2025 12:51:28 -0800 Subject: [PATCH 17/29] fix comments from PR 17 --- bin/host/src/eigenda_fetcher/mod.rs | 2 +- crates/eigenda/src/certificate.rs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index b2f3663..7070cb6 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -152,7 +152,7 @@ where // Acquire a lock on the key-value store and set the preimages. let mut kv_write_lock = self.kv_store.write().await; - // the fourth because 0x01010000 in the beginnin is metadata + // the fourth because 0x01010000 in the beginning is metadata let rollup_data_len = rollup_data.len() as u32; let item_slice = cert.as_ref(); let cert_blob_info = BlobInfo::decode(&mut &item_slice[4..]).unwrap(); diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs index afcb198..03e712d 100644 --- a/crates/eigenda/src/certificate.rs +++ b/crates/eigenda/src/certificate.rs @@ -3,6 +3,9 @@ use alloy_rlp::{RlpDecodable, RlpEncodable}; use alloc::vec::Vec; +// ToDo use prost to generate struct from proto file +// see seggestion, https://github.com/Layr-Labs/hokulea/pull/17#discussion_r1901102921 + #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] pub struct G1Commitment { pub x: [u8; 32], From 67535a7ad9585117669a9d91d0695b16bdf2411e Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 17:02:48 -0500 Subject: [PATCH 18/29] chore: fix bin/client/justfile to use relative path --- bin/client/justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/client/justfile b/bin/client/justfile index 80a4c76..7aabde9 100644 --- a/bin/client/justfile +++ b/bin/client/justfile @@ -69,7 +69,7 @@ run-client-native-against-devnet verbosity='' block_number='' rollup_config_path L1_BEACON_RPC="http://127.0.0.1:5052" L2_RPC="http://127.0.0.1:9545" ROLLUP_NODE_RPC="http://127.0.0.1:7545" - ROLLUP_CONFIG_PATH="/Users/bowenxue/Documents/eigenda-integration/optimism/.devnet/rollup.json" + ROLLUP_CONFIG_PATH="{{justfile_directory()}}/../../optimism/.devnet/rollup.json" if [ -z "{{block_number}}" ]; then BLOCK_NUMBER=$(cast block finalized --json --rpc-url $L2_RPC | jq -r .number | cast 2d) From 642907c16ca45dc6cb1ec00203ed4a31217a343d Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 17:05:09 -0500 Subject: [PATCH 19/29] docs: fix comment typo beginnin->beginning --- crates/proof/src/eigenda_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index c72458f..3f31a71 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -35,7 +35,7 @@ impl EigenDABlobProvider for OracleEigenDAProvider .await .map_err(OracleProviderError::Preimage)?; - // the fourth because 0x01010000 in the beginnin is metadata + // the fourth because 0x01010000 in the beginning is metadata let item_slice = cert.as_ref(); // cert should at least contain 32 bytes for header + 4 bytes for commitment type metadata From 7cfc9d9584c36e7776026cc8c764c8ddbf93fd3c Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 17:08:06 -0500 Subject: [PATCH 20/29] docs: replace ToDo comments with TODO: --- crates/eigenda/src/certificate.rs | 2 +- crates/eigenda/src/eigenda.rs | 6 +++--- crates/proof/src/eigenda_provider.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/eigenda/src/certificate.rs b/crates/eigenda/src/certificate.rs index 03e712d..5394538 100644 --- a/crates/eigenda/src/certificate.rs +++ b/crates/eigenda/src/certificate.rs @@ -3,7 +3,7 @@ use alloy_rlp::{RlpDecodable, RlpEncodable}; use alloc::vec::Vec; -// ToDo use prost to generate struct from proto file +// TODO: use prost to generate struct from proto file // see seggestion, https://github.com/Layr-Labs/hokulea/pull/17#discussion_r1901102921 #[derive(Debug, PartialEq, Clone, RlpEncodable, RlpDecodable)] diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index d35a0cb..56a6878 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -72,15 +72,15 @@ where .reference_block_number as u64; let l1_block_number = block_ref.number; - // ToDo make it part of rollup config + // TODO: make it part of rollup config let stale_gap = 100 as u64; // check staleness - // ToDo this would require the op-rollup to follow the same pattern + // TODO: this would require the op-rollup to follow the same pattern // but passing blockId to proxy which implement the logic, // see https://github.com/ethereum-optimism/optimism/blob/0bb2ff57c8133f1e3983820c0bf238001eca119b/op-alt-da/damgr.go#L211 if rbn + stale_gap < l1_block_number { - // ToDo double check + // TODO: double check return Err(PipelineErrorKind::Temporary(PipelineError::EndOfSource)); } diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index 3f31a71..f7af7ae 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -58,7 +58,7 @@ impl EigenDABlobProvider for OracleEigenDAProvider // even then, it is not that the entire 80 bytes are used. Some bytes are empty // for solidity optimization, I remember. // - // ToDo investigate later to decide a right size + // TODO: investigate later to decide a right size let mut blob_key = [0u8; 96]; // In eigenDA terminology, length describes the number of field element, size describes From 4928ce5bf314620c1253b1e443d13f30e7648eed Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 17:10:16 -0500 Subject: [PATCH 21/29] style: rename BLOB_ENCODING_VERSION -> BLOB_ENCODING_VERSION_0 --- bin/host/src/eigenda_fetcher/mod.rs | 4 ++-- crates/eigenda/src/constant.rs | 2 +- crates/eigenda/src/lib.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index 7070cb6..e3890b6 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -8,7 +8,7 @@ use alloy_rlp::Decodable; use anyhow::{anyhow, Result}; use core::panic; use hokulea_eigenda::BlobInfo; -use hokulea_eigenda::BLOB_ENCODING_VERSION; +use hokulea_eigenda::BLOB_ENCODING_VERSION_0; use hokulea_proof::hint::{ExtendedHint, ExtendedHintType}; use kona_host::{blobs::OnlineBlobProvider, fetcher::Fetcher, kv::KeyValueStore}; use kona_preimage::{PreimageKey, PreimageKeyType}; @@ -174,7 +174,7 @@ where // blob header // https://github.com/Layr-Labs/eigenda/blob/f8b0d31d65b29e60172507074922668f4ca89420/api/clients/codecs/default_blob_codec.go#L25 // raw blob the immediate data just before taking IFFT - raw_blob[1] = BLOB_ENCODING_VERSION; + raw_blob[1] = BLOB_ENCODING_VERSION_0; raw_blob[2..6].copy_from_slice(&rollup_data_len.to_be_bytes()); // encode length as uint32 diff --git a/crates/eigenda/src/constant.rs b/crates/eigenda/src/constant.rs index d2d8d8e..ca9fa5a 100644 --- a/crates/eigenda/src/constant.rs +++ b/crates/eigenda/src/constant.rs @@ -1,3 +1,3 @@ /// This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] /// followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] -pub const BLOB_ENCODING_VERSION: u8 = 0x0; +pub const BLOB_ENCODING_VERSION_0: u8 = 0x0; diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 501fb6c..24b06f2 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -31,4 +31,4 @@ mod certificate; pub use certificate::BlobInfo; mod constant; -pub use constant::BLOB_ENCODING_VERSION; +pub use constant::BLOB_ENCODING_VERSION_0; From 49a4c8070e010c0ad1f6074941c048f8d712e785 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 17:11:19 -0500 Subject: [PATCH 22/29] docs: update readme instructions to make it easier (and more reproducible) to spin up the op devnet (#21) --- README.md | 23 +++--------------- op-devnet.docker-compose.yml.patch | 39 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 op-devnet.docker-compose.yml.patch diff --git a/README.md b/README.md index d340b8f..2ee09cc 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,10 @@ Hokulea is a library to provide the altda providers for a derivation pipeline bu First start the devnet: ```bash -git clone https://github.com/ethereum-optimism/optimism.git -cd optimism -DEVNET_ALTDA=true GENERIC_ALTDA=true make devnet-up +git clone -b v1.10.0 https://github.com/ethereum-optimism/optimism.git +# this patches the optimism devnet to use the eigenda-proxy instead of their da-server +git patch optimism/ops-bedrock/docker-compose.yml < op-devnet.docker-compose.yml.patch +DEVNET_ALTDA=true GENERIC_ALTDA=true make -C ./optimism devnet-up ``` Then run hokulea: ```bash @@ -16,20 +17,4 @@ cd bin/client just run-client-native-against-devnet ``` -To use eigenda proxy within optimism devnet, modify ops-bedrock/docker-compose.yaml: -``` -da-server: - image: ghcr.io/layr-labs/eigenda-proxy:v1.6.1 - environment: - EIGENDA_PROXY_ADDR: 0.0.0.0 - EIGENDA_PROXY_PORT: 3100 - EIGENDA_PROXY_METRICS_ENABLED: true - EIGENDA_PROXY_METRICS_PORT: 7300 - EIGENDA_PROXY_MEMSTORE_ENABLED: true - EIGENDA_PROXY_MEMSTORE_EXPIRATION: 45m - EIGENDA_PROXY_MEMSTORE_PUT_LATENCY: 0s - EIGENDA_PROXY_MEMSTORE_GET_LATENCY: 0s - EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED: true -``` - ![](./hokulea.jpeg) \ No newline at end of file diff --git a/op-devnet.docker-compose.yml.patch b/op-devnet.docker-compose.yml.patch new file mode 100644 index 0000000..83bc7cb --- /dev/null +++ b/op-devnet.docker-compose.yml.patch @@ -0,0 +1,39 @@ +diff --git a/ops-bedrock/docker-compose.yml b/ops-bedrock/docker-compose.yml +index adcaea8f4..5c5e2e8ee 100644 +--- a/ops-bedrock/docker-compose.yml ++++ b/ops-bedrock/docker-compose.yml +@@ -240,22 +240,20 @@ services: + OP_CHALLENGER_NUM_CONFIRMATIONS: 1 + + da-server: +- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:devnet +- build: +- context: ../ +- dockerfile: ops/docker/op-stack-go/Dockerfile +- target: da-server-target +- command: > +- da-server +- --file.path=/data +- --addr=0.0.0.0 +- --port=3100 +- --log.level=debug +- --generic-commitment="${ALTDA_GENERIC_DA}" ++ image: ghcr.io/layr-labs/eigenda-proxy:v1.6.1 ++ environment: ++ EIGENDA_PROXY_ADDR: 0.0.0.0 ++ EIGENDA_PROXY_PORT: 3100 ++ EIGENDA_PROXY_METRICS_ENABLED: "true" ++ EIGENDA_PROXY_METRICS_PORT: 7300 ++ EIGENDA_PROXY_MEMSTORE_ENABLED: "true" ++ EIGENDA_PROXY_MEMSTORE_EXPIRATION: 45m ++ EIGENDA_PROXY_MEMSTORE_PUT_LATENCY: 0s ++ EIGENDA_PROXY_MEMSTORE_GET_LATENCY: 0s ++ EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED: "true" + ports: + - "3100:3100" +- volumes: +- - "da_data:/data" ++ - "6969:7300" + + sentinel: + image: quarry/sentinel # TODO(10141): We need a public image for this (sentinel is out of repo) From 7dd7a13fdb204091a05494224639dcc85f2d93ec Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 17:16:19 -0500 Subject: [PATCH 23/29] style: fix lint errors --- Cargo.lock | 1 - crates/eigenda/Cargo.toml | 1 - crates/eigenda/src/eigenda.rs | 2 +- crates/proof/src/eigenda_provider.rs | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c62a849..ff6e14d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1958,7 +1958,6 @@ dependencies = [ "kona-derive", "op-alloy-protocol", "rust-kzg-bn254", - "thiserror 2.0.7", "tracing", ] diff --git a/crates/eigenda/Cargo.toml b/crates/eigenda/Cargo.toml index c6ba39e..2ca678f 100644 --- a/crates/eigenda/Cargo.toml +++ b/crates/eigenda/Cargo.toml @@ -12,7 +12,6 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true tracing.workspace = true async-trait.workspace = true -thiserror.workspace = true bytes.workspace = true rust-kzg-bn254.workspace = true diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 56a6878..46cc9f5 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -73,7 +73,7 @@ where let l1_block_number = block_ref.number; // TODO: make it part of rollup config - let stale_gap = 100 as u64; + let stale_gap = 100_u64; // check staleness // TODO: this would require the op-rollup to follow the same pattern diff --git a/crates/proof/src/eigenda_provider.rs b/crates/proof/src/eigenda_provider.rs index f7af7ae..e94fff6 100644 --- a/crates/proof/src/eigenda_provider.rs +++ b/crates/proof/src/eigenda_provider.rs @@ -87,7 +87,7 @@ impl EigenDABlobProvider for OracleEigenDAProvider // if field element is 0, it means the host has identified that the data // has breached eigenda invariant, i.e cert is valid - if field_element.len() == 0 { + if field_element.is_empty() { return Err(OracleProviderError::Preimage(PreimageOracleError::Other( "field elememnt is empty, breached eigenda invariant".into(), ))); From c69886f10873ba2f4f0d787af746f394873c74d1 Mon Sep 17 00:00:00 2001 From: Samuel Laferriere Date: Thu, 2 Jan 2025 18:25:38 -0500 Subject: [PATCH 24/29] test: fix decode tests --- crates/eigenda/src/eigenda_data.rs | 38 +++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index cdd8951..96bcad5 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -43,32 +43,48 @@ impl EigenDABlobData { #[cfg(test)] mod tests { + use crate::BLOB_ENCODING_VERSION_0; + use super::*; use alloc::vec; use alloy_primitives::Bytes; + use kona_derive::errors::BlobDecodingError; + + fn generate_blob_data(content: &[u8]) -> EigenDABlobData { + let mut blob = vec![0; 32]; + blob[1] = BLOB_ENCODING_VERSION_0; + blob[2..6].copy_from_slice(&(content.len() as u32).to_be_bytes()); + blob.extend_from_slice(&helpers::convert_by_padding_empty_byte(content)); + EigenDABlobData { + blob: Bytes::from(blob), + } + } #[test] fn test_decode_success() { - let data = EigenDABlobData { - blob: Bytes::from(vec![1, 2, 3, 4]), - }; + let content = vec![1, 2, 3, 4]; + let data = generate_blob_data(&content); let result = data.decode(); assert!(result.is_ok()); - assert_eq!(result.unwrap(), Bytes::from(vec![1, 2, 3, 4])); + assert_eq!(result.unwrap(), Bytes::from(content)); } #[test] - fn test_decode_empty_blob() { - let data = EigenDABlobData { - blob: Bytes::from(vec![]), - }; + fn test_decode_success_empty() { + let content = vec![]; + let data = generate_blob_data(&content); let result = data.decode(); assert!(result.is_ok()); - assert_eq!(result.unwrap(), Bytes::from(vec![])); + assert_eq!(result.unwrap(), Bytes::from(content)); } #[test] - fn test_decode_invalid_blob() { - // TODO: implement this once decode actually does something + fn test_decode_error_invalid_length() { + let data = EigenDABlobData { + blob: Bytes::from(vec![0; 31]), // one byte short of having a full header + }; + let result = data.decode(); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), BlobDecodingError::InvalidLength); } } From 736de0b567295edc665e5e1c9979e41cc219da72 Mon Sep 17 00:00:00 2001 From: Bowen <71761807+scale-chain-sub@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:50:18 -0800 Subject: [PATCH 25/29] add length check in eigenda_data.rs --- crates/eigenda/src/eigenda_data.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/eigenda/src/eigenda_data.rs b/crates/eigenda/src/eigenda_data.rs index 96bcad5..d525fa1 100644 --- a/crates/eigenda/src/eigenda_data.rs +++ b/crates/eigenda/src/eigenda_data.rs @@ -34,6 +34,9 @@ impl EigenDABlobData { helpers::remove_empty_byte_from_padded_bytes_unchecked(codec_data.as_ref()); let blob_content: Bytes = blob_content.into(); + if blob_content.len() < content_size as usize { + return Err(BlobDecodingError::InvalidLength); + } // might insert a FFT here, // take data From 3f36ef8aa8a0a0f22f8993667c38becdad32fac6 Mon Sep 17 00:00:00 2001 From: Bowen <71761807+scale-chain-sub@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:04:52 -0800 Subject: [PATCH 26/29] fix review --- bin/host/src/eigenda_fetcher/mod.rs | 3 +++ crates/eigenda/src/constant.rs | 3 +++ crates/eigenda/src/eigenda.rs | 6 ++---- crates/eigenda/src/lib.rs | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/host/src/eigenda_fetcher/mod.rs b/bin/host/src/eigenda_fetcher/mod.rs index e3890b6..95e4810 100644 --- a/bin/host/src/eigenda_fetcher/mod.rs +++ b/bin/host/src/eigenda_fetcher/mod.rs @@ -184,6 +184,7 @@ where // The preimage oracle key for each field element is the keccak256 hash of // `abi.encodePacked(cert.KZGCommitment, uint256(i))` + // TODO figure out the key size, most likely dependent on smart contract parsing let mut blob_key = [0u8; 96]; blob_key[..32].copy_from_slice(cert_blob_info.blob_header.commitment.x.as_ref()); blob_key[32..64].copy_from_slice(cert_blob_info.blob_header.commitment.y.as_ref()); @@ -204,6 +205,8 @@ where )?; } + // TODO proof is at the random point, but we need to figure out where to generate + // // Write the KZG Proof as the last element, needed for ZK //blob_key[88..].copy_from_slice((blob_length).to_be_bytes().as_ref()); //let blob_key_hash = keccak256(blob_key.as_ref()); diff --git a/crates/eigenda/src/constant.rs b/crates/eigenda/src/constant.rs index ca9fa5a..211c159 100644 --- a/crates/eigenda/src/constant.rs +++ b/crates/eigenda/src/constant.rs @@ -1,3 +1,6 @@ /// This minimal blob encoding contains a 32 byte header = [0x00, version byte, uint32 len of data, 0x00, 0x00,...] /// followed by the encoded data [0x00, 31 bytes of data, 0x00, 31 bytes of data,...] pub const BLOB_ENCODING_VERSION_0: u8 = 0x0; +/// TODO: make it part of rollup config +pub const STALE_GAP: u64 = 100; + diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 46cc9f5..19a91a9 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -3,7 +3,7 @@ use crate::eigenda_blobs::EigenDABlobSource; use crate::traits::EigenDABlobProvider; -use crate::BlobInfo; +use crate::{BlobInfo, STALE_GAP}; use alloy_rlp::Decodable; use alloc::{boxed::Box, fmt::Debug}; @@ -72,14 +72,12 @@ where .reference_block_number as u64; let l1_block_number = block_ref.number; - // TODO: make it part of rollup config - let stale_gap = 100_u64; // check staleness // TODO: this would require the op-rollup to follow the same pattern // but passing blockId to proxy which implement the logic, // see https://github.com/ethereum-optimism/optimism/blob/0bb2ff57c8133f1e3983820c0bf238001eca119b/op-alt-da/damgr.go#L211 - if rbn + stale_gap < l1_block_number { + if rbn + STALE_GAP < l1_block_number { // TODO: double check return Err(PipelineErrorKind::Temporary(PipelineError::EndOfSource)); } diff --git a/crates/eigenda/src/lib.rs b/crates/eigenda/src/lib.rs index 24b06f2..1d02e90 100644 --- a/crates/eigenda/src/lib.rs +++ b/crates/eigenda/src/lib.rs @@ -32,3 +32,4 @@ pub use certificate::BlobInfo; mod constant; pub use constant::BLOB_ENCODING_VERSION_0; +pub use constant::STALE_GAP; From 21fed1a969178cbdeff460ea387a56a020fcb8d0 Mon Sep 17 00:00:00 2001 From: Bowen <71761807+scale-chain-sub@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:06:32 -0800 Subject: [PATCH 27/29] fix lint --- crates/eigenda/src/constant.rs | 1 - crates/eigenda/src/eigenda.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/crates/eigenda/src/constant.rs b/crates/eigenda/src/constant.rs index 211c159..8a5a5c6 100644 --- a/crates/eigenda/src/constant.rs +++ b/crates/eigenda/src/constant.rs @@ -3,4 +3,3 @@ pub const BLOB_ENCODING_VERSION_0: u8 = 0x0; /// TODO: make it part of rollup config pub const STALE_GAP: u64 = 100; - diff --git a/crates/eigenda/src/eigenda.rs b/crates/eigenda/src/eigenda.rs index 19a91a9..83448ef 100644 --- a/crates/eigenda/src/eigenda.rs +++ b/crates/eigenda/src/eigenda.rs @@ -72,7 +72,6 @@ where .reference_block_number as u64; let l1_block_number = block_ref.number; - // check staleness // TODO: this would require the op-rollup to follow the same pattern // but passing blockId to proxy which implement the logic, From f87fd3473715d197b7a4e59aee55d982d2f464b9 Mon Sep 17 00:00:00 2001 From: Bowen Date: Thu, 2 Jan 2025 17:49:06 -0800 Subject: [PATCH 28/29] test --- crates/eigenda/src/codec.rs | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 crates/eigenda/src/codec.rs diff --git a/crates/eigenda/src/codec.rs b/crates/eigenda/src/codec.rs new file mode 100644 index 0000000..1cf31c3 --- /dev/null +++ b/crates/eigenda/src/codec.rs @@ -0,0 +1,4 @@ +// +// +// +// From 66697395b62a143507ea9744c5490aee433155cd Mon Sep 17 00:00:00 2001 From: Bowen Date: Thu, 2 Jan 2025 17:52:51 -0800 Subject: [PATCH 29/29] Revert "test" This reverts commit f87fd3473715d197b7a4e59aee55d982d2f464b9. --- crates/eigenda/src/codec.rs | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 crates/eigenda/src/codec.rs diff --git a/crates/eigenda/src/codec.rs b/crates/eigenda/src/codec.rs deleted file mode 100644 index 1cf31c3..0000000 --- a/crates/eigenda/src/codec.rs +++ /dev/null @@ -1,4 +0,0 @@ -// -// -// -//