PeerDAS spec tests

consensus/types/src/data_column_subnet_id.rs

@@ -2,6 +2,7 @@
 use crate::data_column_sidecar::ColumnIndex;
 use crate::EthSpec;
 use ethereum_types::U256;
+use itertools::Itertools;
 use safe_arith::{ArithError, SafeArith};
 use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
@@ -60,15 +61,15 @@ impl DataColumnSubnetId {
         node_id: U256,
         custody_subnet_count: u64,
     ) -> impl Iterator<Item = DataColumnSubnetId> {
-        // NOTE: we could perform check on `custody_subnet_count` here to ensure that it is a valid
+        // TODO(das): we could perform check on `custody_subnet_count` here to ensure that it is a valid
         // value, but here we assume it is valid.
 
         let mut subnets = SmallVec::<[u64; 32]>::new();
-        let mut offset = 0;
+        let mut current_id = node_id;
         while (subnets.len() as u64) < custody_subnet_count {
-            let offset_node_id = node_id + U256::from(offset);
-            let offset_node_id = offset_node_id.low_u64().to_le_bytes();
-            let hash: [u8; 32] = ethereum_hashing::hash_fixed(&offset_node_id);
+            let mut node_id_bytes = [0u8; 32];
+            current_id.to_little_endian(&mut node_id_bytes);
+            let hash = ethereum_hashing::hash_fixed(&node_id_bytes);
             let hash_prefix = [
                 hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7],
             ];
@@ -79,7 +80,10 @@ impl DataColumnSubnetId {
                 subnets.push(subnet);
             }
 
-            offset += 1
+            if current_id == U256::MAX {
+                current_id = U256::zero()
+            }
+            current_id += U256::one()
         }
         subnets.into_iter().map(DataColumnSubnetId::new)
     }
@@ -90,6 +94,7 @@ impl DataColumnSubnetId {
     ) -> impl Iterator<Item = ColumnIndex> {
         Self::compute_custody_subnets::<E>(node_id, custody_subnet_count)
             .flat_map(|subnet| subnet.columns::<E>())
+            .sorted()
     }
 }
 

testing/ef_tests/check_all_files_accessed.py

@@ -31,6 +31,10 @@
     "tests/.*/.*/ssz_static/LightClientStore",
     # LightClientSnapshot
     "tests/.*/.*/ssz_static/LightClientSnapshot",
+    # Unused kzg methods
+    "tests/.*/.*/kzg/compute_cells",
+    "tests/.*/.*/kzg/recover_all_cells",
+    "tests/.*/.*/kzg/verify_cell_kzg_batch",
     # One of the EF researchers likes to pack the tarballs on a Mac
     ".*\.DS_Store.*",
     # More Mac weirdness.

testing/ef_tests/src/cases.rs

@@ -1,6 +1,6 @@
 use super::*;
 use rayon::prelude::*;
-use std::fmt::Debug;
+use std::fmt::{Debug, Display, Formatter};
 use std::path::{Path, PathBuf};
 use types::ForkName;
 
@@ -18,11 +18,14 @@ mod fork;
 mod fork_choice;
 mod genesis_initialization;
 mod genesis_validity;
+mod get_custody_columns;
 mod kzg_blob_to_kzg_commitment;
 mod kzg_compute_blob_kzg_proof;
+mod kzg_compute_cells_and_kzg_proofs;
 mod kzg_compute_kzg_proof;
 mod kzg_verify_blob_kzg_proof;
 mod kzg_verify_blob_kzg_proof_batch;
+mod kzg_verify_cell_kzg_proof_batch;
 mod kzg_verify_kzg_proof;
 mod merkle_proof_validity;
 mod operations;
@@ -48,11 +51,14 @@ pub use epoch_processing::*;
 pub use fork::ForkTest;
 pub use genesis_initialization::*;
 pub use genesis_validity::*;
+pub use get_custody_columns::*;
 pub use kzg_blob_to_kzg_commitment::*;
 pub use kzg_compute_blob_kzg_proof::*;
+pub use kzg_compute_cells_and_kzg_proofs::*;
 pub use kzg_compute_kzg_proof::*;
 pub use kzg_verify_blob_kzg_proof::*;
 pub use kzg_verify_blob_kzg_proof_batch::*;
+pub use kzg_verify_cell_kzg_proof_batch::*;
 pub use kzg_verify_kzg_proof::*;
 pub use merkle_proof_validity::*;
 pub use operations::*;
@@ -64,6 +70,17 @@ pub use ssz_generic::*;
 pub use ssz_static::*;
 pub use transition::TransitionTest;
 
+#[derive(Debug, PartialEq)]
+pub enum FeatureName {
+    Eip7594,
+}
+
+impl Display for FeatureName {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str("eip7594")
+    }
+}
+
 pub trait LoadCase: Sized {
     /// Load the test case from a test case directory.
     fn load_from_dir(_path: &Path, _fork_name: ForkName) -> Result<Self, Error>;
@@ -84,6 +101,13 @@ pub trait Case: Debug + Sync {
         true
     }
 
+    /// Whether or not this test exists for the given `feature_name`.
+    ///
+    /// Returns `true` by default.
+    fn is_enabled_for_feature(_feature_name: FeatureName) -> bool {
+        true
+    }
+
     /// Execute a test and return the result.
     ///
     /// `case_index` reports the index of the case in the set of test cases. It is not strictly

testing/ef_tests/src/cases/get_custody_columns.rs

@@ -0,0 +1,39 @@
+use super::*;
+use ethereum_types::U256;
+use serde::Deserialize;
+use std::marker::PhantomData;
+use types::DataColumnSubnetId;
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec", deny_unknown_fields)]
+pub struct GetCustodyColumns<E: EthSpec> {
+    pub node_id: String,
+    pub custody_subnet_count: u64,
+    pub result: Vec<u64>,
+    #[serde(skip)]
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> LoadCase for GetCustodyColumns<E> {
+    fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
+        decode::yaml_decode_file(path.join("meta.yaml").as_path())
+    }
+}
+
+impl<E: EthSpec> Case for GetCustodyColumns<E> {
+    fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
+        let node_id = U256::from_dec_str(&self.node_id)
+            .map_err(|e| Error::FailedToParseTest(format!("{e:?}")))?;
+        let computed =
+            DataColumnSubnetId::compute_custody_columns::<E>(node_id, self.custody_subnet_count)
+                .collect::<Vec<_>>();
+        let expected = &self.result;
+        if computed == *expected {
+            Ok(())
+        } else {
+            Err(Error::NotEqual(format!(
+                "Got {computed:?}\nExpected {expected:?}"
+            )))
+        }
+    }
+}

testing/ef_tests/src/cases/kzg_blob_to_kzg_commitment.rs

@@ -31,6 +31,10 @@ impl<E: EthSpec> Case for KZGBlobToKZGCommitment<E> {
         fork_name == ForkName::Deneb
     }
 
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name != FeatureName::Eip7594
+    }
+
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let kzg = get_kzg()?;
 

testing/ef_tests/src/cases/kzg_compute_blob_kzg_proof.rs

@@ -32,6 +32,10 @@ impl<E: EthSpec> Case for KZGComputeBlobKZGProof<E> {
         fork_name == ForkName::Deneb
     }
 
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name != FeatureName::Eip7594
+    }
+
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGComputeBlobKZGProofInput| -> Result<_, Error> {
             let blob = parse_blob::<E>(&input.blob)?;

testing/ef_tests/src/cases/kzg_compute_cells_and_kzg_proofs.rs

@@ -0,0 +1,69 @@
+use super::*;
+use crate::case_result::compare_result;
+use kzg::KzgProof;
+use kzg::{Blob as KzgBlob, Cell};
+use serde::Deserialize;
+use std::marker::PhantomData;
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KZGComputeCellsAndKzgProofsInput {
+    pub blob: String,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(bound = "E: EthSpec", deny_unknown_fields)]
+pub struct KZGComputeCellsAndKZGProofs<E: EthSpec> {
+    pub input: KZGComputeCellsAndKzgProofsInput,
+    pub output: Option<(Vec<String>, Vec<String>)>,
+    #[serde(skip)]
+    _phantom: PhantomData<E>,
+}
+
+impl<E: EthSpec> LoadCase for KZGComputeCellsAndKZGProofs<E> {
+    fn load_from_dir(path: &Path, _fork_name: ForkName) -> Result<Self, Error> {
+        decode::yaml_decode_file(path.join("data.yaml").as_path())
+    }
+}
+
+impl<E: EthSpec> Case for KZGComputeCellsAndKZGProofs<E> {
+    fn is_enabled_for_fork(fork_name: ForkName) -> bool {
+        fork_name == ForkName::Deneb
+    }
+
+    fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
+        // TODO(das): lazy init Kzg as a static ref
+        let kzg = get_kzg()?;
+        let cells_and_proofs = parse_blob::<E>(&self.input.blob).and_then(|blob| {
+            let blob = KzgBlob::from_bytes(&blob).map_err(|e| {
+                Error::InternalError(format!("Failed to convert blob to kzg blob: {e:?}"))
+            })?;
+            kzg.compute_cells_and_proofs(&blob).map_err(|e| {
+                Error::InternalError(format!("Failed to compute cells and kzg proofs: {e:?}"))
+            })
+        });
+
+        let expected = self.output.as_ref().and_then(|(cells, proofs)| {
+            parse_cells_and_proofs::<E>(cells, proofs)
+                .map(|(cells, proofs)| {
+                    (
+                        cells
+                            .try_into()
+                            .map_err(|e| {
+                                Error::FailedToParseTest(format!("Failed to parse cells: {e:?}"))
+                            })
+                            .unwrap(),
+                        proofs
+                            .try_into()
+                            .map_err(|e| {
+                                Error::FailedToParseTest(format!("Failed to parse proofs: {e:?}"))
+                            })
+                            .unwrap(),
+                    )
+                })
+                .ok()
+        });
+
+        compare_result::<(Box<[Cell; 128]>, Box<[KzgProof; 128]>), _>(&cells_and_proofs, &expected)
+    }
+}

testing/ef_tests/src/cases/kzg_compute_kzg_proof.rs

@@ -39,6 +39,10 @@ impl<E: EthSpec> Case for KZGComputeKZGProof<E> {
         fork_name == ForkName::Deneb
     }
 
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name != FeatureName::Eip7594
+    }
+
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGComputeKZGProofInput| -> Result<_, Error> {
             let blob = parse_blob::<E>(&input.blob)?;

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof.rs

@@ -2,7 +2,7 @@ use super::*;
 use crate::case_result::compare_result;
 use beacon_chain::kzg_utils::validate_blob;
 use eth2_network_config::TRUSTED_SETUP_BYTES;
-use kzg::{Error as KzgError, Kzg, KzgCommitment, KzgProof, TrustedSetup};
+use kzg::{Cell, Error as KzgError, Kzg, KzgCommitment, KzgProof, TrustedSetup};
 use serde::Deserialize;
 use std::marker::PhantomData;
 use types::Blob;
@@ -14,6 +14,33 @@ pub fn get_kzg() -> Result<Kzg, Error> {
         .map_err(|e| Error::InternalError(format!("Failed to initialize kzg: {:?}", e)))
 }
 
+// TODO(das) `EthSpec` can be removed once KZG lib exposes the `BytesPerCell` constant.
+pub fn parse_cells_and_proofs<E: EthSpec>(
+    cells: &Vec<String>,
+    proofs: &Vec<String>,
+) -> Result<(Vec<Cell>, Vec<KzgProof>), Error> {
+    let cells = cells
+        .iter()
+        .map(|s| parse_cell::<E>(s.as_str()))
+        .collect::<Result<Vec<_>, Error>>()?;
+
+    let proofs = proofs
+        .iter()
+        .map(|s| parse_proof(s.as_str()))
+        .collect::<Result<Vec<_>, Error>>()?;
+
+    Ok((cells, proofs))
+}
+
+pub fn parse_cell<E: EthSpec>(cell: &str) -> Result<Cell, Error> {
+    hex::decode(strip_0x(cell)?)
+        .map_err(|e| Error::FailedToParseTest(format!("Failed to parse cell: {:?}", e)))
+        .and_then(|bytes| {
+            Cell::from_bytes(bytes.as_ref())
+                .map_err(|e| Error::FailedToParseTest(format!("Failed to parse proof: {:?}", e)))
+        })
+}
+
 pub fn parse_proof(proof: &str) -> Result<KzgProof, Error> {
     hex::decode(strip_0x(proof)?)
         .map_err(|e| Error::FailedToParseTest(format!("Failed to parse proof: {:?}", e)))
@@ -80,6 +107,10 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProof<E> {
         fork_name == ForkName::Deneb
     }
 
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name != FeatureName::Eip7594
+    }
+
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGVerifyBlobKZGProofInput| -> Result<(Blob<E>, KzgCommitment, KzgProof), Error> {
             let blob = parse_blob::<E>(&input.blob)?;

testing/ef_tests/src/cases/kzg_verify_blob_kzg_proof_batch.rs

@@ -33,6 +33,10 @@ impl<E: EthSpec> Case for KZGVerifyBlobKZGProofBatch<E> {
         fork_name == ForkName::Deneb
     }
 
+    fn is_enabled_for_feature(feature_name: FeatureName) -> bool {
+        feature_name != FeatureName::Eip7594
+    }
+
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
         let parse_input = |input: &KZGVerifyBlobKZGProofBatchInput| -> Result<_, Error> {
             let blobs = input