From 2b455c17f695bfbf83a8667f38ab9143b960ebeb Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 23 May 2024 12:15:35 +0200 Subject: [PATCH 01/24] create fork-choice.md with availability checks --- specs/_features/eip7594/fork-choice.md | 164 +++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 specs/_features/eip7594/fork-choice.md diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md new file mode 100644 index 0000000000..15acf0d137 --- /dev/null +++ b/specs/_features/eip7594/fork-choice.md @@ -0,0 +1,164 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) +- [Containers](#containers) +- [Helpers](#helpers) + - [Extended `PayloadAttributes`](#extended-payloadattributes) + - [`is_data_available`](#is_data_available) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594 + +### Helpers + +### `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + # Unimplemented function which returns the node_id and custody_subnet_count + node_id, custody_subnet_count = get_custody_parameters() + columns_to_retrieve = get_custody_columns(node_id, custody_subnet_count) + if require_peer_sampling: + columns_to_retrieve += get_sampling_columns() + column_sidecars = retrieve_column_sidecars(beacon_block_root, columns_to_retrieve) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +### `is_chain_available` + +```python +def is_chain_available(beacon_block_root: Root) -> bool: + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS <= current_epoch + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(parent_root) + ) + +``` + +### `get_head` + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + require_peer_sampling = compute_epoch_at_slot(slot) + 2 <= current_epoch + # Get available children for the current slot + children = [ + root for root in blocks.keys() + if ( + blocks[root].parent_root == head + and is_data_available(root, require_peer_sampling) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, block_root, pulled_up_state) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + update_unrealized_checkpoints(store, pulled_up_state.current_justified_checkpoint, pulled_up_state.finalized_checkpoint) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, pulled_up_state.current_justified_checkpoint, pulled_up_state.finalized_checkpoint) +``` \ No newline at end of file From 7c72ec851d982bd27951f453e2f7e82117285e4d Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 23 May 2024 18:26:12 +0200 Subject: [PATCH 02/24] validator custody and parameter changes --- specs/_features/eip7594/das-core.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index dc50365b1e..21a7271e3a 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -73,15 +73,18 @@ We define the following Python custom types for type hinting and readability: | Name | Value | Description | | - | - | - | -| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `32` | The number of data column sidecar subnets used in the gossipsub protocol | +| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `128` | The number of data column sidecar subnets used in the gossipsub protocol | ### Custody setting | Name | Value | Description | | - | - | - | -| `SAMPLES_PER_SLOT` | `8` | Number of `DataColumn` random samples a node queries per slot | -| `CUSTODY_REQUIREMENT` | `1` | Minimum number of subnets an honest node custodies and serves samples from | -| `TARGET_NUMBER_OF_PEERS` | `70` | Suggested minimum peer count | +| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | +| `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | +| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | +| `BALANCE_PER_CUSTODY_SUBNET` | `Gwei(32*10**9)` | Balance increment corresponding to one additional subnet to custody | +| `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | + ### Containers @@ -201,7 +204,15 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock, ### Custody requirement -Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). +Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached downloads and custodies a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` subnets per slot, as well as `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` additional subnets, where `total_node_balance` is the sum of the balances of all validators attached to that node. + +```python +def get_validators_custody_requirement(state: BeaconState, validator_indices: List[ValidatorIndex]) -> uint64: + total_node_balance = sum(state.balances[index] for index in validator_indices) + return VALIDATOR_CUSTODY_REQUIREMENT + (total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET) +``` + +The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_subnet_count: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node). From 9159bd0836f554e6c19c3a1070facc2b397b8e63 Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 24 May 2024 12:19:25 +0200 Subject: [PATCH 03/24] small fixes --- configs/mainnet.yaml | 10 +++++---- configs/minimal.yaml | 10 +++++---- specs/_features/eip7594/das-core.md | 4 ++-- specs/_features/eip7594/fork-choice.md | 29 +++++++++++++++++--------- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 7051873ce9..0cd3b4c660 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -157,11 +157,13 @@ WHISK_PROPOSER_SELECTION_GAP: 2 # EIP7594 NUMBER_OF_COLUMNS: 128 MAX_CELLS_IN_EXTENDED_MATRIX: 768 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 -SAMPLES_PER_SLOT: 8 -CUSTODY_REQUIREMENT: 1 -TARGET_NUMBER_OF_PEERS: 70 +SAMPLES_PER_SLOT: 16 +CUSTODY_REQUIREMENT: 4 +VALIDATOR_CUSTODY_REQUIREMENT: 8 +BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 32000000000 # 2**5 * 10**9 (= 32,000,000,000) +TARGET_NUMBER_OF_PEERS: 100 # [New in Electra:EIP7251] MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 # 2**7 * 10**9 (= 128,000,000,000) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 8e2a222d47..21ad6497df 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -156,11 +156,13 @@ WHISK_PROPOSER_SELECTION_GAP: 1 # EIP7594 NUMBER_OF_COLUMNS: 128 MAX_CELLS_IN_EXTENDED_MATRIX: 768 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 -SAMPLES_PER_SLOT: 8 -CUSTODY_REQUIREMENT: 1 -TARGET_NUMBER_OF_PEERS: 70 +SAMPLES_PER_SLOT: 16 +CUSTODY_REQUIREMENT: 4 +VALIDATOR_CUSTODY_REQUIREMENT: 8 +BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 32000000000 # 2**5 * 10**9 (= 32,000,000,000) +TARGET_NUMBER_OF_PEERS: 100 # [New in Electra:EIP7251] MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 # 2**6 * 10**9 (= 64,000,000,000) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 21a7271e3a..0431fbec3c 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -82,7 +82,7 @@ We define the following Python custom types for type hinting and readability: | `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | | `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | | `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | -| `BALANCE_PER_CUSTODY_SUBNET` | `Gwei(32*10**9)` | Balance increment corresponding to one additional subnet to custody | +| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | | `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | @@ -209,7 +209,7 @@ Each node *without attached validators* downloads and custodies a minimum of `CU ```python def get_validators_custody_requirement(state: BeaconState, validator_indices: List[ValidatorIndex]) -> uint64: total_node_balance = sum(state.balances[index] for index in validator_indices) - return VALIDATOR_CUSTODY_REQUIREMENT + (total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET) + return VALIDATOR_CUSTODY_REQUIREMENT + (total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET) ``` The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 15acf0d137..ba3c36d1ea 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -41,16 +41,16 @@ def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False ### `is_chain_available` ```python -def is_chain_available(beacon_block_root: Root) -> bool: +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: block = store.blocks[beacon_block_root] block_epoch = compute_epoch_at_slot(block.slot) current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS <= current_epoch + if block_epoch + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS <= current_epoch: return True parent_root = block.parent_root return ( is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(parent_root) + and is_chain_available(store, parent_root) ) ``` @@ -64,13 +64,15 @@ def get_head(store: Store) -> Root: # Execute the LMD-GHOST fork choice head = store.justified_checkpoint.root while True: - require_peer_sampling = compute_epoch_at_slot(slot) + 2 <= current_epoch # Get available children for the current slot children = [ - root for root in blocks.keys() + root for (root, block) in blocks.items() if ( - blocks[root].parent_root == head - and is_data_available(root, require_peer_sampling) + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) ) ] if len(children) == 0: @@ -80,6 +82,11 @@ def get_head(store: Store) -> Root: head = max(children, key=lambda root: (get_weight(store, root), root)) ``` +```python +def is_peer_sampling_required(store, slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_epoch(store) +``` + ## Updated fork-choice handlers ### `on_block` @@ -119,7 +126,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available pulled_up_state = state.copy() process_justification_and_finalization(pulled_up_state) - assert is_chain_available(pulled_up_state.current_justified_checkpoint.root) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) # Add new block to the store store.blocks[block_root] = block @@ -154,11 +161,13 @@ The application of `processing_justification_and_finalization` now happens in `o ```python def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - update_unrealized_checkpoints(store, pulled_up_state.current_justified_checkpoint, pulled_up_state.finalized_checkpoint) + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) # If the block is from a prior epoch, apply the realized values block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) current_epoch = get_current_store_epoch(store) if block_epoch < current_epoch: - update_checkpoints(store, pulled_up_state.current_justified_checkpoint, pulled_up_state.finalized_checkpoint) + update_checkpoints(store, unrealized_justified, unrealized_finalized) ``` \ No newline at end of file From 56e9d3844eaf981bb89c8453853de7c529ff2931 Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 24 May 2024 14:27:04 +0200 Subject: [PATCH 04/24] change validator custody to 6, plus two extra per 32 ETH --- configs/mainnet.yaml | 4 ++-- configs/minimal.yaml | 4 ++-- specs/_features/eip7594/das-core.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 0cd3b4c660..3afd2f6ae0 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -161,8 +161,8 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 16 CUSTODY_REQUIREMENT: 4 -VALIDATOR_CUSTODY_REQUIREMENT: 8 -BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 32000000000 # 2**5 * 10**9 (= 32,000,000,000) +VALIDATOR_CUSTODY_REQUIREMENT: 6 +BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 16000000000 # 2**4 * 10**9 (= 16,000,000,000) TARGET_NUMBER_OF_PEERS: 100 # [New in Electra:EIP7251] diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 21ad6497df..a836a8519e 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -160,8 +160,8 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 16 CUSTODY_REQUIREMENT: 4 -VALIDATOR_CUSTODY_REQUIREMENT: 8 -BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 32000000000 # 2**5 * 10**9 (= 32,000,000,000) +VALIDATOR_CUSTODY_REQUIREMENT: 6 +BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 16000000000 # 2**4 * 10**9 (= 16,000,000,000) TARGET_NUMBER_OF_PEERS: 100 # [New in Electra:EIP7251] diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 0431fbec3c..fb1b9f8fce 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -81,8 +81,8 @@ We define the following Python custom types for type hinting and readability: | - | - | - | | `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | | `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | -| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | -| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | +| `VALIDATOR_CUSTODY_REQUIREMENT` | `6` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | +| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(16 * 10**9)` | Balance increment corresponding to one additional subnet to custody | | `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | From 54157a23e259c190a01c005e2570d19ec8012324 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 25 Jun 2024 19:23:24 +0800 Subject: [PATCH 05/24] wip --- pysetup/spec_builders/eip7594.py | 19 ++++++++++++++ specs/_features/eip7594/das-core.md | 2 +- specs/_features/eip7594/fork-choice.md | 36 +++++++++++++++++++++----- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index 3329378320..d392617838 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -12,6 +12,23 @@ def imports(cls, preset_name: str): return f''' from eth2spec.deneb import {preset_name} as deneb ''' + + + @classmethod + def sundry_functions(cls) -> str: + return """ +def get_custody_parameters() -> Tuple[NodeID, uint64]: + return NodeID(1), uint64(1) + + +def get_sampling_columns() -> Sequence[ColumnIndex]: + return [ColumnIndex(1), ColumnIndex(2)] + + +def retrieve_column_sidecars(beacon_block_root: Root, + columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]: + return [DataColumnSidecar()] +""" @classmethod def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: @@ -27,3 +44,5 @@ def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: return { 'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value, } + + diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index fb1b9f8fce..eccb4f7c90 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -207,7 +207,7 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock, Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached downloads and custodies a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` subnets per slot, as well as `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` additional subnets, where `total_node_balance` is the sum of the balances of all validators attached to that node. ```python -def get_validators_custody_requirement(state: BeaconState, validator_indices: List[ValidatorIndex]) -> uint64: +def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: total_node_balance = sum(state.balances[index] for index in validator_indices) return VALIDATOR_CUSTODY_REQUIREMENT + (total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET) ``` diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index ba3c36d1ea..420eeedc6c 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -6,12 +6,18 @@ - [Introduction](#introduction) -- [Containers](#containers) -- [Helpers](#helpers) - - [Extended `PayloadAttributes`](#extended-payloadattributes) - - [`is_data_available`](#is_data_available) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) @@ -22,7 +28,21 @@ This is the modification of the fork choice accompanying EIP-7594 ### Helpers -### `is_data_available` +#### `get_custody_parameters` + +`def get_custody_parameters() -> Tuple[NodeID, uint64]` + +#### `get_sampling_columns` + + +`def get_sampling_columns() -> Sequence[ColumnIndex]` + + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### `is_data_available` ```python def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: @@ -38,7 +58,7 @@ def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False ) ``` -### `is_chain_available` +#### `is_chain_available` ```python def is_chain_available(store: Store, beacon_block_root: Root) -> bool: @@ -55,7 +75,7 @@ def is_chain_available(store: Store, beacon_block_root: Root) -> bool: ``` -### `get_head` +#### `get_head` ```python def get_head(store: Store) -> Root: @@ -82,6 +102,8 @@ def get_head(store: Store) -> Root: head = max(children, key=lambda root: (get_weight(store, root), root)) ``` +#### `is_peer_sampling_required` + ```python def is_peer_sampling_required(store, slot): return compute_epoch_at_slot(slot) + 2 <= get_current_epoch(store) From f98241b212077d4c390ea77066ffe0a08970d572 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 09:24:51 +0200 Subject: [PATCH 06/24] simplify is_data_available + small fixes --- pysetup/spec_builders/eip7594.py | 12 ++---------- specs/_features/eip7594/fork-choice.md | 15 +++++---------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index d392617838..a1e7251693 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -17,17 +17,9 @@ def imports(cls, preset_name: str): @classmethod def sundry_functions(cls) -> str: return """ -def get_custody_parameters() -> Tuple[NodeID, uint64]: - return NodeID(1), uint64(1) - - -def get_sampling_columns() -> Sequence[ColumnIndex]: - return [ColumnIndex(1), ColumnIndex(2)] - - def retrieve_column_sidecars(beacon_block_root: Root, - columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]: - return [DataColumnSidecar()] + require_peer_sampling: bool) -> Sequence[DataColumnSidecar]: + return [] """ @classmethod diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 420eeedc6c..48b38d5d78 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -46,12 +46,7 @@ This is the modification of the fork choice accompanying EIP-7594 ```python def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - # Unimplemented function which returns the node_id and custody_subnet_count - node_id, custody_subnet_count = get_custody_parameters() - columns_to_retrieve = get_custody_columns(node_id, custody_subnet_count) - if require_peer_sampling: - columns_to_retrieve += get_sampling_columns() - column_sidecars = retrieve_column_sidecars(beacon_block_root, columns_to_retrieve) + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) return all( verify_data_column_sidecar_kzg_proofs(column_sidecar) for column_sidecar in column_sidecars @@ -65,7 +60,7 @@ def is_chain_available(store: Store, beacon_block_root: Root) -> bool: block = store.blocks[beacon_block_root] block_epoch = compute_epoch_at_slot(block.slot) current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS <= current_epoch: + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: return True parent_root = block.parent_root return ( @@ -105,8 +100,8 @@ def get_head(store: Store) -> Root: #### `is_peer_sampling_required` ```python -def is_peer_sampling_required(store, slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_epoch(store) +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) ``` ## Updated fork-choice handlers @@ -170,7 +165,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, block_root, pulled_up_state) + compute_pulled_up_tip(store, pulled_up_state, block_root) ``` #### Pull-up tip helpers From e4e30f35f241cac6eba02175ffe86a8fcbb0bdf5 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 09:47:03 +0200 Subject: [PATCH 07/24] only run deneb on_block tests with deneb phase --- .../test/deneb/fork_choice/test_on_block.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index 4ab7d819a9..0be9b86322 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -2,9 +2,12 @@ from eth2spec.test.context import ( spec_state_test, - with_deneb_and_later, + with_phases([DENEB]), + with_phases, ) +from eth2spec.test.helpers.constants import DENEB + from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) @@ -34,7 +37,7 @@ def get_block_with_blob(spec, state, rng=None): return block, blobs, blob_kzg_proofs -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_simple_blob_data(spec, state): rng = Random(1234) @@ -69,7 +72,7 @@ def test_simple_blob_data(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_incorrect_proof(spec, state): rng = Random(1234) @@ -97,7 +100,7 @@ def test_invalid_incorrect_proof(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_data_unavailable(spec, state): rng = Random(1234) @@ -125,7 +128,7 @@ def test_invalid_data_unavailable(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_wrong_proofs_length(spec, state): rng = Random(1234) @@ -153,7 +156,7 @@ def test_invalid_wrong_proofs_length(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_wrong_blobs_length(spec, state): rng = Random(1234) From 6723efd65b8e02941b6baa3440b79d4a6f9b9e65 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 09:47:33 +0200 Subject: [PATCH 08/24] handle is_chain_available in tests --- .../eth2spec/test/helpers/fork_choice.py | 22 +++++++++++++++++++ .../pyspec/eth2spec/test/helpers/forks.py | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 094e2e8a5c..2b9f0e6eb0 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -8,6 +8,7 @@ state_transition_with_full_block, ) +from eth2spec.test.helpers.forks import is_post_eip_7594 class BlobData(NamedTuple): """ @@ -151,6 +152,27 @@ def on_tick_and_append_step(spec, store, time, test_steps): def run_on_block(spec, store, signed_block, valid=True): + if is_post_eip_7594(spec): + def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: + if beacon_block_root == b'\x00' * 32: + # anchor block has current_justified_checkpoint root 0x0 + return True + block = store.blocks[beacon_block_root] + block_epoch = spec.compute_epoch_at_slot(block.slot) + current_epoch = spec.get_current_store_epoch(store) + if block_epoch + spec.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + spec.is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) + spec.is_chain_available = is_chain_available + else: + def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: + return True + spec.is_chain_available = is_chain_available + if not valid: try: spec.on_block(store, signed_block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index 5ea03c31b3..4c5375c147 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -1,6 +1,6 @@ from .constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, - ELECTRA, WHISK, + ELECTRA, WHISK, EIP7594, PREVIOUS_FORK_OF, ) @@ -44,6 +44,9 @@ def is_post_electra(spec): def is_post_whisk(spec): return is_post_fork(spec.fork, WHISK) +def is_post_eip_7594(spec): + return is_post_fork(spec.fork, EIP7594) + def get_spec_for_fork_version(spec, fork_version, phases): if phases is None: From 40c55e6ae16cc4eb646d8f198033f3e30263577f Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 11:16:32 +0200 Subject: [PATCH 09/24] typo fix --- .../core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index 0be9b86322..148e56ec44 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -2,7 +2,6 @@ from eth2spec.test.context import ( spec_state_test, - with_phases([DENEB]), with_phases, ) From fcca8fd5f22b7aa1bb1d89b61ffee9c86df41122 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 11:57:05 +0200 Subject: [PATCH 10/24] fix lint --- tests/core/pyspec/eth2spec/test/helpers/fork_choice.py | 9 +++++---- tests/core/pyspec/eth2spec/test/helpers/forks.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 2b9f0e6eb0..8d2a210901 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -10,6 +10,7 @@ from eth2spec.test.helpers.forks import is_post_eip_7594 + class BlobData(NamedTuple): """ The return values of ``retrieve_blobs_and_proofs`` helper. @@ -153,7 +154,7 @@ def on_tick_and_append_step(spec, store, time, test_steps): def run_on_block(spec, store, signed_block, valid=True): if is_post_eip_7594(spec): - def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: + def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: if beacon_block_root == b'\x00' * 32: # anchor block has current_justified_checkpoint root 0x0 return True @@ -164,12 +165,12 @@ def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: return True parent_root = block.parent_root return ( - spec.is_data_available(beacon_block_root, require_peer_sampling=True) + spec.is_data_available(beacon_block_root, require_peer_sampling=True) and is_chain_available(store, parent_root) ) spec.is_chain_available = is_chain_available - else: - def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: + else: + def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: return True spec.is_chain_available = is_chain_available diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index 4c5375c147..c3d1eee8d5 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -44,6 +44,7 @@ def is_post_electra(spec): def is_post_whisk(spec): return is_post_fork(spec.fork, WHISK) + def is_post_eip_7594(spec): return is_post_fork(spec.fork, EIP7594) From 107dda69b6476f21b9467e9564b1229d31b887bf Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 14:02:40 +0200 Subject: [PATCH 11/24] deal with the genesis edge case directly in the fork_choice spec --- specs/_features/eip7594/fork-choice.md | 4 +++- .../eth2spec/test/helpers/fork_choice.py | 23 ------------------- .../pyspec/eth2spec/test/helpers/forks.py | 6 +---- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 48b38d5d78..d20f8a33b8 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -143,7 +143,9 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available pulled_up_state = state.copy() process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + # Do not make the check in the Genesis edge case, where current_justified_checkpoint is not set + if pulled_up_state.current_justified_checkpoint.root in store.blocks: + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) # Add new block to the store store.blocks[block_root] = block diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 8d2a210901..094e2e8a5c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -8,8 +8,6 @@ state_transition_with_full_block, ) -from eth2spec.test.helpers.forks import is_post_eip_7594 - class BlobData(NamedTuple): """ @@ -153,27 +151,6 @@ def on_tick_and_append_step(spec, store, time, test_steps): def run_on_block(spec, store, signed_block, valid=True): - if is_post_eip_7594(spec): - def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: - if beacon_block_root == b'\x00' * 32: - # anchor block has current_justified_checkpoint root 0x0 - return True - block = store.blocks[beacon_block_root] - block_epoch = spec.compute_epoch_at_slot(block.slot) - current_epoch = spec.get_current_store_epoch(store) - if block_epoch + spec.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - spec.is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) - spec.is_chain_available = is_chain_available - else: - def is_chain_available(store: spec.Store, beacon_block_root: spec.Root) -> bool: - return True - spec.is_chain_available = is_chain_available - if not valid: try: spec.on_block(store, signed_block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index c3d1eee8d5..5ea03c31b3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -1,6 +1,6 @@ from .constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, - ELECTRA, WHISK, EIP7594, + ELECTRA, WHISK, PREVIOUS_FORK_OF, ) @@ -45,10 +45,6 @@ def is_post_whisk(spec): return is_post_fork(spec.fork, WHISK) -def is_post_eip_7594(spec): - return is_post_fork(spec.fork, EIP7594) - - def get_spec_for_fork_version(spec, fork_version, phases): if phases is None: return spec From e9398f6c8227cc25d82d4d5df2b7479841fd3109 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 14:31:07 +0200 Subject: [PATCH 12/24] move genesis edge case check in is_chain_available --- specs/_features/eip7594/fork-choice.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index d20f8a33b8..5849e2b61f 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -57,6 +57,9 @@ def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False ```python def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + if beacon_block_root not in store.blocks: + # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set + return True block = store.blocks[beacon_block_root] block_epoch = compute_epoch_at_slot(block.slot) current_epoch = get_current_store_epoch(store) @@ -143,9 +146,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available pulled_up_state = state.copy() process_justification_and_finalization(pulled_up_state) - # Do not make the check in the Genesis edge case, where current_justified_checkpoint is not set - if pulled_up_state.current_justified_checkpoint.root in store.blocks: - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) # Add new block to the store store.blocks[block_root] = block From 22456c8aa4c5e15efaec343de0cb7432db4a7ab0 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 16:27:27 +0200 Subject: [PATCH 13/24] change validator custody according to Justin's suggestion Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- configs/mainnet.yaml | 4 ++-- configs/minimal.yaml | 4 ++-- specs/_features/eip7594/das-core.md | 11 ++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 3afd2f6ae0..0cd3b4c660 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -161,8 +161,8 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 16 CUSTODY_REQUIREMENT: 4 -VALIDATOR_CUSTODY_REQUIREMENT: 6 -BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 16000000000 # 2**4 * 10**9 (= 16,000,000,000) +VALIDATOR_CUSTODY_REQUIREMENT: 8 +BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 32000000000 # 2**5 * 10**9 (= 32,000,000,000) TARGET_NUMBER_OF_PEERS: 100 # [New in Electra:EIP7251] diff --git a/configs/minimal.yaml b/configs/minimal.yaml index a836a8519e..21ad6497df 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -160,8 +160,8 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 16 CUSTODY_REQUIREMENT: 4 -VALIDATOR_CUSTODY_REQUIREMENT: 6 -BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 16000000000 # 2**4 * 10**9 (= 16,000,000,000) +VALIDATOR_CUSTODY_REQUIREMENT: 8 +BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET: 32000000000 # 2**5 * 10**9 (= 32,000,000,000) TARGET_NUMBER_OF_PEERS: 100 # [New in Electra:EIP7251] diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index eccb4f7c90..6b18e4c366 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -81,8 +81,8 @@ We define the following Python custom types for type hinting and readability: | - | - | - | | `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | | `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | -| `VALIDATOR_CUSTODY_REQUIREMENT` | `6` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | -| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(16 * 10**9)` | Balance increment corresponding to one additional subnet to custody | +| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | +| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | | `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | @@ -204,12 +204,13 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock, ### Custody requirement -Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached downloads and custodies a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` subnets per slot, as well as `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` additional subnets, where `total_node_balance` is the sum of the balances of all validators attached to that node. +Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached, and with the sum of the balances of all attached validators being `total_node_balance`, downloads and custodies `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` subnets per slot, with a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` and of course a maximum of `DATA_COLUMN_SIDECAR_SUBNET_COUNT`. ```python -def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: +def get_validators_custody_requirement(state: BeaconState, validator_indices: List[ValidatorIndex]) -> uint64: total_node_balance = sum(state.balances[index] for index in validator_indices) - return VALIDATOR_CUSTODY_REQUIREMENT + (total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET) + count = total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET + return min(max(count, VALIDATOR_CUSTODY_REQUIREMENT), DATA_COLUMN_SIDECAR_SUBNET_COUNT) ``` The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). From 0f1e471b4d4d1ff2fcc0aff3b4c7c9734fdd08a8 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 27 Jun 2024 16:35:00 +0200 Subject: [PATCH 14/24] fix typo --- specs/_features/eip7594/das-core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 6b18e4c366..5d03878f76 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -207,7 +207,7 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock, Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached, and with the sum of the balances of all attached validators being `total_node_balance`, downloads and custodies `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` subnets per slot, with a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` and of course a maximum of `DATA_COLUMN_SIDECAR_SUBNET_COUNT`. ```python -def get_validators_custody_requirement(state: BeaconState, validator_indices: List[ValidatorIndex]) -> uint64: +def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: total_node_balance = sum(state.balances[index] for index in validator_indices) count = total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET return min(max(count, VALIDATOR_CUSTODY_REQUIREMENT), DATA_COLUMN_SIDECAR_SUBNET_COUNT) From ef6492471f848b07e8faa0ef6aa7ddcd3aec8a49 Mon Sep 17 00:00:00 2001 From: fradamt <104826920+fradamt@users.noreply.github.com> Date: Tue, 9 Jul 2024 16:30:57 +0200 Subject: [PATCH 15/24] remove unused helpers --- specs/_features/eip7594/fork-choice.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 5849e2b61f..e09058a5f0 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -28,16 +28,6 @@ This is the modification of the fork choice accompanying EIP-7594 ### Helpers -#### `get_custody_parameters` - -`def get_custody_parameters() -> Tuple[NodeID, uint64]` - -#### `get_sampling_columns` - - -`def get_sampling_columns() -> Sequence[ColumnIndex]` - - #### `retrieve_column_sidecars` `def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` @@ -190,4 +180,4 @@ def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root current_epoch = get_current_store_epoch(store) if block_epoch < current_epoch: update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` \ No newline at end of file +``` From 82103dd337a9632e91c17ceda10b0da4120faba4 Mon Sep 17 00:00:00 2001 From: fradamt <104826920+fradamt@users.noreply.github.com> Date: Wed, 10 Jul 2024 17:51:04 +0200 Subject: [PATCH 16/24] Apply suggestions from code review Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/fork-choice.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index e09058a5f0..c1346d518f 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -24,7 +24,7 @@ ## Introduction -This is the modification of the fork choice accompanying EIP-7594 +This is the modification of the fork choice accompanying EIP-7594. ### Helpers @@ -32,7 +32,7 @@ This is the modification of the fork choice accompanying EIP-7594 `def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` -#### `is_data_available` +#### Modified `is_data_available` ```python def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: @@ -43,7 +43,7 @@ def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False ) ``` -#### `is_chain_available` +#### New `is_chain_available` ```python def is_chain_available(store: Store, beacon_block_root: Root) -> bool: @@ -90,7 +90,7 @@ def get_head(store: Store) -> Root: head = max(children, key=lambda root: (get_weight(store, root), root)) ``` -#### `is_peer_sampling_required` +#### New `is_peer_sampling_required` ```python def is_peer_sampling_required(store: Store, slot: Slot): From 4f3c150e9b1a5afa1f08da57450ae4ec80212e8d Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 11 Jul 2024 15:59:41 +0200 Subject: [PATCH 17/24] apply suggestions from code review Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- .../spec_builders/eip7594_20240626155910.py | 40 +++ .../spec_builders/eip7594_20240711153253.py | 39 +++ .../eip7594/das-core_20240627163448.md | 309 ++++++++++++++++++ .../eip7594/das-core_20240711153213.md | 308 +++++++++++++++++ .../eip7594/fork-choice_20240711152138.md | 183 +++++++++++ .../eip7594/fork-choice_20240711152321.md | 183 +++++++++++ .../eip7594/fork-choice_20240711153141.md | 186 +++++++++++ .../eip7594/fork-choice_20240711153757.md | 190 +++++++++++ .../eip7594/fork-choice_20240711155539.md | 195 +++++++++++ .../eip7594/fork-choice_20240711155548.md | 194 +++++++++++ .../eip7594/fork-choice_20240711155631.md | 190 +++++++++++ pysetup/spec_builders/eip7594.py | 1 - specs/_features/eip7594/das-core.md | 1 - specs/_features/eip7594/fork-choice.md | 21 +- 14 files changed, 2031 insertions(+), 9 deletions(-) create mode 100644 .history/pysetup/spec_builders/eip7594_20240626155910.py create mode 100644 .history/pysetup/spec_builders/eip7594_20240711153253.py create mode 100644 .history/specs/_features/eip7594/das-core_20240627163448.md create mode 100644 .history/specs/_features/eip7594/das-core_20240711153213.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711152138.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711152321.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711153141.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711153757.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711155539.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711155548.md create mode 100644 .history/specs/_features/eip7594/fork-choice_20240711155631.md diff --git a/.history/pysetup/spec_builders/eip7594_20240626155910.py b/.history/pysetup/spec_builders/eip7594_20240626155910.py new file mode 100644 index 0000000000..a1e7251693 --- /dev/null +++ b/.history/pysetup/spec_builders/eip7594_20240626155910.py @@ -0,0 +1,40 @@ +from typing import Dict + +from .base import BaseSpecBuilder +from ..constants import EIP7594 + + +class EIP7594SpecBuilder(BaseSpecBuilder): + fork: str = EIP7594 + + @classmethod + def imports(cls, preset_name: str): + return f''' +from eth2spec.deneb import {preset_name} as deneb +''' + + + @classmethod + def sundry_functions(cls) -> str: + return """ +def retrieve_column_sidecars(beacon_block_root: Root, + require_peer_sampling: bool) -> Sequence[DataColumnSidecar]: + return [] +""" + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + return { + 'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value, + 'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value, + 'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value, + 'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value, + } + + @classmethod + def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: + return { + 'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value, + } + + diff --git a/.history/pysetup/spec_builders/eip7594_20240711153253.py b/.history/pysetup/spec_builders/eip7594_20240711153253.py new file mode 100644 index 0000000000..3c13d0ea43 --- /dev/null +++ b/.history/pysetup/spec_builders/eip7594_20240711153253.py @@ -0,0 +1,39 @@ +from typing import Dict + +from .base import BaseSpecBuilder +from ..constants import EIP7594 + + +class EIP7594SpecBuilder(BaseSpecBuilder): + fork: str = EIP7594 + + @classmethod + def imports(cls, preset_name: str): + return f''' +from eth2spec.deneb import {preset_name} as deneb +''' + + + @classmethod + def sundry_functions(cls) -> str: + return """ +def retrieve_column_sidecars(beacon_block_root: Root, + require_peer_sampling: bool) -> Sequence[DataColumnSidecar]: + return [] +""" + + @classmethod + def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: + return { + 'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value, + 'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value, + 'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value, + 'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value, + } + + @classmethod + def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: + return { + 'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value, + } + diff --git a/.history/specs/_features/eip7594/das-core_20240627163448.md b/.history/specs/_features/eip7594/das-core_20240627163448.md new file mode 100644 index 0000000000..5d03878f76 --- /dev/null +++ b/.history/specs/_features/eip7594/das-core_20240627163448.md @@ -0,0 +1,309 @@ +# EIP-7594 -- Data Availability Sampling Core + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Constants](#constants) + - [Misc](#misc) +- [Custom types](#custom-types) +- [Configuration](#configuration) + - [Data size](#data-size) + - [Networking](#networking) + - [Custody setting](#custody-setting) + - [Containers](#containers) + - [`DataColumnSidecar`](#datacolumnsidecar) + - [Helper functions](#helper-functions) + - [`get_custody_columns`](#get_custody_columns) + - [`compute_extended_matrix`](#compute_extended_matrix) + - [`recover_matrix`](#recover_matrix) + - [`get_data_column_sidecars`](#get_data_column_sidecars) +- [Custody](#custody) + - [Custody requirement](#custody-requirement) + - [Public, deterministic selection](#public-deterministic-selection) +- [Peer discovery](#peer-discovery) +- [Extended data](#extended-data) +- [Column gossip](#column-gossip) + - [Parameters](#parameters) +- [Peer sampling](#peer-sampling) +- [Peer scoring](#peer-scoring) +- [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding) +- [DAS providers](#das-providers) +- [A note on fork choice](#a-note-on-fork-choice) +- [FAQs](#faqs) + - [Row (blob) custody](#row-blob-custody) + - [Subnet stability](#subnet-stability) + + + + +## Constants + +The following values are (non-configurable) constants used throughout the specification. + +### Misc + +| Name | Value | +| - | - | +| `UINT256_MAX` | `uint256(2**256 - 1)` | + +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `DataColumn` | `List[Cell, MAX_BLOB_COMMITMENTS_PER_BLOCK]` | The data of each column in EIP-7594 | +| `ExtendedMatrix` | `List[Cell, MAX_CELLS_IN_EXTENDED_MATRIX]` | The full data of one-dimensional erasure coding extended blobs (in row major format). | + +## Configuration + +### Data size + +| Name | Value | Description | +| - | - | - | +| `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix. | +| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix`. | + +### Networking + +| Name | Value | Description | +| - | - | - | +| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `128` | The number of data column sidecar subnets used in the gossipsub protocol | + +### Custody setting + +| Name | Value | Description | +| - | - | - | +| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | +| `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | +| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | +| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | +| `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | + + +### Containers + +#### `DataColumnSidecar` + +```python +class DataColumnSidecar(Container): + index: ColumnIndex # Index of column in extended matrix + column: DataColumn + kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] + kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK] + signed_block_header: SignedBeaconBlockHeader + kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH] +``` + +### Helper functions + +#### `get_custody_columns` + +```python +def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: + assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT + + subnet_ids: List[uint64] = [] + current_id = uint256(node_id) + while len(subnet_ids) < custody_subnet_count: + subnet_id = ( + bytes_to_uint64(hash(uint_to_bytes(uint256(current_id)))[0:8]) + % DATA_COLUMN_SIDECAR_SUBNET_COUNT + ) + if subnet_id not in subnet_ids: + subnet_ids.append(subnet_id) + if current_id == UINT256_MAX: + # Overflow prevention + current_id = NodeID(0) + current_id += 1 + + assert len(subnet_ids) == len(set(subnet_ids)) + + columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT + return sorted([ + ColumnIndex(DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnet_id) + for i in range(columns_per_subnet) + for subnet_id in subnet_ids + ]) +``` + +#### `compute_extended_matrix` + +```python +def compute_extended_matrix(blobs: Sequence[Blob]) -> ExtendedMatrix: + """ + Return the full ``ExtendedMatrix``. + + This helper demonstrates the relationship between blobs and ``ExtendedMatrix``. + The data structure for storing cells is implementation-dependent. + """ + extended_matrix = [] + for blob in blobs: + extended_matrix.extend(compute_cells(blob)) + return ExtendedMatrix(extended_matrix) +``` + +#### `recover_matrix` + +```python +def recover_matrix(cells_dict: Dict[Tuple[BlobIndex, CellID], Cell], blob_count: uint64) -> ExtendedMatrix: + """ + Return the recovered ``ExtendedMatrix``. + + This helper demonstrates how to apply ``recover_all_cells``. + The data structure for storing cells is implementation-dependent. + """ + extended_matrix: List[Cell] = [] + for blob_index in range(blob_count): + cell_ids = [cell_id for b_index, cell_id in cells_dict.keys() if b_index == blob_index] + cells = [cells_dict[(BlobIndex(blob_index), cell_id)] for cell_id in cell_ids] + + all_cells_for_row = recover_all_cells(cell_ids, cells) + extended_matrix.extend(all_cells_for_row) + return ExtendedMatrix(extended_matrix) +``` + +#### `get_data_column_sidecars` + +```python +def get_data_column_sidecars(signed_block: SignedBeaconBlock, + blobs: Sequence[Blob]) -> Sequence[DataColumnSidecar]: + signed_block_header = compute_signed_block_header(signed_block) + block = signed_block.message + kzg_commitments_inclusion_proof = compute_merkle_proof( + block.body, + get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'), + ) + cells_and_proofs = [compute_cells_and_kzg_proofs(blob) for blob in blobs] + blob_count = len(blobs) + cells = [cells_and_proofs[i][0] for i in range(blob_count)] + proofs = [cells_and_proofs[i][1] for i in range(blob_count)] + sidecars = [] + for column_index in range(NUMBER_OF_COLUMNS): + column = DataColumn([cells[row_index][column_index] + for row_index in range(blob_count)]) + kzg_proof_of_column = [proofs[row_index][column_index] + for row_index in range(blob_count)] + sidecars.append(DataColumnSidecar( + index=column_index, + column=column, + kzg_commitments=block.body.blob_kzg_commitments, + kzg_proofs=kzg_proof_of_column, + signed_block_header=signed_block_header, + kzg_commitments_inclusion_proof=kzg_commitments_inclusion_proof, + )) + return sidecars +``` + +## Custody + +### Custody requirement + +Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached, and with the sum of the balances of all attached validators being `total_node_balance`, downloads and custodies `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` subnets per slot, with a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` and of course a maximum of `DATA_COLUMN_SIDECAR_SUBNET_COUNT`. + +```python +def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: + total_node_balance = sum(state.balances[index] for index in validator_indices) + count = total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET + return min(max(count, VALIDATOR_CUSTODY_REQUIREMENT), DATA_COLUMN_SIDECAR_SUBNET_COUNT) +``` + +The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). + +A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_subnet_count: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node). + +A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns. + +### Public, deterministic selection + +The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_columns`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public. + +*Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody. + +## Peer discovery + +At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands. + +A node runs a background peer discovery process, maintaining at least `TARGET_NUMBER_OF_PEERS` of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. + +`TARGET_NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling. + +*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes. + +*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked. + +## Extended data + +In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension. `compute_extended_matrix` demonstrates the relationship between blobs and custom type `ExtendedMatrix`. + +## Column gossip + +### Parameters + +For each column -- use `data_column_sidecar_{subnet_id}` subnets, where `subnet_id` can be computed with the `compute_subnet_for_data_column_sidecar(column_index: ColumnIndex)` helper. The sidecars can be computed with the `get_data_column_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob])` helper. + +To custody a particular column, a node joins the respective gossip subnet. Verifiable samples from their respective column are gossiped on the assigned subnet. + +## Peer sampling + +A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. At each slot, a node makes `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success. + +## Peer scoring + +Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer. + +## Reconstruction and cross-seeding + +If the node obtains 50%+ of all the columns, they can reconstruct the full data matrix via `recover_matrix` helper. + +If a node fails to sample a peer or fails to get a column on the column subnet, a node can utilize the Req/Resp message to query the missing column from other peers. + +Once the node obtain the column, the node SHOULD send the missing columns to the column subnets. + +*Note*: A node always maintains a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction. + +*Note*: There are timing considerations to analyze -- at what point does a node consider samples missing and choose to reconstruct and cross-seed. + +*Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms. + +## DAS providers + +A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery. + +DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service. + +Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms. + +## A note on fork choice + +*Fork choice spec TBD, but it will just be a replacement of `is_data_available()` call in Deneb with column sampling instead of full download. Note the `is_data_available(slot_N)` will likely do a `-1` follow distance so that you just need to check the availability of slot `N-1` for slot `N` (starting with the block proposer of `N`).* + +The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of a particular design impacting it. + +In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc. + +For example, the fork choice rule might require validators to do successful DAS on slot `N` to be able to include block of slot `N` in its fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that they don't hurt the aggregate security. For example, the rule could be — DAS must be completed for slot N-1 for a child block in N to be included in the fork choice. + +Such trailing techniques and their analysis will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules. + +## FAQs + +### Row (blob) custody + +In the one-dimension construction, a node samples the peers by requesting the whole `DataColumn`. In reconstruction, a node can reconstruct all the blobs by 50% of the columns. Note that nodes can still download the row via `blob_sidecar_{subnet_id}` subnets. + +The potential benefits of having row custody could include: + +1. Allow for more "natural" distribution of data to consumers -- e.g., roll-ups -- but honestly, they won't know a priori which row their blob is going to be included in in the block, so they would either need to listen to all rows or download a particular row after seeing the block. The former looks just like listening to column [0, N) and the latter is req/resp instead of gossiping. +2. Help with some sort of distributed reconstruction. Those with full rows can compute extensions and seed missing samples to the network. This would either need to be able to send individual points on the gossip or would need some sort of req/resp faculty, potentially similar to an `IHAVEPOINTBITFIELD` and `IWANTSAMPLE`. + +However, for simplicity, we don't assign row custody assignments to nodes in the current design. + +### Subnet stability + +To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_columns`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs. diff --git a/.history/specs/_features/eip7594/das-core_20240711153213.md b/.history/specs/_features/eip7594/das-core_20240711153213.md new file mode 100644 index 0000000000..fbde23fd1a --- /dev/null +++ b/.history/specs/_features/eip7594/das-core_20240711153213.md @@ -0,0 +1,308 @@ +# EIP-7594 -- Data Availability Sampling Core + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Constants](#constants) + - [Misc](#misc) +- [Custom types](#custom-types) +- [Configuration](#configuration) + - [Data size](#data-size) + - [Networking](#networking) + - [Custody setting](#custody-setting) + - [Containers](#containers) + - [`DataColumnSidecar`](#datacolumnsidecar) + - [Helper functions](#helper-functions) + - [`get_custody_columns`](#get_custody_columns) + - [`compute_extended_matrix`](#compute_extended_matrix) + - [`recover_matrix`](#recover_matrix) + - [`get_data_column_sidecars`](#get_data_column_sidecars) +- [Custody](#custody) + - [Custody requirement](#custody-requirement) + - [Public, deterministic selection](#public-deterministic-selection) +- [Peer discovery](#peer-discovery) +- [Extended data](#extended-data) +- [Column gossip](#column-gossip) + - [Parameters](#parameters) +- [Peer sampling](#peer-sampling) +- [Peer scoring](#peer-scoring) +- [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding) +- [DAS providers](#das-providers) +- [A note on fork choice](#a-note-on-fork-choice) +- [FAQs](#faqs) + - [Row (blob) custody](#row-blob-custody) + - [Subnet stability](#subnet-stability) + + + + +## Constants + +The following values are (non-configurable) constants used throughout the specification. + +### Misc + +| Name | Value | +| - | - | +| `UINT256_MAX` | `uint256(2**256 - 1)` | + +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `DataColumn` | `List[Cell, MAX_BLOB_COMMITMENTS_PER_BLOCK]` | The data of each column in EIP-7594 | +| `ExtendedMatrix` | `List[Cell, MAX_CELLS_IN_EXTENDED_MATRIX]` | The full data of one-dimensional erasure coding extended blobs (in row major format). | + +## Configuration + +### Data size + +| Name | Value | Description | +| - | - | - | +| `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix. | +| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix`. | + +### Networking + +| Name | Value | Description | +| - | - | - | +| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `128` | The number of data column sidecar subnets used in the gossipsub protocol | + +### Custody setting + +| Name | Value | Description | +| - | - | - | +| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | +| `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | +| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | +| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | +| `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | + +### Containers + +#### `DataColumnSidecar` + +```python +class DataColumnSidecar(Container): + index: ColumnIndex # Index of column in extended matrix + column: DataColumn + kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] + kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK] + signed_block_header: SignedBeaconBlockHeader + kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH] +``` + +### Helper functions + +#### `get_custody_columns` + +```python +def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: + assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT + + subnet_ids: List[uint64] = [] + current_id = uint256(node_id) + while len(subnet_ids) < custody_subnet_count: + subnet_id = ( + bytes_to_uint64(hash(uint_to_bytes(uint256(current_id)))[0:8]) + % DATA_COLUMN_SIDECAR_SUBNET_COUNT + ) + if subnet_id not in subnet_ids: + subnet_ids.append(subnet_id) + if current_id == UINT256_MAX: + # Overflow prevention + current_id = NodeID(0) + current_id += 1 + + assert len(subnet_ids) == len(set(subnet_ids)) + + columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT + return sorted([ + ColumnIndex(DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnet_id) + for i in range(columns_per_subnet) + for subnet_id in subnet_ids + ]) +``` + +#### `compute_extended_matrix` + +```python +def compute_extended_matrix(blobs: Sequence[Blob]) -> ExtendedMatrix: + """ + Return the full ``ExtendedMatrix``. + + This helper demonstrates the relationship between blobs and ``ExtendedMatrix``. + The data structure for storing cells is implementation-dependent. + """ + extended_matrix = [] + for blob in blobs: + extended_matrix.extend(compute_cells(blob)) + return ExtendedMatrix(extended_matrix) +``` + +#### `recover_matrix` + +```python +def recover_matrix(cells_dict: Dict[Tuple[BlobIndex, CellID], Cell], blob_count: uint64) -> ExtendedMatrix: + """ + Return the recovered ``ExtendedMatrix``. + + This helper demonstrates how to apply ``recover_all_cells``. + The data structure for storing cells is implementation-dependent. + """ + extended_matrix: List[Cell] = [] + for blob_index in range(blob_count): + cell_ids = [cell_id for b_index, cell_id in cells_dict.keys() if b_index == blob_index] + cells = [cells_dict[(BlobIndex(blob_index), cell_id)] for cell_id in cell_ids] + + all_cells_for_row = recover_all_cells(cell_ids, cells) + extended_matrix.extend(all_cells_for_row) + return ExtendedMatrix(extended_matrix) +``` + +#### `get_data_column_sidecars` + +```python +def get_data_column_sidecars(signed_block: SignedBeaconBlock, + blobs: Sequence[Blob]) -> Sequence[DataColumnSidecar]: + signed_block_header = compute_signed_block_header(signed_block) + block = signed_block.message + kzg_commitments_inclusion_proof = compute_merkle_proof( + block.body, + get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'), + ) + cells_and_proofs = [compute_cells_and_kzg_proofs(blob) for blob in blobs] + blob_count = len(blobs) + cells = [cells_and_proofs[i][0] for i in range(blob_count)] + proofs = [cells_and_proofs[i][1] for i in range(blob_count)] + sidecars = [] + for column_index in range(NUMBER_OF_COLUMNS): + column = DataColumn([cells[row_index][column_index] + for row_index in range(blob_count)]) + kzg_proof_of_column = [proofs[row_index][column_index] + for row_index in range(blob_count)] + sidecars.append(DataColumnSidecar( + index=column_index, + column=column, + kzg_commitments=block.body.blob_kzg_commitments, + kzg_proofs=kzg_proof_of_column, + signed_block_header=signed_block_header, + kzg_commitments_inclusion_proof=kzg_commitments_inclusion_proof, + )) + return sidecars +``` + +## Custody + +### Custody requirement + +Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached, and with the sum of the balances of all attached validators being `total_node_balance`, downloads and custodies `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` subnets per slot, with a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` and of course a maximum of `DATA_COLUMN_SIDECAR_SUBNET_COUNT`. + +```python +def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: + total_node_balance = sum(state.balances[index] for index in validator_indices) + count = total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET + return min(max(count, VALIDATOR_CUSTODY_REQUIREMENT), DATA_COLUMN_SIDECAR_SUBNET_COUNT) +``` + +The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). + +A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_subnet_count: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node). + +A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns. + +### Public, deterministic selection + +The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_columns`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public. + +*Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody. + +## Peer discovery + +At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands. + +A node runs a background peer discovery process, maintaining at least `TARGET_NUMBER_OF_PEERS` of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. + +`TARGET_NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling. + +*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes. + +*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked. + +## Extended data + +In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension. `compute_extended_matrix` demonstrates the relationship between blobs and custom type `ExtendedMatrix`. + +## Column gossip + +### Parameters + +For each column -- use `data_column_sidecar_{subnet_id}` subnets, where `subnet_id` can be computed with the `compute_subnet_for_data_column_sidecar(column_index: ColumnIndex)` helper. The sidecars can be computed with the `get_data_column_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob])` helper. + +To custody a particular column, a node joins the respective gossip subnet. Verifiable samples from their respective column are gossiped on the assigned subnet. + +## Peer sampling + +A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. At each slot, a node makes `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success. + +## Peer scoring + +Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer. + +## Reconstruction and cross-seeding + +If the node obtains 50%+ of all the columns, they can reconstruct the full data matrix via `recover_matrix` helper. + +If a node fails to sample a peer or fails to get a column on the column subnet, a node can utilize the Req/Resp message to query the missing column from other peers. + +Once the node obtain the column, the node SHOULD send the missing columns to the column subnets. + +*Note*: A node always maintains a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction. + +*Note*: There are timing considerations to analyze -- at what point does a node consider samples missing and choose to reconstruct and cross-seed. + +*Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms. + +## DAS providers + +A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery. + +DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service. + +Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms. + +## A note on fork choice + +*Fork choice spec TBD, but it will just be a replacement of `is_data_available()` call in Deneb with column sampling instead of full download. Note the `is_data_available(slot_N)` will likely do a `-1` follow distance so that you just need to check the availability of slot `N-1` for slot `N` (starting with the block proposer of `N`).* + +The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of a particular design impacting it. + +In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc. + +For example, the fork choice rule might require validators to do successful DAS on slot `N` to be able to include block of slot `N` in its fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that they don't hurt the aggregate security. For example, the rule could be — DAS must be completed for slot N-1 for a child block in N to be included in the fork choice. + +Such trailing techniques and their analysis will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules. + +## FAQs + +### Row (blob) custody + +In the one-dimension construction, a node samples the peers by requesting the whole `DataColumn`. In reconstruction, a node can reconstruct all the blobs by 50% of the columns. Note that nodes can still download the row via `blob_sidecar_{subnet_id}` subnets. + +The potential benefits of having row custody could include: + +1. Allow for more "natural" distribution of data to consumers -- e.g., roll-ups -- but honestly, they won't know a priori which row their blob is going to be included in in the block, so they would either need to listen to all rows or download a particular row after seeing the block. The former looks just like listening to column [0, N) and the latter is req/resp instead of gossiping. +2. Help with some sort of distributed reconstruction. Those with full rows can compute extensions and seed missing samples to the network. This would either need to be able to send individual points on the gossip or would need some sort of req/resp faculty, potentially similar to an `IHAVEPOINTBITFIELD` and `IWANTSAMPLE`. + +However, for simplicity, we don't assign row custody assignments to nodes in the current design. + +### Subnet stability + +To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_columns`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs. diff --git a/.history/specs/_features/eip7594/fork-choice_20240711152138.md b/.history/specs/_features/eip7594/fork-choice_20240711152138.md new file mode 100644 index 0000000000..c1346d518f --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711152138.md @@ -0,0 +1,183 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + if beacon_block_root not in store.blocks: + # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) + +``` + +#### `get_head` + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711152321.md b/.history/specs/_features/eip7594/fork-choice_20240711152321.md new file mode 100644 index 0000000000..32cac43fc4 --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711152321.md @@ -0,0 +1,183 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + if beacon_block_root not in store.blocks: + # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) + +``` + +#### Modified `get_head` + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711153141.md b/.history/specs/_features/eip7594/fork-choice_20240711153141.md new file mode 100644 index 0000000000..9956f64b5b --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711153141.md @@ -0,0 +1,186 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + if beacon_block_root not in store.blocks: + # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) +``` + +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +For blocks from the current or previous epoch (which cannot yet be finalized), this is established through +a custody check, while for blocks older than two epochs through a full peer sampling check. + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711153757.md b/.history/specs/_features/eip7594/fork-choice_20240711153757.md new file mode 100644 index 0000000000..55bc645cbd --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711153757.md @@ -0,0 +1,190 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + """ + Checks if all ancestors of `beacon_block_root` within the custody period are + available, as determined by `is_data_available` with peer sampling enabled + """ + if beacon_block_root not in store.blocks: + # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) +``` + +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +For blocks from the current or previous epoch (which cannot yet be finalized), this is established through +a custody check, while for blocks older than two epochs through a full peer sampling check. + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711155539.md b/.history/specs/_features/eip7594/fork-choice_20240711155539.md new file mode 100644 index 0000000000..c6bb7f401d --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711155539.md @@ -0,0 +1,195 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never +results in adding to `store` a block with an unavailable ancestor, because blocks are not added to +`store` unless their whole ancestry has already has been. This situation could only present itself +around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + """ + Checks if all ancestors of `beacon_block_root` within the custody period are + available, as determined by `is_data_available` with peer sampling enabled + """ + if beacon_block_root not in store.blocks: + # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) +``` + +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +For blocks from the current or previous epoch (which cannot yet be finalized), this is established through +a custody check, while for blocks older than two epochs through a full peer sampling check. + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711155548.md b/.history/specs/_features/eip7594/fork-choice_20240711155548.md new file mode 100644 index 0000000000..0230b43cdf --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711155548.md @@ -0,0 +1,194 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### `retrieve_column_sidecars` + +`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never +results in adding to `store` a block with an unavailable ancestor, because blocks are not added to +`store` unless their whole ancestry has already has been. This situation could only present itself +around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + """ + Checks if all ancestors of `beacon_block_root` within the custody period are + available, as determined by `is_data_available` with peer sampling enabled + """ + if beacon_block_root not in store.blocks: + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) +``` + +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +For blocks from the current or previous epoch (which cannot yet be finalized), this is established through +a custody check, while for blocks older than two epochs through a full peer sampling check. + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711155631.md b/.history/specs/_features/eip7594/fork-choice_20240711155631.md new file mode 100644 index 0000000000..1f0ec0b232 --- /dev/null +++ b/.history/specs/_features/eip7594/fork-choice_20240711155631.md @@ -0,0 +1,190 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [`get_custody_parameters`](#get_custody_parameters) + - [`get_sampling_columns`](#get_sampling_columns) + - [`retrieve_column_sidecars`](#retrieve_column_sidecars) + - [`is_data_available`](#is_data_available) + - [`is_chain_available`](#is_chain_available) + - [`get_head`](#get_head) + - [`is_peer_sampling_required`](#is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### New `is_chain_available` + +*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never +results in adding to `store` a block with an unavailable ancestor, because blocks are not added to +`store` unless their whole ancestry has already has been. This situation could only present itself +around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. + +```python +def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + """ + Checks if all ancestors of `beacon_block_root` within the custody period are + available, as determined by `is_data_available` with peer sampling enabled + """ + if beacon_block_root not in store.blocks: + return True + block = store.blocks[beacon_block_root] + block_epoch = compute_epoch_at_slot(block.slot) + current_epoch = get_current_store_epoch(store) + if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: + return True + parent_root = block.parent_root + return ( + is_data_available(beacon_block_root, require_peer_sampling=True) + and is_chain_available(store, parent_root) + ) +``` + +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +For blocks from the current or previous epoch (which cannot yet be finalized), this is established through +a custody check, while for blocks older than two epochs through a full peer sampling check. + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available( + root, + require_peer_sampling=is_peer_sampling_required(store, block.slot) + ) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +#### New `is_peer_sampling_required` + +```python +def is_peer_sampling_required(store: Store, slot: Slot): + return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) +``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The blob data availability check is removed and replaced with an availability +check on the on the justified checkpoint in the "pulled up state" of the block, after +applying `process_justification_and_finalization`. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available + pulled_up_state = state.copy() + process_justification_and_finalization(pulled_up_state) + assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, pulled_up_state, block_root) +``` + +#### Pull-up tip helpers + +##### `compute_pulled_up_tip` + +Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +The application of `processing_justification_and_finalization` now happens in `on_block`. + +```python +def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: + store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint + unrealized_justified = pulled_up_state.current_justified_checkpoint + unrealized_finalized = pulled_up_state.finalized_checkpoint + update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) + + # If the block is from a prior epoch, apply the realized values + block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) + current_epoch = get_current_store_epoch(store) + if block_epoch < current_epoch: + update_checkpoints(store, unrealized_justified, unrealized_finalized) +``` diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index a1e7251693..3c13d0ea43 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -37,4 +37,3 @@ def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: 'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value, } - diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 5d03878f76..fbde23fd1a 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -85,7 +85,6 @@ We define the following Python custom types for type hinting and readability: | `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | | `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | - ### Containers #### `DataColumnSidecar` diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index c1346d518f..1f0ec0b232 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -28,10 +28,6 @@ This is the modification of the fork choice accompanying EIP-7594. ### Helpers -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - #### Modified `is_data_available` ```python @@ -45,10 +41,18 @@ def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False #### New `is_chain_available` +*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never +results in adding to `store` a block with an unavailable ancestor, because blocks are not added to +`store` unless their whole ancestry has already has been. This situation could only present itself +around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. + ```python def is_chain_available(store: Store, beacon_block_root: Root) -> bool: + """ + Checks if all ancestors of `beacon_block_root` within the custody period are + available, as determined by `is_data_available` with peer sampling enabled + """ if beacon_block_root not in store.blocks: - # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set return True block = store.blocks[beacon_block_root] block_epoch = compute_epoch_at_slot(block.slot) @@ -60,10 +64,13 @@ def is_chain_available(store: Store, beacon_block_root: Root) -> bool: is_data_available(beacon_block_root, require_peer_sampling=True) and is_chain_available(store, parent_root) ) - ``` -#### `get_head` +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +For blocks from the current or previous epoch (which cannot yet be finalized), this is established through +a custody check, while for blocks older than two epochs through a full peer sampling check. ```python def get_head(store: Store) -> Root: From 908f37db703a3a0fa5e0a3718bce54907f4a9f30 Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 11 Jul 2024 16:05:35 +0200 Subject: [PATCH 18/24] doctoc --- specs/_features/eip7594/fork-choice.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 1f0ec0b232..028db64786 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -7,13 +7,10 @@ - [Introduction](#introduction) - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) + - [Modified `is_data_available`](#modified-is_data_available) + - [New `is_chain_available`](#new-is_chain_available) + - [Modified `get_head`](#modified-get_head) + - [New `is_peer_sampling_required`](#new-is_peer_sampling_required) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [`on_block`](#on_block) - [Pull-up tip helpers](#pull-up-tip-helpers) From a2db18cf3cf92bd194dfe03107f3a851a5e77c9e Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 12 Jul 2024 00:01:49 +0200 Subject: [PATCH 19/24] remove .history folder --- .../spec_builders/eip7594_20240626155910.py | 40 --- .../spec_builders/eip7594_20240711153253.py | 39 --- .../eip7594/das-core_20240627163448.md | 309 ------------------ .../eip7594/das-core_20240711153213.md | 308 ----------------- .../eip7594/fork-choice_20240711152138.md | 183 ----------- .../eip7594/fork-choice_20240711152321.md | 183 ----------- .../eip7594/fork-choice_20240711153141.md | 186 ----------- .../eip7594/fork-choice_20240711153757.md | 190 ----------- .../eip7594/fork-choice_20240711155539.md | 195 ----------- .../eip7594/fork-choice_20240711155548.md | 194 ----------- .../eip7594/fork-choice_20240711155631.md | 190 ----------- 11 files changed, 2017 deletions(-) delete mode 100644 .history/pysetup/spec_builders/eip7594_20240626155910.py delete mode 100644 .history/pysetup/spec_builders/eip7594_20240711153253.py delete mode 100644 .history/specs/_features/eip7594/das-core_20240627163448.md delete mode 100644 .history/specs/_features/eip7594/das-core_20240711153213.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711152138.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711152321.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711153141.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711153757.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711155539.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711155548.md delete mode 100644 .history/specs/_features/eip7594/fork-choice_20240711155631.md diff --git a/.history/pysetup/spec_builders/eip7594_20240626155910.py b/.history/pysetup/spec_builders/eip7594_20240626155910.py deleted file mode 100644 index a1e7251693..0000000000 --- a/.history/pysetup/spec_builders/eip7594_20240626155910.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import Dict - -from .base import BaseSpecBuilder -from ..constants import EIP7594 - - -class EIP7594SpecBuilder(BaseSpecBuilder): - fork: str = EIP7594 - - @classmethod - def imports(cls, preset_name: str): - return f''' -from eth2spec.deneb import {preset_name} as deneb -''' - - - @classmethod - def sundry_functions(cls) -> str: - return """ -def retrieve_column_sidecars(beacon_block_root: Root, - require_peer_sampling: bool) -> Sequence[DataColumnSidecar]: - return [] -""" - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - return { - 'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value, - 'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value, - 'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value, - 'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value, - } - - @classmethod - def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: - return { - 'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value, - } - - diff --git a/.history/pysetup/spec_builders/eip7594_20240711153253.py b/.history/pysetup/spec_builders/eip7594_20240711153253.py deleted file mode 100644 index 3c13d0ea43..0000000000 --- a/.history/pysetup/spec_builders/eip7594_20240711153253.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Dict - -from .base import BaseSpecBuilder -from ..constants import EIP7594 - - -class EIP7594SpecBuilder(BaseSpecBuilder): - fork: str = EIP7594 - - @classmethod - def imports(cls, preset_name: str): - return f''' -from eth2spec.deneb import {preset_name} as deneb -''' - - - @classmethod - def sundry_functions(cls) -> str: - return """ -def retrieve_column_sidecars(beacon_block_root: Root, - require_peer_sampling: bool) -> Sequence[DataColumnSidecar]: - return [] -""" - - @classmethod - def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: - return { - 'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value, - 'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value, - 'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value, - 'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value, - } - - @classmethod - def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]: - return { - 'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value, - } - diff --git a/.history/specs/_features/eip7594/das-core_20240627163448.md b/.history/specs/_features/eip7594/das-core_20240627163448.md deleted file mode 100644 index 5d03878f76..0000000000 --- a/.history/specs/_features/eip7594/das-core_20240627163448.md +++ /dev/null @@ -1,309 +0,0 @@ -# EIP-7594 -- Data Availability Sampling Core - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Constants](#constants) - - [Misc](#misc) -- [Custom types](#custom-types) -- [Configuration](#configuration) - - [Data size](#data-size) - - [Networking](#networking) - - [Custody setting](#custody-setting) - - [Containers](#containers) - - [`DataColumnSidecar`](#datacolumnsidecar) - - [Helper functions](#helper-functions) - - [`get_custody_columns`](#get_custody_columns) - - [`compute_extended_matrix`](#compute_extended_matrix) - - [`recover_matrix`](#recover_matrix) - - [`get_data_column_sidecars`](#get_data_column_sidecars) -- [Custody](#custody) - - [Custody requirement](#custody-requirement) - - [Public, deterministic selection](#public-deterministic-selection) -- [Peer discovery](#peer-discovery) -- [Extended data](#extended-data) -- [Column gossip](#column-gossip) - - [Parameters](#parameters) -- [Peer sampling](#peer-sampling) -- [Peer scoring](#peer-scoring) -- [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding) -- [DAS providers](#das-providers) -- [A note on fork choice](#a-note-on-fork-choice) -- [FAQs](#faqs) - - [Row (blob) custody](#row-blob-custody) - - [Subnet stability](#subnet-stability) - - - - -## Constants - -The following values are (non-configurable) constants used throughout the specification. - -### Misc - -| Name | Value | -| - | - | -| `UINT256_MAX` | `uint256(2**256 - 1)` | - -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `DataColumn` | `List[Cell, MAX_BLOB_COMMITMENTS_PER_BLOCK]` | The data of each column in EIP-7594 | -| `ExtendedMatrix` | `List[Cell, MAX_CELLS_IN_EXTENDED_MATRIX]` | The full data of one-dimensional erasure coding extended blobs (in row major format). | - -## Configuration - -### Data size - -| Name | Value | Description | -| - | - | - | -| `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix. | -| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix`. | - -### Networking - -| Name | Value | Description | -| - | - | - | -| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `128` | The number of data column sidecar subnets used in the gossipsub protocol | - -### Custody setting - -| Name | Value | Description | -| - | - | - | -| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | -| `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | -| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | -| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | -| `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | - - -### Containers - -#### `DataColumnSidecar` - -```python -class DataColumnSidecar(Container): - index: ColumnIndex # Index of column in extended matrix - column: DataColumn - kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] - kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK] - signed_block_header: SignedBeaconBlockHeader - kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH] -``` - -### Helper functions - -#### `get_custody_columns` - -```python -def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: - assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT - - subnet_ids: List[uint64] = [] - current_id = uint256(node_id) - while len(subnet_ids) < custody_subnet_count: - subnet_id = ( - bytes_to_uint64(hash(uint_to_bytes(uint256(current_id)))[0:8]) - % DATA_COLUMN_SIDECAR_SUBNET_COUNT - ) - if subnet_id not in subnet_ids: - subnet_ids.append(subnet_id) - if current_id == UINT256_MAX: - # Overflow prevention - current_id = NodeID(0) - current_id += 1 - - assert len(subnet_ids) == len(set(subnet_ids)) - - columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT - return sorted([ - ColumnIndex(DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnet_id) - for i in range(columns_per_subnet) - for subnet_id in subnet_ids - ]) -``` - -#### `compute_extended_matrix` - -```python -def compute_extended_matrix(blobs: Sequence[Blob]) -> ExtendedMatrix: - """ - Return the full ``ExtendedMatrix``. - - This helper demonstrates the relationship between blobs and ``ExtendedMatrix``. - The data structure for storing cells is implementation-dependent. - """ - extended_matrix = [] - for blob in blobs: - extended_matrix.extend(compute_cells(blob)) - return ExtendedMatrix(extended_matrix) -``` - -#### `recover_matrix` - -```python -def recover_matrix(cells_dict: Dict[Tuple[BlobIndex, CellID], Cell], blob_count: uint64) -> ExtendedMatrix: - """ - Return the recovered ``ExtendedMatrix``. - - This helper demonstrates how to apply ``recover_all_cells``. - The data structure for storing cells is implementation-dependent. - """ - extended_matrix: List[Cell] = [] - for blob_index in range(blob_count): - cell_ids = [cell_id for b_index, cell_id in cells_dict.keys() if b_index == blob_index] - cells = [cells_dict[(BlobIndex(blob_index), cell_id)] for cell_id in cell_ids] - - all_cells_for_row = recover_all_cells(cell_ids, cells) - extended_matrix.extend(all_cells_for_row) - return ExtendedMatrix(extended_matrix) -``` - -#### `get_data_column_sidecars` - -```python -def get_data_column_sidecars(signed_block: SignedBeaconBlock, - blobs: Sequence[Blob]) -> Sequence[DataColumnSidecar]: - signed_block_header = compute_signed_block_header(signed_block) - block = signed_block.message - kzg_commitments_inclusion_proof = compute_merkle_proof( - block.body, - get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'), - ) - cells_and_proofs = [compute_cells_and_kzg_proofs(blob) for blob in blobs] - blob_count = len(blobs) - cells = [cells_and_proofs[i][0] for i in range(blob_count)] - proofs = [cells_and_proofs[i][1] for i in range(blob_count)] - sidecars = [] - for column_index in range(NUMBER_OF_COLUMNS): - column = DataColumn([cells[row_index][column_index] - for row_index in range(blob_count)]) - kzg_proof_of_column = [proofs[row_index][column_index] - for row_index in range(blob_count)] - sidecars.append(DataColumnSidecar( - index=column_index, - column=column, - kzg_commitments=block.body.blob_kzg_commitments, - kzg_proofs=kzg_proof_of_column, - signed_block_header=signed_block_header, - kzg_commitments_inclusion_proof=kzg_commitments_inclusion_proof, - )) - return sidecars -``` - -## Custody - -### Custody requirement - -Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached, and with the sum of the balances of all attached validators being `total_node_balance`, downloads and custodies `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` subnets per slot, with a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` and of course a maximum of `DATA_COLUMN_SIDECAR_SUBNET_COUNT`. - -```python -def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: - total_node_balance = sum(state.balances[index] for index in validator_indices) - count = total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET - return min(max(count, VALIDATOR_CUSTODY_REQUIREMENT), DATA_COLUMN_SIDECAR_SUBNET_COUNT) -``` - -The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). - -A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_subnet_count: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node). - -A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns. - -### Public, deterministic selection - -The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_columns`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public. - -*Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody. - -## Peer discovery - -At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands. - -A node runs a background peer discovery process, maintaining at least `TARGET_NUMBER_OF_PEERS` of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. - -`TARGET_NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling. - -*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes. - -*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked. - -## Extended data - -In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension. `compute_extended_matrix` demonstrates the relationship between blobs and custom type `ExtendedMatrix`. - -## Column gossip - -### Parameters - -For each column -- use `data_column_sidecar_{subnet_id}` subnets, where `subnet_id` can be computed with the `compute_subnet_for_data_column_sidecar(column_index: ColumnIndex)` helper. The sidecars can be computed with the `get_data_column_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob])` helper. - -To custody a particular column, a node joins the respective gossip subnet. Verifiable samples from their respective column are gossiped on the assigned subnet. - -## Peer sampling - -A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. At each slot, a node makes `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success. - -## Peer scoring - -Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer. - -## Reconstruction and cross-seeding - -If the node obtains 50%+ of all the columns, they can reconstruct the full data matrix via `recover_matrix` helper. - -If a node fails to sample a peer or fails to get a column on the column subnet, a node can utilize the Req/Resp message to query the missing column from other peers. - -Once the node obtain the column, the node SHOULD send the missing columns to the column subnets. - -*Note*: A node always maintains a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction. - -*Note*: There are timing considerations to analyze -- at what point does a node consider samples missing and choose to reconstruct and cross-seed. - -*Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms. - -## DAS providers - -A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery. - -DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service. - -Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms. - -## A note on fork choice - -*Fork choice spec TBD, but it will just be a replacement of `is_data_available()` call in Deneb with column sampling instead of full download. Note the `is_data_available(slot_N)` will likely do a `-1` follow distance so that you just need to check the availability of slot `N-1` for slot `N` (starting with the block proposer of `N`).* - -The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of a particular design impacting it. - -In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc. - -For example, the fork choice rule might require validators to do successful DAS on slot `N` to be able to include block of slot `N` in its fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that they don't hurt the aggregate security. For example, the rule could be — DAS must be completed for slot N-1 for a child block in N to be included in the fork choice. - -Such trailing techniques and their analysis will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules. - -## FAQs - -### Row (blob) custody - -In the one-dimension construction, a node samples the peers by requesting the whole `DataColumn`. In reconstruction, a node can reconstruct all the blobs by 50% of the columns. Note that nodes can still download the row via `blob_sidecar_{subnet_id}` subnets. - -The potential benefits of having row custody could include: - -1. Allow for more "natural" distribution of data to consumers -- e.g., roll-ups -- but honestly, they won't know a priori which row their blob is going to be included in in the block, so they would either need to listen to all rows or download a particular row after seeing the block. The former looks just like listening to column [0, N) and the latter is req/resp instead of gossiping. -2. Help with some sort of distributed reconstruction. Those with full rows can compute extensions and seed missing samples to the network. This would either need to be able to send individual points on the gossip or would need some sort of req/resp faculty, potentially similar to an `IHAVEPOINTBITFIELD` and `IWANTSAMPLE`. - -However, for simplicity, we don't assign row custody assignments to nodes in the current design. - -### Subnet stability - -To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_columns`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs. diff --git a/.history/specs/_features/eip7594/das-core_20240711153213.md b/.history/specs/_features/eip7594/das-core_20240711153213.md deleted file mode 100644 index fbde23fd1a..0000000000 --- a/.history/specs/_features/eip7594/das-core_20240711153213.md +++ /dev/null @@ -1,308 +0,0 @@ -# EIP-7594 -- Data Availability Sampling Core - -**Notice**: This document is a work-in-progress for researchers and implementers. - -## Table of contents - - - - - -- [Constants](#constants) - - [Misc](#misc) -- [Custom types](#custom-types) -- [Configuration](#configuration) - - [Data size](#data-size) - - [Networking](#networking) - - [Custody setting](#custody-setting) - - [Containers](#containers) - - [`DataColumnSidecar`](#datacolumnsidecar) - - [Helper functions](#helper-functions) - - [`get_custody_columns`](#get_custody_columns) - - [`compute_extended_matrix`](#compute_extended_matrix) - - [`recover_matrix`](#recover_matrix) - - [`get_data_column_sidecars`](#get_data_column_sidecars) -- [Custody](#custody) - - [Custody requirement](#custody-requirement) - - [Public, deterministic selection](#public-deterministic-selection) -- [Peer discovery](#peer-discovery) -- [Extended data](#extended-data) -- [Column gossip](#column-gossip) - - [Parameters](#parameters) -- [Peer sampling](#peer-sampling) -- [Peer scoring](#peer-scoring) -- [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding) -- [DAS providers](#das-providers) -- [A note on fork choice](#a-note-on-fork-choice) -- [FAQs](#faqs) - - [Row (blob) custody](#row-blob-custody) - - [Subnet stability](#subnet-stability) - - - - -## Constants - -The following values are (non-configurable) constants used throughout the specification. - -### Misc - -| Name | Value | -| - | - | -| `UINT256_MAX` | `uint256(2**256 - 1)` | - -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `DataColumn` | `List[Cell, MAX_BLOB_COMMITMENTS_PER_BLOCK]` | The data of each column in EIP-7594 | -| `ExtendedMatrix` | `List[Cell, MAX_CELLS_IN_EXTENDED_MATRIX]` | The full data of one-dimensional erasure coding extended blobs (in row major format). | - -## Configuration - -### Data size - -| Name | Value | Description | -| - | - | - | -| `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix. | -| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix`. | - -### Networking - -| Name | Value | Description | -| - | - | - | -| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `128` | The number of data column sidecar subnets used in the gossipsub protocol | - -### Custody setting - -| Name | Value | Description | -| - | - | - | -| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | -| `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | -| `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | -| `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody | -| `TARGET_NUMBER_OF_PEERS` | `100` | Suggested minimum peer count | - -### Containers - -#### `DataColumnSidecar` - -```python -class DataColumnSidecar(Container): - index: ColumnIndex # Index of column in extended matrix - column: DataColumn - kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] - kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK] - signed_block_header: SignedBeaconBlockHeader - kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH] -``` - -### Helper functions - -#### `get_custody_columns` - -```python -def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: - assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT - - subnet_ids: List[uint64] = [] - current_id = uint256(node_id) - while len(subnet_ids) < custody_subnet_count: - subnet_id = ( - bytes_to_uint64(hash(uint_to_bytes(uint256(current_id)))[0:8]) - % DATA_COLUMN_SIDECAR_SUBNET_COUNT - ) - if subnet_id not in subnet_ids: - subnet_ids.append(subnet_id) - if current_id == UINT256_MAX: - # Overflow prevention - current_id = NodeID(0) - current_id += 1 - - assert len(subnet_ids) == len(set(subnet_ids)) - - columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT - return sorted([ - ColumnIndex(DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnet_id) - for i in range(columns_per_subnet) - for subnet_id in subnet_ids - ]) -``` - -#### `compute_extended_matrix` - -```python -def compute_extended_matrix(blobs: Sequence[Blob]) -> ExtendedMatrix: - """ - Return the full ``ExtendedMatrix``. - - This helper demonstrates the relationship between blobs and ``ExtendedMatrix``. - The data structure for storing cells is implementation-dependent. - """ - extended_matrix = [] - for blob in blobs: - extended_matrix.extend(compute_cells(blob)) - return ExtendedMatrix(extended_matrix) -``` - -#### `recover_matrix` - -```python -def recover_matrix(cells_dict: Dict[Tuple[BlobIndex, CellID], Cell], blob_count: uint64) -> ExtendedMatrix: - """ - Return the recovered ``ExtendedMatrix``. - - This helper demonstrates how to apply ``recover_all_cells``. - The data structure for storing cells is implementation-dependent. - """ - extended_matrix: List[Cell] = [] - for blob_index in range(blob_count): - cell_ids = [cell_id for b_index, cell_id in cells_dict.keys() if b_index == blob_index] - cells = [cells_dict[(BlobIndex(blob_index), cell_id)] for cell_id in cell_ids] - - all_cells_for_row = recover_all_cells(cell_ids, cells) - extended_matrix.extend(all_cells_for_row) - return ExtendedMatrix(extended_matrix) -``` - -#### `get_data_column_sidecars` - -```python -def get_data_column_sidecars(signed_block: SignedBeaconBlock, - blobs: Sequence[Blob]) -> Sequence[DataColumnSidecar]: - signed_block_header = compute_signed_block_header(signed_block) - block = signed_block.message - kzg_commitments_inclusion_proof = compute_merkle_proof( - block.body, - get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'), - ) - cells_and_proofs = [compute_cells_and_kzg_proofs(blob) for blob in blobs] - blob_count = len(blobs) - cells = [cells_and_proofs[i][0] for i in range(blob_count)] - proofs = [cells_and_proofs[i][1] for i in range(blob_count)] - sidecars = [] - for column_index in range(NUMBER_OF_COLUMNS): - column = DataColumn([cells[row_index][column_index] - for row_index in range(blob_count)]) - kzg_proof_of_column = [proofs[row_index][column_index] - for row_index in range(blob_count)] - sidecars.append(DataColumnSidecar( - index=column_index, - column=column, - kzg_commitments=block.body.blob_kzg_commitments, - kzg_proofs=kzg_proof_of_column, - signed_block_header=signed_block_header, - kzg_commitments_inclusion_proof=kzg_commitments_inclusion_proof, - )) - return sidecars -``` - -## Custody - -### Custody requirement - -Each node *without attached validators* downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. A node with validators attached downloads and custodies a higher minimum of subnets per slot, determined by `get_validators_custody_requirement(state, validator_indices)`. Here, `state` is the current `BeaconState` and `validator_indices` is the list of indices corresponding to validators attached to the node. Any node with at least one validator attached, and with the sum of the balances of all attached validators being `total_node_balance`, downloads and custodies `total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` subnets per slot, with a minimum of `VALIDATOR_CUSTODY_REQUIREMENT` and of course a maximum of `DATA_COLUMN_SIDECAR_SUBNET_COUNT`. - -```python -def get_validators_custody_requirement(state: BeaconState, validator_indices: Sequence[ValidatorIndex]) -> uint64: - total_node_balance = sum(state.balances[index] for index in validator_indices) - count = total_node_balance // BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET - return min(max(count, VALIDATOR_CUSTODY_REQUIREMENT), DATA_COLUMN_SIDECAR_SUBNET_COUNT) -``` - -The particular subnets that the node is required to custody are selected pseudo-randomly (more on this below). - -A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_subnet_count: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node). - -A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns. - -### Public, deterministic selection - -The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_columns`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public. - -*Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody. - -## Peer discovery - -At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands. - -A node runs a background peer discovery process, maintaining at least `TARGET_NUMBER_OF_PEERS` of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. - -`TARGET_NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling. - -*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes. - -*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked. - -## Extended data - -In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension. `compute_extended_matrix` demonstrates the relationship between blobs and custom type `ExtendedMatrix`. - -## Column gossip - -### Parameters - -For each column -- use `data_column_sidecar_{subnet_id}` subnets, where `subnet_id` can be computed with the `compute_subnet_for_data_column_sidecar(column_index: ColumnIndex)` helper. The sidecars can be computed with the `get_data_column_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob])` helper. - -To custody a particular column, a node joins the respective gossip subnet. Verifiable samples from their respective column are gossiped on the assigned subnet. - -## Peer sampling - -A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. At each slot, a node makes `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success. - -## Peer scoring - -Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer. - -## Reconstruction and cross-seeding - -If the node obtains 50%+ of all the columns, they can reconstruct the full data matrix via `recover_matrix` helper. - -If a node fails to sample a peer or fails to get a column on the column subnet, a node can utilize the Req/Resp message to query the missing column from other peers. - -Once the node obtain the column, the node SHOULD send the missing columns to the column subnets. - -*Note*: A node always maintains a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction. - -*Note*: There are timing considerations to analyze -- at what point does a node consider samples missing and choose to reconstruct and cross-seed. - -*Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms. - -## DAS providers - -A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery. - -DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service. - -Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms. - -## A note on fork choice - -*Fork choice spec TBD, but it will just be a replacement of `is_data_available()` call in Deneb with column sampling instead of full download. Note the `is_data_available(slot_N)` will likely do a `-1` follow distance so that you just need to check the availability of slot `N-1` for slot `N` (starting with the block proposer of `N`).* - -The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of a particular design impacting it. - -In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc. - -For example, the fork choice rule might require validators to do successful DAS on slot `N` to be able to include block of slot `N` in its fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that they don't hurt the aggregate security. For example, the rule could be — DAS must be completed for slot N-1 for a child block in N to be included in the fork choice. - -Such trailing techniques and their analysis will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules. - -## FAQs - -### Row (blob) custody - -In the one-dimension construction, a node samples the peers by requesting the whole `DataColumn`. In reconstruction, a node can reconstruct all the blobs by 50% of the columns. Note that nodes can still download the row via `blob_sidecar_{subnet_id}` subnets. - -The potential benefits of having row custody could include: - -1. Allow for more "natural" distribution of data to consumers -- e.g., roll-ups -- but honestly, they won't know a priori which row their blob is going to be included in in the block, so they would either need to listen to all rows or download a particular row after seeing the block. The former looks just like listening to column [0, N) and the latter is req/resp instead of gossiping. -2. Help with some sort of distributed reconstruction. Those with full rows can compute extensions and seed missing samples to the network. This would either need to be able to send individual points on the gossip or would need some sort of req/resp faculty, potentially similar to an `IHAVEPOINTBITFIELD` and `IWANTSAMPLE`. - -However, for simplicity, we don't assign row custody assignments to nodes in the current design. - -### Subnet stability - -To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_columns`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs. diff --git a/.history/specs/_features/eip7594/fork-choice_20240711152138.md b/.history/specs/_features/eip7594/fork-choice_20240711152138.md deleted file mode 100644 index c1346d518f..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711152138.md +++ /dev/null @@ -1,183 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - if beacon_block_root not in store.blocks: - # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) - -``` - -#### `get_head` - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711152321.md b/.history/specs/_features/eip7594/fork-choice_20240711152321.md deleted file mode 100644 index 32cac43fc4..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711152321.md +++ /dev/null @@ -1,183 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - if beacon_block_root not in store.blocks: - # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) - -``` - -#### Modified `get_head` - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711153141.md b/.history/specs/_features/eip7594/fork-choice_20240711153141.md deleted file mode 100644 index 9956f64b5b..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711153141.md +++ /dev/null @@ -1,186 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - if beacon_block_root not in store.blocks: - # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) -``` - -#### Modified `get_head` - -*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. -For blocks from the current or previous epoch (which cannot yet be finalized), this is established through -a custody check, while for blocks older than two epochs through a full peer sampling check. - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711153757.md b/.history/specs/_features/eip7594/fork-choice_20240711153757.md deleted file mode 100644 index 55bc645cbd..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711153757.md +++ /dev/null @@ -1,190 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - """ - Checks if all ancestors of `beacon_block_root` within the custody period are - available, as determined by `is_data_available` with peer sampling enabled - """ - if beacon_block_root not in store.blocks: - # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) -``` - -#### Modified `get_head` - -*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. -For blocks from the current or previous epoch (which cannot yet be finalized), this is established through -a custody check, while for blocks older than two epochs through a full peer sampling check. - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711155539.md b/.history/specs/_features/eip7594/fork-choice_20240711155539.md deleted file mode 100644 index c6bb7f401d..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711155539.md +++ /dev/null @@ -1,195 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never -results in adding to `store` a block with an unavailable ancestor, because blocks are not added to -`store` unless their whole ancestry has already has been. This situation could only present itself -around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - """ - Checks if all ancestors of `beacon_block_root` within the custody period are - available, as determined by `is_data_available` with peer sampling enabled - """ - if beacon_block_root not in store.blocks: - # Deal with Genesis edge cases, where current_justified_checkpoint and parent_root are not set - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) -``` - -#### Modified `get_head` - -*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. -For blocks from the current or previous epoch (which cannot yet be finalized), this is established through -a custody check, while for blocks older than two epochs through a full peer sampling check. - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711155548.md b/.history/specs/_features/eip7594/fork-choice_20240711155548.md deleted file mode 100644 index 0230b43cdf..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711155548.md +++ /dev/null @@ -1,194 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### `retrieve_column_sidecars` - -`def retrieve_column_sidecars(beacon_block_root: Root, columns_to_retrieve: Sequence[ColumnIndex]) -> Sequence[DataColumnSidecar]` - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never -results in adding to `store` a block with an unavailable ancestor, because blocks are not added to -`store` unless their whole ancestry has already has been. This situation could only present itself -around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - """ - Checks if all ancestors of `beacon_block_root` within the custody period are - available, as determined by `is_data_available` with peer sampling enabled - """ - if beacon_block_root not in store.blocks: - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) -``` - -#### Modified `get_head` - -*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. -For blocks from the current or previous epoch (which cannot yet be finalized), this is established through -a custody check, while for blocks older than two epochs through a full peer sampling check. - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` diff --git a/.history/specs/_features/eip7594/fork-choice_20240711155631.md b/.history/specs/_features/eip7594/fork-choice_20240711155631.md deleted file mode 100644 index 1f0ec0b232..0000000000 --- a/.history/specs/_features/eip7594/fork-choice_20240711155631.md +++ /dev/null @@ -1,190 +0,0 @@ -# EIP-7594 -- Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) - - [Helpers](#helpers) - - [`get_custody_parameters`](#get_custody_parameters) - - [`get_sampling_columns`](#get_sampling_columns) - - [`retrieve_column_sidecars`](#retrieve_column_sidecars) - - [`is_data_available`](#is_data_available) - - [`is_chain_available`](#is_chain_available) - - [`get_head`](#get_head) - - [`is_peer_sampling_required`](#is_peer_sampling_required) -- [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) - - - - -## Introduction - -This is the modification of the fork choice accompanying EIP-7594. - -### Helpers - -#### Modified `is_data_available` - -```python -def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: - column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) - return all( - verify_data_column_sidecar_kzg_proofs(column_sidecar) - for column_sidecar in column_sidecars - ) -``` - -#### New `is_chain_available` - -*Note*: if `beacon_block_root` is not found in `store.blocks`, we return `True`. Note that this never -results in adding to `store` a block with an unavailable ancestor, because blocks are not added to -`store` unless their whole ancestry has already has been. This situation could only present itself -around Genesis, where `current_justified_checkpoint` and `parent_root` are initially not set. - -```python -def is_chain_available(store: Store, beacon_block_root: Root) -> bool: - """ - Checks if all ancestors of `beacon_block_root` within the custody period are - available, as determined by `is_data_available` with peer sampling enabled - """ - if beacon_block_root not in store.blocks: - return True - block = store.blocks[beacon_block_root] - block_epoch = compute_epoch_at_slot(block.slot) - current_epoch = get_current_store_epoch(store) - if block_epoch + MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS <= current_epoch: - return True - parent_root = block.parent_root - return ( - is_data_available(beacon_block_root, require_peer_sampling=True) - and is_chain_available(store, parent_root) - ) -``` - -#### Modified `get_head` - -*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. -For blocks from the current or previous epoch (which cannot yet be finalized), this is established through -a custody check, while for blocks older than two epochs through a full peer sampling check. - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available( - root, - require_peer_sampling=is_peer_sampling_required(store, block.slot) - ) - ) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - -#### New `is_peer_sampling_required` - -```python -def is_peer_sampling_required(store: Store, slot: Slot): - return compute_epoch_at_slot(slot) + 2 <= get_current_store_epoch(store) -``` - -## Updated fork-choice handlers - -### `on_block` - -*Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - """ - Run ``on_block`` upon receiving a new block. - """ - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - finalized_checkpoint_block = get_checkpoint_block( - store, - block.parent_root, - store.finalized_checkpoint.epoch, - ) - assert store.finalized_checkpoint.root == finalized_checkpoint_block - - # Check the block is valid and compute the post-state - block_root = hash_tree_root(block) - state_transition(state, signed_block, True) - - # [New in EIP7594] Do not import the block if its unrealized justified checkpoint is not available - pulled_up_state = state.copy() - process_justification_and_finalization(pulled_up_state) - assert is_chain_available(store, pulled_up_state.current_justified_checkpoint.root) - - # Add new block to the store - store.blocks[block_root] = block - # Add new state for this block to the store - store.block_states[block_root] = state - - # Add block timeliness to the store - time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT - is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT - is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval - store.block_timeliness[hash_tree_root(block)] = is_timely - - # Add proposer score boost if the block is timely and not conflicting with an existing block - is_first_block = store.proposer_boost_root == Root() - if is_timely and is_first_block: - store.proposer_boost_root = hash_tree_root(block) - - # Update checkpoints in store if necessary - update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) - - # Eagerly compute unrealized justification and finality. - compute_pulled_up_tip(store, pulled_up_state, block_root) -``` - -#### Pull-up tip helpers - -##### `compute_pulled_up_tip` - -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. -The application of `processing_justification_and_finalization` now happens in `on_block`. - -```python -def compute_pulled_up_tip(store: Store, pulled_up_state: BeaconState, block_root: Root) -> None: - store.unrealized_justifications[block_root] = pulled_up_state.current_justified_checkpoint - unrealized_justified = pulled_up_state.current_justified_checkpoint - unrealized_finalized = pulled_up_state.finalized_checkpoint - update_unrealized_checkpoints(store, unrealized_justified, unrealized_finalized) - - # If the block is from a prior epoch, apply the realized values - block_epoch = compute_epoch_at_slot(store.blocks[block_root].slot) - current_epoch = get_current_store_epoch(store) - if block_epoch < current_epoch: - update_checkpoints(store, unrealized_justified, unrealized_finalized) -``` From 19804a33cecabb65d4bd7ae3d1db9fb2c11a98f1 Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 12 Jul 2024 01:03:24 +0200 Subject: [PATCH 20/24] Apply suggestions from code review Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/fork-choice.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 028db64786..44bf5c8ed5 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -29,6 +29,10 @@ This is the modification of the fork choice accompanying EIP-7594. ```python def is_data_available(beacon_block_root: Root, require_peer_sampling: bool=False) -> bool: + # `retrieve_column_sidecars` is implementation and context dependent, replacing `retrieve_blobs_and_proofs`. + # For the given block root, it returns all column sidecars to custody, and, if `require_peer_sampling` is `True`, + # also all column sidecars selected for peer sampling, or raises an exception if they are not available. The p2p + # network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs. column_sidecars = retrieve_column_sidecars(beacon_block_root, require_peer_sampling) return all( verify_data_column_sidecar_kzg_proofs(column_sidecar) @@ -47,7 +51,7 @@ around Genesis, where `current_justified_checkpoint` and `parent_root` are initi def is_chain_available(store: Store, beacon_block_root: Root) -> bool: """ Checks if all ancestors of `beacon_block_root` within the custody period are - available, as determined by `is_data_available` with peer sampling enabled + available, as determined by `is_data_available` with peer sampling enabled. """ if beacon_block_root not in store.blocks: return True @@ -167,9 +171,9 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: #### Pull-up tip helpers -##### `compute_pulled_up_tip` +##### Modified `compute_pulled_up_tip` -Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. +*Note*: Modified to take `pulled_up_state`, the block's state after applying `processing_justification_and_finalization`. The application of `processing_justification_and_finalization` now happens in `on_block`. ```python From 1613d2e3bf41d74d8eb3931346ecaf6e5e110335 Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 12 Jul 2024 01:06:00 +0200 Subject: [PATCH 21/24] Small fix --- specs/_features/eip7594/fork-choice.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 44bf5c8ed5..0672978e96 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -107,11 +107,11 @@ def is_peer_sampling_required(store: Store, slot: Slot): ## Updated fork-choice handlers -### `on_block` +### Modified `on_block` *Note*: The blob data availability check is removed and replaced with an availability -check on the on the justified checkpoint in the "pulled up state" of the block, after -applying `process_justification_and_finalization`. +check on the justified checkpoint in the "pulled up state" of the block, which is +the state after applying `process_justification_and_finalization`. ```python def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: From 62006c99178f99d056a6d01081751e1cbc4ea5cb Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 12 Jul 2024 01:08:14 +0200 Subject: [PATCH 22/24] doctoc --- specs/_features/eip7594/fork-choice.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 0672978e96..523b411dbe 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -12,9 +12,9 @@ - [Modified `get_head`](#modified-get_head) - [New `is_peer_sampling_required`](#new-is_peer_sampling_required) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - - [`on_block`](#on_block) + - [Modified `on_block`](#modified-on_block) - [Pull-up tip helpers](#pull-up-tip-helpers) - - [`compute_pulled_up_tip`](#compute_pulled_up_tip) + - [Modified `compute_pulled_up_tip`](#modified-compute_pulled_up_tip) From 3f5ba2e5ad95a87c4d402246f00eb47712081b33 Mon Sep 17 00:00:00 2001 From: fradamt <104826920+fradamt@users.noreply.github.com> Date: Mon, 29 Jul 2024 07:29:23 +0200 Subject: [PATCH 23/24] typo --- specs/_features/eip7594/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 523b411dbe..e66b479433 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -69,7 +69,7 @@ def is_chain_available(store: Store, beacon_block_root: Root) -> bool: #### Modified `get_head` -*Note*: children of the current `head` are required to be available in order to be consider by the fork-choice. +*Note*: children of the current `head` are required to be available in order to be considered by the fork-choice. For blocks from the current or previous epoch (which cannot yet be finalized), this is established through a custody check, while for blocks older than two epochs through a full peer sampling check. From 8dbd8747ddfcb1dbacb5450e5ecb5aca6dd006eb Mon Sep 17 00:00:00 2001 From: fradamt <104826920+fradamt@users.noreply.github.com> Date: Mon, 5 Aug 2024 16:55:03 +0200 Subject: [PATCH 24/24] fix wrong type --- specs/_features/eip7594/das-core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 1867ecf8b2..055d4c1911 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -81,7 +81,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Description | | - | - | - | -| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumn` random samples a node queries per slot | +| `SAMPLES_PER_SLOT` | `16` | Number of `DataColumnSidecar` random samples a node queries per slot | | `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | | `VALIDATOR_CUSTODY_REQUIREMENT` | `8` | Minimum number of subnets an honest node with validators attached custodies and serves samples from | | `BALANCE_PER_ADDITIONAL_CUSTODY_SUBNET` | `Gwei(32 * 10**9)` | Balance increment corresponding to one additional subnet to custody |