From e2a661668ebe154f8b84eb9c5102f398fae72a13 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 12 Dec 2024 16:38:39 +0100 Subject: [PATCH 01/47] historical fee service: working on calculating SMAs --- packages/services/src/historical_fees.rs | 301 ++++++++++++++++++ packages/services/src/lib.rs | 1 + packages/services/src/state_committer.rs | 2 + .../src/state_committer/fee_optimization.rs | 78 +++++ packages/services/tests/historical_fees.rs | 27 ++ 5 files changed, 409 insertions(+) create mode 100644 packages/services/src/historical_fees.rs create mode 100644 packages/services/src/state_committer/fee_optimization.rs create mode 100644 packages/services/tests/historical_fees.rs diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs new file mode 100644 index 00000000..66d77f58 --- /dev/null +++ b/packages/services/src/historical_fees.rs @@ -0,0 +1,301 @@ +pub mod port { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct Fees { + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct BlockFees { + pub height: u64, + pub fees: Fees, + } + + pub mod l1 { + use std::ops::RangeInclusive; + + use itertools::Itertools; + use nonempty::NonEmpty; + + use super::BlockFees; + + #[derive(Debug)] + pub struct SequentialBlockFees { + fees: Vec, + } + + impl IntoIterator for SequentialBlockFees { + type Item = BlockFees; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.fees.into_iter() + } + } + + // Cannot be empty + #[allow(clippy::len_without_is_empty)] + impl SequentialBlockFees { + pub fn len(&self) -> usize { + self.fees.len() + } + } + + #[derive(Debug)] + pub struct InvalidSequence(String); + + impl std::error::Error for InvalidSequence {} + + impl std::fmt::Display for InvalidSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } + } + + impl TryFrom> for SequentialBlockFees { + type Error = InvalidSequence; + fn try_from(mut fees: Vec) -> Result { + if fees.is_empty() { + return Err(InvalidSequence("Input cannot be empty".to_string())); + } + + fees.sort_by_key(|f| f.height); + + let is_sequential = fees + .iter() + .tuple_windows() + .all(|(l, r)| l.height + 1 == r.height); + + if !is_sequential { + return Err(InvalidSequence( + "blocks are not sequential by height".to_string(), + )); + } + + Ok(Self { fees }) + } + } + + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait FeesProvider { + async fn fees(&self, height_range: RangeInclusive) -> NonEmpty; + async fn current_block_height(&self) -> u64; + } + + #[cfg(feature = "test-helpers")] + pub mod testing { + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use nonempty::NonEmpty; + + use crate::{ + historical_fees::port::{BlockFees, Fees}, + types::CollectNonEmpty, + }; + + use super::FeesProvider; + + pub struct TestFeesProvider { + fees: BTreeMap, + } + + impl FeesProvider for TestFeesProvider { + async fn current_block_height(&self) -> u64 { + *self.fees.keys().last().unwrap() + } + + async fn fees(&self, height_range: RangeInclusive) -> NonEmpty { + self.fees + .iter() + .skip_while(|(height, _)| !height_range.contains(height)) + .take_while(|(height, _)| height_range.contains(height)) + .map(|(height, fees)| BlockFees { + height: *height, + fees: *fees, + }) + .collect_nonempty() + .unwrap() + } + } + + impl TestFeesProvider { + pub fn new(blocks: impl IntoIterator) -> Self { + Self { + fees: blocks.into_iter().collect(), + } + } + } + + pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { + (0..num_blocks) + .map(|i| { + ( + i, + Fees { + base_fee_per_gas: i as u128, + reward: i as u128, + base_fee_per_blob_gas: i as u128, + }, + ) + }) + .collect() + } + } + } + + pub mod service { + use super::{l1::FeesProvider, Fees}; + + pub struct HistoricalFeesProvider

{ + fees_provider: P, + } + impl

HistoricalFeesProvider

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } + } + } + + impl HistoricalFeesProvider

{ + pub async fn calculate_sma(&self, last_n_blocks: u32) -> Fees { + let fees = self.fees_provider.fees(0..=last_n_blocks as u64).await; + + eprintln!("got fees: {:?}", fees); + + let a = fees.last().fees; + eprintln!("got fees: {:?}", a); + a + } + } + } +} + +#[cfg(test)] +mod tests { + use port::{l1::SequentialBlockFees, BlockFees, Fees}; + + use super::*; + + #[test] + fn given_sequential_block_fees_when_valid_then_creation_succeeds() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn given_sequential_block_fees_when_empty_then_creation_fails() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn given_sequential_block_fees_when_non_sequential_then_creation_fails() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height\")" + ); + } + + #[test] + fn given_sequential_block_fees_when_valid_then_can_iterate_over_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + assert_eq!( + iterated_fees, block_fees, + "Expected iterator to yield the same block fees" + ); + } +} diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 0e1391a1..29b18fbf 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -3,6 +3,7 @@ pub mod block_committer; pub mod block_importer; pub mod cost_reporter; pub mod health_reporter; +pub mod historical_fees; pub mod state_committer; pub mod state_listener; pub mod state_pruner; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index ca2dcf3f..4395b606 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,3 +1,5 @@ +mod fee_optimization; + pub mod service { use std::{num::NonZeroUsize, time::Duration}; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs new file mode 100644 index 00000000..e8a6b3f9 --- /dev/null +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -0,0 +1,78 @@ +use std::{ops::RangeInclusive, time::Duration}; + +use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider}; +use nonempty::NonEmpty; +use rayon::range_inclusive; + +#[derive(Debug, Clone, Copy)] +pub struct Config { + pub short_term_sma_num_blocks: u32, + pub long_term_sma_num_blocks: u32, +} + +pub struct SendOrWaitDecider

{ + price_service: HistoricalFeesProvider

, + config: Config, +} + +impl

SendOrWaitDecider

{ + pub fn new(price_service: HistoricalFeesProvider

, config: Config) -> Self { + Self { + price_service, + config, + } + } +} + +impl SendOrWaitDecider

{ + pub async fn should_send_blob_tx(&self) -> bool { + let short_term_sma = self + .price_service + .calculate_sma(self.config.short_term_sma_num_blocks) + .await; + + let long_term_sma = self + .price_service + .calculate_sma(self.config.long_term_sma_num_blocks) + .await; + + return short_term_sma.base_fee_per_gas < long_term_sma.base_fee_per_gas; + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + time::Duration, + }; + + use crate::historical_fees::port::{ + l1::testing::{incrementing_fees, TestFeesProvider}, + BlockFees, Fees, + }; + + use crate::types::CollectNonEmpty; + + use super::*; + + #[tokio::test] + async fn should_send_if_shortterm_sma_lower_than_longterm_sma() { + // given + let config = Config { + short_term_sma_num_blocks: 5, + long_term_sma_num_blocks: 50, + }; + + let fees_provider = TestFeesProvider::new(incrementing_fees(50)); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let sut = SendOrWaitDecider::new(price_service, config); + + // when + let decision = sut.should_send_blob_tx().await; + + // then + assert!(decision, "Should have sent"); + } +} diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs new file mode 100644 index 00000000..125fd01c --- /dev/null +++ b/packages/services/tests/historical_fees.rs @@ -0,0 +1,27 @@ +use std::{collections::BTreeMap, ops::RangeInclusive}; + +use nonempty::NonEmpty; +use services::{ + historical_fees::port::{ + l1::{testing, FeesProvider}, + service::HistoricalFeesProvider, + BlockFees, Fees, + }, + types::CollectNonEmpty, +}; + +#[tokio::test] +async fn calculates_sma_correctly_for_last_1_lock() { + // given + let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let price_service = HistoricalFeesProvider::new(fees_provider); + let last_n_blocks = 1; + + // when + let sma = price_service.calculate_sma(last_n_blocks).await; + + // then + assert_eq!(sma.base_fee_per_gas, 5); + assert_eq!(sma.reward, 5); + assert_eq!(sma.base_fee_per_blob_gas, 5); +} From 8eb3243a5432948aeeca499797063ba47498a68b Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 13:37:21 +0100 Subject: [PATCH 02/47] starting to add decision logic based on sma price --- packages/services/src/historical_fees.rs | 53 ++- .../src/state_committer/fee_optimization.rs | 379 +++++++++++++++++- packages/services/tests/historical_fees.rs | 19 +- 3 files changed, 428 insertions(+), 23 deletions(-) diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs index 66d77f58..bc43b279 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/historical_fees.rs @@ -1,5 +1,5 @@ pub mod port { - #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] pub struct Fees { pub base_fee_per_gas: u128, pub reward: u128, @@ -134,9 +134,9 @@ pub mod port { ( i, Fees { - base_fee_per_gas: i as u128, - reward: i as u128, - base_fee_per_blob_gas: i as u128, + base_fee_per_gas: i as u128 + 1, + reward: i as u128 + 1, + base_fee_per_blob_gas: i as u128 + 1, }, ) }) @@ -146,7 +146,11 @@ pub mod port { } pub mod service { - use super::{l1::FeesProvider, Fees}; + use std::ops::RangeInclusive; + + use nonempty::NonEmpty; + + use super::{l1::FeesProvider, BlockFees, Fees}; pub struct HistoricalFeesProvider

{ fees_provider: P, @@ -158,14 +162,39 @@ pub mod port { } impl HistoricalFeesProvider

{ - pub async fn calculate_sma(&self, last_n_blocks: u32) -> Fees { - let fees = self.fees_provider.fees(0..=last_n_blocks as u64).await; - - eprintln!("got fees: {:?}", fees); + // TODO: segfault fail or signal if missing blocks/holes present + // TODO: segfault cache fees/save to db + // TODO: segfault job to update fees in the background + pub async fn calculate_sma(&self, last_n_blocks: u64) -> Fees { + let current_height = self.fees_provider.current_block_height().await; + + let starting_block = current_height.saturating_sub(last_n_blocks.saturating_sub(1)); + let fees = self + .fees_provider + .fees(starting_block..=current_height) + .await; + + Self::mean(&fees) + } - let a = fees.last().fees; - eprintln!("got fees: {:?}", a); - a + fn mean(fees: &NonEmpty) -> Fees { + let total = fees + .iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + let count = fees.len() as u128; + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + } } } } diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index e8a6b3f9..0a49d98e 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,13 +1,14 @@ use std::{ops::RangeInclusive, time::Duration}; -use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider}; +use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider, Fees}; +use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use nonempty::NonEmpty; use rayon::range_inclusive; #[derive(Debug, Clone, Copy)] pub struct Config { - pub short_term_sma_num_blocks: u32, - pub long_term_sma_num_blocks: u32, + pub short_term_sma_num_blocks: u64, + pub long_term_sma_num_blocks: u64, } pub struct SendOrWaitDecider

{ @@ -25,7 +26,7 @@ impl

SendOrWaitDecider

{ } impl SendOrWaitDecider

{ - pub async fn should_send_blob_tx(&self) -> bool { + pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { let short_term_sma = self .price_service .calculate_sma(self.config.short_term_sma_num_blocks) @@ -36,7 +37,25 @@ impl SendOrWaitDecider

{ .calculate_sma(self.config.long_term_sma_num_blocks) .await; - return short_term_sma.base_fee_per_gas < long_term_sma.base_fee_per_gas; + let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + eprintln!( + "Short term: {:?}, Long term: {:?}, Short term tx price: {}, Long term tx price: {}", + short_term_sma, long_term_sma, short_term_tx_price, long_term_tx_price + ); + + short_term_tx_price < long_term_tx_price + } + + // TODO: Segfault maybe dont leak so much eth abstractions + fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { + const DATA_GAS_PER_BLOB: u128 = 131_072u128; + const INTRINSIC_GAS: u128 = 21_000u128; + + let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; + let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; + + base_fee + blob_fee + fees.reward } } @@ -57,22 +76,362 @@ mod tests { use super::*; #[tokio::test] - async fn should_send_if_shortterm_sma_lower_than_longterm_sma() { + async fn should_send_if_shortterm_prices_lower_than_longterm_ones() { // given let config = Config { - short_term_sma_num_blocks: 5, - long_term_sma_num_blocks: 50, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ]; - let fees_provider = TestFeesProvider::new(incrementing_fees(50)); + let fees_provider = TestFeesProvider::new(fees); let price_service = HistoricalFeesProvider::new(fees_provider); + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 40, + reward: 40, + base_fee_per_blob_gas: 40 + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 20 + } + ); + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; // when - let decision = sut.should_send_blob_tx().await; + let decision = sut.should_send_blob_tx(num_blobs).await; // then assert!(decision, "Should have sent"); } + + #[tokio::test] + async fn wont_send_because_normal_gas_too_expensive() { + // given + let config = Config { + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 10000, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 10000, + reward: 20, + base_fee_per_blob_gas: 20, + }, + ), + ]; + + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 3366, + reward: 40, + base_fee_per_blob_gas: 40 + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 10000, + reward: 20, + base_fee_per_blob_gas: 20 + } + ); + + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; + + // when + let decision = sut.should_send_blob_tx(num_blobs).await; + + // then + assert!(!decision, "Should not have sent"); + } + + #[tokio::test] + async fn wont_send_because_blob_gas_too_expensive() { + // given + let config = Config { + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 1000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 1000, + }, + ), + ]; + + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 40, + reward: 40, + base_fee_per_blob_gas: 366 + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 20, + reward: 20, + base_fee_per_blob_gas: 1000 + } + ); + + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; + + // when + let decision = sut.should_send_blob_tx(num_blobs).await; + + // then + assert!(!decision, "Should not have sent"); + } + + #[tokio::test] + async fn wont_send_because_rewards_too_expensive() { + // given + let config = Config { + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + }; + let fees = [ + ( + 1, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 20, + reward: 100_000_000, + base_fee_per_blob_gas: 20, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 20, + reward: 100_000_000, + base_fee_per_blob_gas: 20, + }, + ), + ]; + + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + + let short_sma = price_service.calculate_sma(2).await; + let long_sma = price_service.calculate_sma(6).await; + assert_eq!( + long_sma, + Fees { + base_fee_per_gas: 40, + reward: 33333366, + base_fee_per_blob_gas: 40, + } + ); + assert_eq!( + short_sma, + Fees { + base_fee_per_gas: 20, + reward: 100_000_000, + base_fee_per_blob_gas: 20 + } + ); + + let sut = SendOrWaitDecider::new(price_service, config); + let num_blobs = 6; + + // when + let decision = sut.should_send_blob_tx(num_blobs).await; + + // then + assert!(!decision, "Should not have sent"); + } } diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs index 125fd01c..a8317561 100644 --- a/packages/services/tests/historical_fees.rs +++ b/packages/services/tests/historical_fees.rs @@ -11,7 +11,7 @@ use services::{ }; #[tokio::test] -async fn calculates_sma_correctly_for_last_1_lock() { +async fn calculates_sma_correctly_for_last_1_block() { // given let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); let price_service = HistoricalFeesProvider::new(fees_provider); @@ -25,3 +25,20 @@ async fn calculates_sma_correctly_for_last_1_lock() { assert_eq!(sma.reward, 5); assert_eq!(sma.base_fee_per_blob_gas, 5); } + +#[tokio::test] +async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let price_service = HistoricalFeesProvider::new(fees_provider); + let last_n_blocks = 5; + + // when + let sma = price_service.calculate_sma(last_n_blocks).await; + + // then + let mean = (5 + 4 + 3 + 2 + 1) / 5; + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); +} From f7cf478e4338937f01df33cdd069c1df1b369336 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 13:50:26 +0100 Subject: [PATCH 03/47] cleanup tests --- packages/services/src/historical_fees.rs | 2 +- .../src/state_committer/fee_optimization.rs | 351 ++++-------------- packages/services/tests/historical_fees.rs | 12 +- 3 files changed, 76 insertions(+), 289 deletions(-) diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/historical_fees.rs index bc43b279..416e9902 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/historical_fees.rs @@ -146,7 +146,7 @@ pub mod port { } pub mod service { - use std::ops::RangeInclusive; + use nonempty::NonEmpty; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 0a49d98e..d9a0bbeb 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,9 +1,4 @@ -use std::{ops::RangeInclusive, time::Duration}; - use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider, Fees}; -use alloy::eips::eip4844::DATA_GAS_PER_BLOB; -use nonempty::NonEmpty; -use rayon::range_inclusive; #[derive(Debug, Clone, Copy)] pub struct Config { @@ -39,10 +34,6 @@ impl SendOrWaitDecider

{ let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - eprintln!( - "Short term: {:?}, Long term: {:?}, Short term tx price: {}, Long term tx price: {}", - short_term_sma, long_term_sma, short_term_tx_price, long_term_tx_price - ); short_term_tx_price < long_term_tx_price } @@ -61,20 +52,22 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { - use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - time::Duration, - }; - - use crate::historical_fees::port::{ - l1::testing::{incrementing_fees, TestFeesProvider}, - BlockFees, Fees, - }; - use crate::types::CollectNonEmpty; + use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; use super::*; + fn constant_fees(num_blocks: u64, fees: Fees) -> Vec<(u64, Fees)> { + (0..=num_blocks).map(|height| (height, fees)).collect() + } + + fn update_last_n_fees(fees: &mut [(u64, Fees)], num_latest: u64, updated_fees: Fees) { + fees.iter_mut() + .rev() + .take(num_latest as usize) + .for_each(|(_, f)| *f = updated_fees); + } + #[tokio::test] async fn should_send_if_shortterm_prices_lower_than_longterm_ones() { // given @@ -82,79 +75,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ]; - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 40, - reward: 40, - base_fee_per_blob_gas: 40 - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 20, - base_fee_per_blob_gas: 20 - } + base_fee_per_blob_gas: 20, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; @@ -172,79 +115,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 10000, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 10000, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ), - ]; - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 3366, - reward: 40, - base_fee_per_blob_gas: 40 - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 10000, reward: 20, - base_fee_per_blob_gas: 20 - } + base_fee_per_blob_gas: 20, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; @@ -262,79 +155,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 1000, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 1000, - }, - ), - ]; - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 40, - reward: 40, - base_fee_per_blob_gas: 366 - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 20, - base_fee_per_blob_gas: 1000 - } + base_fee_per_blob_gas: 1000, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; @@ -352,79 +195,29 @@ mod tests { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = [ - ( - 1, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 2, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 3, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 4, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - ), - ( - 5, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 20, - }, - ), - ( - 6, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 20, - }, - ), - ]; - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - let short_sma = price_service.calculate_sma(2).await; - let long_sma = price_service.calculate_sma(6).await; - assert_eq!( - long_sma, + let mut fees = constant_fees( + config.long_term_sma_num_blocks, Fees { - base_fee_per_gas: 40, - reward: 33333366, - base_fee_per_blob_gas: 40, - } + base_fee_per_gas: 50, + reward: 50, + base_fee_per_blob_gas: 50, + }, ); - assert_eq!( - short_sma, + + update_last_n_fees( + &mut fees, + config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 100_000_000, - base_fee_per_blob_gas: 20 - } + base_fee_per_blob_gas: 20, + }, ); + let fees_provider = TestFeesProvider::new(fees); + let price_service = HistoricalFeesProvider::new(fees_provider); + let sut = SendOrWaitDecider::new(price_service, config); let num_blobs = 6; diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs index a8317561..b5fb6ba4 100644 --- a/packages/services/tests/historical_fees.rs +++ b/packages/services/tests/historical_fees.rs @@ -1,14 +1,8 @@ -use std::{collections::BTreeMap, ops::RangeInclusive}; -use nonempty::NonEmpty; -use services::{ - historical_fees::port::{ - l1::{testing, FeesProvider}, +use services::historical_fees::port::{ + l1::testing, service::HistoricalFeesProvider, - BlockFees, Fees, - }, - types::CollectNonEmpty, -}; + }; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { From b3ec73b4b997a1aa6c9de1c59b722595d6b142f0 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:11:58 +0100 Subject: [PATCH 04/47] shorten tests --- .../src/state_committer/fee_optimization.rs | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index d9a0bbeb..0bf719c6 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -61,6 +61,20 @@ mod tests { (0..=num_blocks).map(|height| (height, fees)).collect() } + fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (config.long_term_sma_num_blocks - config.short_term_sma_num_blocks) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, config.short_term_sma_num_blocks as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() + } + fn update_last_n_fees(fees: &mut [(u64, Fees)], num_latest: u64, updated_fees: Fees) { fees.iter_mut() .rev() @@ -76,18 +90,13 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 20, @@ -116,22 +125,17 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { - base_fee_per_gas: 10000, - reward: 20, - base_fee_per_blob_gas: 20, + base_fee_per_gas: 100, + reward: 100_000_000, + base_fee_per_blob_gas: 100, }, ); @@ -156,22 +160,17 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 1000, + reward: 100_000_000, + base_fee_per_blob_gas: 100, }, ); @@ -196,18 +195,13 @@ mod tests { long_term_sma_num_blocks: 6, }; - let mut fees = constant_fees( - config.long_term_sma_num_blocks, + let fees = generate_fees( + config, Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50, }, - ); - - update_last_n_fees( - &mut fees, - config.short_term_sma_num_blocks, Fees { base_fee_per_gas: 20, reward: 100_000_000, From 6599fcfaa9b312896e817c9813120fd4131c6f10 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:17:38 +0100 Subject: [PATCH 05/47] parameterized tests --- .../src/state_committer/fee_optimization.rs | 168 ++++-------------- 1 file changed, 33 insertions(+), 135 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 0bf719c6..a05f5624 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -21,6 +21,7 @@ impl

SendOrWaitDecider

{ } impl SendOrWaitDecider

{ + // TODO: segfault validate blob number pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { let short_term_sma = self .price_service @@ -52,14 +53,9 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { - - use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; - use super::*; - - fn constant_fees(num_blocks: u64, fees: Fees) -> Vec<(u64, Fees)> { - (0..=num_blocks).map(|height| (height, fees)).collect() - } + use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; + use test_case::test_case; fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( @@ -75,35 +71,39 @@ mod tests { .collect() } - fn update_last_n_fees(fees: &mut [(u64, Fees)], num_latest: u64, updated_fees: Fees) { - fees.iter_mut() - .rev() - .take(num_latest as usize) - .for_each(|(_, f)| *f = updated_fees); - } - #[tokio::test] - async fn should_send_if_shortterm_prices_lower_than_longterm_ones() { + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, + true; "Should send because short-term prices lower than long-term" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 10_000, reward: 20, base_fee_per_blob_gas: 20 }, + false; "Wont send because normal gas too expensive" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 1000 }, + false; "Wont send because blob gas too expensive" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 20, reward: 100_000_000, base_fee_per_blob_gas: 20 }, + false; "Wont send because rewards too expensive" + )] + async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + expected_decision: bool, + ) { // given let config = Config { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 20, - reward: 20, - base_fee_per_blob_gas: 20, - }, - ); - + let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); let price_service = HistoricalFeesProvider::new(fees_provider); @@ -111,114 +111,12 @@ mod tests { let num_blobs = 6; // when - let decision = sut.should_send_blob_tx(num_blobs).await; + let should_send = sut.should_send_blob_tx(num_blobs).await; // then - assert!(decision, "Should have sent"); - } - - #[tokio::test] - async fn wont_send_because_normal_gas_too_expensive() { - // given - let config = Config { - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - }; - - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 100, - reward: 100_000_000, - base_fee_per_blob_gas: 100, - }, + assert_eq!( + should_send, expected_decision, + "Expected decision: {expected_decision}, got: {should_send}", ); - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; - - // when - let decision = sut.should_send_blob_tx(num_blobs).await; - - // then - assert!(!decision, "Should not have sent"); - } - - #[tokio::test] - async fn wont_send_because_blob_gas_too_expensive() { - // given - let config = Config { - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - }; - - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 100, - }, - ); - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; - - // when - let decision = sut.should_send_blob_tx(num_blobs).await; - - // then - assert!(!decision, "Should not have sent"); - } - - #[tokio::test] - async fn wont_send_because_rewards_too_expensive() { - // given - let config = Config { - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - }; - - let fees = generate_fees( - config, - Fees { - base_fee_per_gas: 50, - reward: 50, - base_fee_per_blob_gas: 50, - }, - Fees { - base_fee_per_gas: 20, - reward: 100_000_000, - base_fee_per_blob_gas: 20, - }, - ); - - let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); - - let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; - - // when - let decision = sut.should_send_blob_tx(num_blobs).await; - - // then - assert!(!decision, "Should not have sent"); } } From 281086b76c57f38a1a74f80d22144431814e6b62 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:31:00 +0100 Subject: [PATCH 06/47] add more tests --- .../src/state_committer/fee_optimization.rs | 52 ++++++++++++++++--- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index a05f5624..a854ff09 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -73,28 +73,65 @@ mod tests { #[tokio::test] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, - true; "Should send because short-term prices lower than long-term" + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, // Old fees + Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, // New fees + 6, // num_blobs + true; + "Should send because all short-term prices lower than long-term" )] #[test_case( Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, Fees { base_fee_per_gas: 10_000, reward: 20, base_fee_per_blob_gas: 20 }, - false; "Wont send because normal gas too expensive" + 6, + false; + "Wont send because normal gas too expensive" )] #[test_case( Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 1000 }, - false; "Wont send because blob gas too expensive" + 6, + false; + "Wont send because blob gas too expensive" )] #[test_case( Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, Fees { base_fee_per_gas: 20, reward: 100_000_000, base_fee_per_blob_gas: 20 }, - false; "Wont send because rewards too expensive" + 6, + false; + "Wont send because rewards too expensive" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + 1, + false; + "Fees identical" + )] + #[test_case( + Fees { base_fee_per_gas: 500, reward: 500, base_fee_per_blob_gas: 500 }, + Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, + 6, + true; + "6 blobs significantly cheaper in short-term" + )] + #[test_case( + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, + Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 70 }, + 6, + false; + "6 blobs slightly more expensive blob gas in short-term" + )] + #[test_case( + Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, + Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, + 5, + false; + "Five blobs but identical fees" )] async fn parameterized_send_or_wait_tests( old_fees: Fees, new_fees: Fees, + num_blobs: u32, expected_decision: bool, ) { // given @@ -108,7 +145,6 @@ mod tests { let price_service = HistoricalFeesProvider::new(fees_provider); let sut = SendOrWaitDecider::new(price_service, config); - let num_blobs = 6; // when let should_send = sut.should_send_blob_tx(num_blobs).await; @@ -116,7 +152,7 @@ mod tests { // then assert_eq!( should_send, expected_decision, - "Expected decision: {expected_decision}, got: {should_send}", + "For num_blobs={num_blobs}: Expected decision: {expected_decision}, got: {should_send}", ); } } From 8b74db3dfeed473bfcd3fc58086c2cac5ad14773 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:35:12 +0100 Subject: [PATCH 07/47] more edge cases --- .../src/state_committer/fee_optimization.rs | 105 +++++++++++------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index a854ff09..67c77e59 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -57,6 +57,7 @@ mod tests { use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; use test_case::test_case; + // Function to generate historical fees data fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, @@ -73,60 +74,88 @@ mod tests { #[tokio::test] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, // Old fees - Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 20 }, // New fees - 6, // num_blobs + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees + 10, // num_blobs true; - "Should send because all short-term prices lower than long-term" + "Should send because all short-term fees are lower than long-term" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 10_000, reward: 20, base_fee_per_blob_gas: 20 }, - 6, - false; - "Wont send because normal gas too expensive" + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + 5, + true; + "Should send because short-term base_fee_per_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 20, reward: 20, base_fee_per_blob_gas: 1000 }, - 6, - false; - "Wont send because blob gas too expensive" + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + 5, + false; + "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 20, reward: 100_000_000, base_fee_per_blob_gas: 20 }, - 6, - false; - "Wont send because rewards too expensive" + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, // New fees + 5, + true; + "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - 1, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, // New fees + 5, false; - "Fees identical" + "Should not send because short-term base_fee_per_blob_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 500, reward: 500, base_fee_per_blob_gas: 500 }, - Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, - 6, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, // New fees + 5, true; - "6 blobs significantly cheaper in short-term" + "Should send because short-term reward is lower" )] #[test_case( - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 50 }, - Fees { base_fee_per_gas: 50, reward: 50, base_fee_per_blob_gas: 70 }, - 6, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, // New fees + 5, false; - "6 blobs slightly more expensive blob gas in short-term" + "Should not send because short-term reward is higher" )] #[test_case( - Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, - Fees { base_fee_per_gas: 100, reward: 100, base_fee_per_blob_gas: 100 }, - 5, + Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, // Old fees + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, // New fees + 10, + true; + "Should send because multiple short-term fees are lower" + )] + #[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // New fees + 10, false; - "Five blobs but identical fees" + "Should not send because all fees are identical" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, // New fees + 0, + true; + "Zero blobs but short-term base_fee_per_gas and reward are lower" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, // New fees + 0, + false; + "Zero blobs but short-term reward is higher" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees + Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, // New fees + 0, + true; + "Zero blobs dont care about higher short-term base_fee_per_blob_gas" )] async fn parameterized_send_or_wait_tests( old_fees: Fees, @@ -134,7 +163,7 @@ mod tests { num_blobs: u32, expected_decision: bool, ) { - // given + // Given let config = Config { short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, @@ -146,10 +175,10 @@ mod tests { let sut = SendOrWaitDecider::new(price_service, config); - // when + // When let should_send = sut.should_send_blob_tx(num_blobs).await; - // then + // Then assert_eq!( should_send, expected_decision, "For num_blobs={num_blobs}: Expected decision: {expected_decision}, got: {should_send}", From cc451cfe643763c0aa480d64df2f6ecfd84343d1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:42:16 +0100 Subject: [PATCH 08/47] use sequential fees struct, rename price service to fee analytics --- .../{historical_fees.rs => fee_analytics.rs} | 66 +++++++++++-------- packages/services/src/lib.rs | 2 +- .../src/state_committer/fee_optimization.rs | 18 ++--- packages/services/tests/historical_fees.rs | 14 ++-- 4 files changed, 52 insertions(+), 48 deletions(-) rename packages/services/src/{historical_fees.rs => fee_analytics.rs} (89%) diff --git a/packages/services/src/historical_fees.rs b/packages/services/src/fee_analytics.rs similarity index 89% rename from packages/services/src/historical_fees.rs rename to packages/services/src/fee_analytics.rs index 416e9902..50ae41c0 100644 --- a/packages/services/src/historical_fees.rs +++ b/packages/services/src/fee_analytics.rs @@ -80,7 +80,7 @@ pub mod port { #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] pub trait FeesProvider { - async fn fees(&self, height_range: RangeInclusive) -> NonEmpty; + async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees; async fn current_block_height(&self) -> u64; } @@ -88,14 +88,15 @@ pub mod port { pub mod testing { use std::{collections::BTreeMap, ops::RangeInclusive}; + use itertools::Itertools; use nonempty::NonEmpty; use crate::{ - historical_fees::port::{BlockFees, Fees}, + fee_analytics::port::{BlockFees, Fees}, types::CollectNonEmpty, }; - use super::FeesProvider; + use super::{FeesProvider, SequentialBlockFees}; pub struct TestFeesProvider { fees: BTreeMap, @@ -106,8 +107,9 @@ pub mod port { *self.fees.keys().last().unwrap() } - async fn fees(&self, height_range: RangeInclusive) -> NonEmpty { - self.fees + async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + let fees = self + .fees .iter() .skip_while(|(height, _)| !height_range.contains(height)) .take_while(|(height, _)| height_range.contains(height)) @@ -115,8 +117,9 @@ pub mod port { height: *height, fees: *fees, }) - .collect_nonempty() - .unwrap() + .collect_vec(); + + fees.try_into().unwrap() } } @@ -146,22 +149,24 @@ pub mod port { } pub mod service { - use nonempty::NonEmpty; - use super::{l1::FeesProvider, BlockFees, Fees}; + use super::{ + l1::{FeesProvider, SequentialBlockFees}, + BlockFees, Fees, + }; - pub struct HistoricalFeesProvider

{ + pub struct FeeAnalytics

{ fees_provider: P, } - impl

HistoricalFeesProvider

{ + impl

FeeAnalytics

{ pub fn new(fees_provider: P) -> Self { Self { fees_provider } } } - impl HistoricalFeesProvider

{ + impl FeeAnalytics

{ // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees/save to db // TODO: segfault job to update fees in the background @@ -174,12 +179,14 @@ pub mod port { .fees(starting_block..=current_height) .await; - Self::mean(&fees) + Self::mean(fees) } - fn mean(fees: &NonEmpty) -> Fees { + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + let total = fees - .iter() + .into_iter() .map(|bf| bf.fees) .fold(Fees::default(), |acc, f| Fees { base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, @@ -187,8 +194,6 @@ pub mod port { base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, }); - let count = fees.len() as u128; - // TODO: segfault should we round to nearest here? Fees { base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), @@ -207,7 +212,7 @@ mod tests { use super::*; #[test] - fn given_sequential_block_fees_when_valid_then_creation_succeeds() { + fn can_create_valid_sequential_fees() { // Given let block_fees = vec![ BlockFees { @@ -241,7 +246,7 @@ mod tests { } #[test] - fn given_sequential_block_fees_when_empty_then_creation_fails() { + fn sequential_fees_cannot_be_empty() { // Given let block_fees: Vec = vec![]; @@ -260,7 +265,7 @@ mod tests { } #[test] - fn given_sequential_block_fees_when_non_sequential_then_creation_fails() { + fn fees_must_be_sequential() { // Given let block_fees = vec![ BlockFees { @@ -295,18 +300,13 @@ mod tests { ); } + // TODO: segfault add more tests so that the in-order iteration invariant is properly tested #[test] - fn given_sequential_block_fees_when_valid_then_can_iterate_over_fees() { + fn produced_iterator_gives_correct_values() { // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, BlockFees { height: 2, fees: Fees { @@ -315,6 +315,14 @@ mod tests { base_fee_per_blob_gas: 15, }, }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, ]; let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 29b18fbf..69e04048 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,8 +2,8 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; +pub mod fee_analytics; pub mod health_reporter; -pub mod historical_fees; pub mod state_committer; pub mod state_listener; pub mod state_pruner; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 67c77e59..37b4abc8 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,4 +1,4 @@ -use crate::historical_fees::port::{l1::FeesProvider, service::HistoricalFeesProvider, Fees}; +use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; #[derive(Debug, Clone, Copy)] pub struct Config { @@ -7,14 +7,14 @@ pub struct Config { } pub struct SendOrWaitDecider

{ - price_service: HistoricalFeesProvider

, + fee_analytics: FeeAnalytics

, config: Config, } impl

SendOrWaitDecider

{ - pub fn new(price_service: HistoricalFeesProvider

, config: Config) -> Self { + pub fn new(fee_analytics: FeeAnalytics

, config: Config) -> Self { Self { - price_service, + fee_analytics, config, } } @@ -24,12 +24,12 @@ impl SendOrWaitDecider

{ // TODO: segfault validate blob number pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { let short_term_sma = self - .price_service + .fee_analytics .calculate_sma(self.config.short_term_sma_num_blocks) .await; let long_term_sma = self - .price_service + .fee_analytics .calculate_sma(self.config.long_term_sma_num_blocks) .await; @@ -54,7 +54,7 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { use super::*; - use crate::historical_fees::port::{l1::testing::TestFeesProvider, Fees}; + use crate::fee_analytics::port::{l1::testing::TestFeesProvider, Fees}; use test_case::test_case; // Function to generate historical fees data @@ -171,9 +171,9 @@ mod tests { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); - let price_service = HistoricalFeesProvider::new(fees_provider); + let analytics_service = FeeAnalytics::new(fees_provider); - let sut = SendOrWaitDecider::new(price_service, config); + let sut = SendOrWaitDecider::new(analytics_service, config); // When let should_send = sut.should_send_blob_tx(num_blobs).await; diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/historical_fees.rs index b5fb6ba4..8f6c9d48 100644 --- a/packages/services/tests/historical_fees.rs +++ b/packages/services/tests/historical_fees.rs @@ -1,18 +1,14 @@ - -use services::historical_fees::port::{ - l1::testing, - service::HistoricalFeesProvider, - }; +use services::fee_analytics::port::{l1::testing, service::FeeAnalytics}; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { // given let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); - let price_service = HistoricalFeesProvider::new(fees_provider); + let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 1; // when - let sma = price_service.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(last_n_blocks).await; // then assert_eq!(sma.base_fee_per_gas, 5); @@ -24,11 +20,11 @@ async fn calculates_sma_correctly_for_last_1_block() { async fn calculates_sma_correctly_for_last_5_blocks() { // given let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); - let price_service = HistoricalFeesProvider::new(fees_provider); + let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 5; // when - let sma = price_service.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(last_n_blocks).await; // then let mean = (5 + 4 + 3 + 2 + 1) / 5; From fea814ae3edfa1fdcd4ba0620f6d1fa4eda4f5eb Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 14:53:40 +0100 Subject: [PATCH 09/47] add activation fee threshold --- .../src/state_committer/fee_optimization.rs | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 37b4abc8..870b7470 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -2,6 +2,7 @@ use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; #[derive(Debug, Clone, Copy)] pub struct Config { + pub sma_activation_fee_treshold: u128, pub short_term_sma_num_blocks: u64, pub long_term_sma_num_blocks: u64, } @@ -34,6 +35,10 @@ impl SendOrWaitDecider

{ .await; let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + if short_term_tx_price < self.config.sma_activation_fee_treshold { + return true; + } + let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); short_term_tx_price < long_term_tx_price @@ -72,18 +77,35 @@ mod tests { .collect() } - #[tokio::test] #[test_case( Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees - 10, // num_blobs + 6, + 0, true; "Should send because all short-term fees are lower than long-term" )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + 6, + 0, + false; + "Should not send because all short-term fees are higher than long-term" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + 6, + 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + true; + "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" + )] #[test_case( Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, true; "Should send because short-term base_fee_per_gas is lower" )] @@ -91,6 +113,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, false; "Should not send because short-term base_fee_per_gas is higher" )] @@ -98,6 +121,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, // New fees 5, + 0, true; "Should send because short-term base_fee_per_blob_gas is lower" )] @@ -105,6 +129,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, // New fees 5, + 0, false; "Should not send because short-term base_fee_per_blob_gas is higher" )] @@ -112,6 +137,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, true; "Should send because short-term reward is lower" )] @@ -119,20 +145,23 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, // New fees 5, + 0, false; "Should not send because short-term reward is higher" )] #[test_case( Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, // Old fees Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, // New fees - 10, + 6, + 0, true; "Should send because multiple short-term fees are lower" )] #[test_case( Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // New fees - 10, + 6, + 0, false; "Should not send because all fees are identical" )] @@ -140,6 +169,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, // New fees 0, + 0, true; "Zero blobs but short-term base_fee_per_gas and reward are lower" )] @@ -147,6 +177,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, // New fees 0, + 0, false; "Zero blobs but short-term reward is higher" )] @@ -154,17 +185,21 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, // New fees 0, + 0, true; "Zero blobs dont care about higher short-term base_fee_per_blob_gas" )] + #[tokio::test] async fn parameterized_send_or_wait_tests( old_fees: Fees, new_fees: Fees, num_blobs: u32, + activation_fee_treshold: u128, expected_decision: bool, ) { // Given let config = Config { + sma_activation_fee_treshold: activation_fee_treshold, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, }; From 3bfcca0afe4228ea0a2a936aa9fe9884a94a9f3c Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 15:19:35 +0100 Subject: [PATCH 10/47] add comparison strategy --- .../src/state_committer/fee_optimization.rs | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 870b7470..ff604efc 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,10 +1,26 @@ use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; +// TODO: segfault validate percentages +#[derive(Debug, Clone, Copy)] +pub enum ComparisonStrategy { + /// Short-term fee must be at most (1 - percentage) of the long-term fee. + /// For example, if percentage = 0.1 (10%), then: + /// short_term ≤ long_term * 0.9. + StrictlyLessOrEqualByPercent(f64), + + /// Short-term fee may be more expensive, but not by more than the given percentage. + /// For example, if percentage = 0.1 (10%), then: + /// short_term ≤ long_term * 1.1. + /// Short-term can be cheaper by any amount. + WithinVicinityOfPriceByPercent(f64), +} + #[derive(Debug, Clone, Copy)] pub struct Config { pub sma_activation_fee_treshold: u128, pub short_term_sma_num_blocks: u64, pub long_term_sma_num_blocks: u64, + pub comparison_strategy: ComparisonStrategy, } pub struct SendOrWaitDecider

{ @@ -35,13 +51,30 @@ impl SendOrWaitDecider

{ .await; let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + if short_term_tx_price < self.config.sma_activation_fee_treshold { return true; } let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - short_term_tx_price < long_term_tx_price + let percentage = match self.config.comparison_strategy { + ComparisonStrategy::StrictlyLessOrEqualByPercent(p) => 1.0 - p, + ComparisonStrategy::WithinVicinityOfPriceByPercent(p) => 1.0 + p, + }; + + // TODO: segfault proper type conversions + let allowed_max = (long_term_tx_price as f64 * percentage) as u128; + + eprintln!( + "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", + short_term_tx_price, + long_term_tx_price, + allowed_max, + short_term_tx_price.saturating_sub(allowed_max) + ); + + short_term_tx_price <= allowed_max } // TODO: Segfault maybe dont leak so much eth abstractions @@ -82,7 +115,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees 6, 0, - true; + true; "Should send because all short-term fees are lower than long-term" )] #[test_case( @@ -90,7 +123,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees 6, 0, - false; + false; "Should not send because all short-term fees are higher than long-term" )] #[test_case( @@ -98,7 +131,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees 6, 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, - true; + true; "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" )] #[test_case( @@ -202,6 +235,7 @@ mod tests { sma_activation_fee_treshold: activation_fee_treshold, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0), }; let fees = generate_fees(config, old_fees, new_fees); From 7e5769f927514063d59706b629636e09b381b2ce Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 15:26:19 +0100 Subject: [PATCH 11/47] add tests for treshold strategy --- .../src/state_committer/fee_optimization.rs | 247 +++++++++++++----- 1 file changed, 188 insertions(+), 59 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index ff604efc..06a9ee7c 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -63,18 +63,18 @@ impl SendOrWaitDecider

{ ComparisonStrategy::WithinVicinityOfPriceByPercent(p) => 1.0 + p, }; - // TODO: segfault proper type conversions - let allowed_max = (long_term_tx_price as f64 * percentage) as u128; + // TODO: segfault proper type conversions, probably max(,,) - min(,,) <= delta + let treshold = (long_term_tx_price as f64 * percentage) as u128; eprintln!( "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", short_term_tx_price, long_term_tx_price, - allowed_max, - short_term_tx_price.saturating_sub(allowed_max) + treshold, + short_term_tx_price.saturating_sub(treshold) ); - short_term_tx_price <= allowed_max + short_term_tx_price < treshold } // TODO: Segfault maybe dont leak so much eth abstractions @@ -111,146 +111,275 @@ mod tests { } #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // New fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because all short-term fees are lower than long-term" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because all short-term fees are higher than long-term" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, // Old fees - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 },// New fees + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + Config { + sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because short-term base_fee_per_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because short-term base_fee_per_blob_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because short-term reward is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, // New fees + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because short-term reward is higher" )] #[test_case( - Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, // Old fees - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, // New fees + Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Should send because multiple short-term fees are lower" )] #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, // New fees + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Should not send because all fees are identical" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, // New fees - 0, + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Zero blobs but short-term base_fee_per_gas and reward are lower" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, // New fees - 0, + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, false; "Zero blobs but short-term reward is higher" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, // Old fees - Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, // New fees - 0, + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, true; "Zero blobs dont care about higher short-term base_fee_per_blob_gas" )] + // New Tests (Introducing other Comparison Strategies) + #[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.1) + }, + true; + "Should send (cheaper) with StrictlyLessOrEqualByPercent(10%)" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + // Strictly less or equal by 0% means must be cheaper or equal, which it's not + comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + }, + false; + "Should not send (more expensive) with StrictlyLessOrEqualByPercent(0%)" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + // Below threshold means we send anyway + sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + }, + true; + "Below activation threshold, send anyway for WithinVicinityOfPriceByPercent(10%)" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 3200, reward: 5000, base_fee_per_blob_gas: 3200 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + }, + true; + "Within vicinity of price by 10%: short_term slightly more expensive but allowed" + )] + #[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_activation_fee_treshold: 0, + short_term_sma_num_blocks: 2, + long_term_sma_num_blocks: 6, + comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.0) + }, + false; + "Within vicinity with 0% means must not exceed long-term, not sending" + )] #[tokio::test] async fn parameterized_send_or_wait_tests( old_fees: Fees, new_fees: Fees, num_blobs: u32, - activation_fee_treshold: u128, + config: Config, expected_decision: bool, ) { - // Given - let config = Config { - sma_activation_fee_treshold: activation_fee_treshold, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0), - }; - let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); let analytics_service = FeeAnalytics::new(fees_provider); let sut = SendOrWaitDecider::new(analytics_service, config); - // When let should_send = sut.should_send_blob_tx(num_blobs).await; - // Then assert_eq!( should_send, expected_decision, - "For num_blobs={num_blobs}: Expected decision: {expected_decision}, got: {should_send}", + "For num_blobs={num_blobs}, config={:?}: Expected decision: {}, got: {}", + config, expected_decision, should_send ); } } From 855ac2603bf21ddeb19985d39c2e9625497d69fa Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 16:59:14 +0100 Subject: [PATCH 12/47] l1 adapter for historical fees, about to test out algo on historical data --- Cargo.lock | 1 + packages/adapters/eth/Cargo.toml | 1 + packages/adapters/eth/src/lib.rs | 134 +++++++++++++++++- packages/adapters/eth/src/websocket.rs | 11 +- .../adapters/eth/src/websocket/connection.rs | 20 ++- .../websocket/health_tracking_middleware.rs | 18 ++- .../{historical_fees.rs => fee_analytics.rs} | 0 7 files changed, 177 insertions(+), 8 deletions(-) rename packages/services/tests/{historical_fees.rs => fee_analytics.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 6b99e671..9b64b27d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2673,6 +2673,7 @@ dependencies = [ "serde", "serde_json", "services", + "static_assertions", "test-case", "thiserror 1.0.69", "tokio", diff --git a/packages/adapters/eth/Cargo.toml b/packages/adapters/eth/Cargo.toml index 6c18d2c9..0ab31d3b 100644 --- a/packages/adapters/eth/Cargo.toml +++ b/packages/adapters/eth/Cargo.toml @@ -21,6 +21,7 @@ alloy = { workspace = true, features = [ "rpc-types", "reqwest-rustls-tls", ] } +static_assertions = { workspace = true } async-trait = { workspace = true } aws-config = { workspace = true, features = ["default"] } aws-sdk-kms = { workspace = true, features = ["default"] } diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 38e069fc..2836fb33 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -1,13 +1,19 @@ -use std::num::{NonZeroU32, NonZeroUsize}; +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroUsize}, + ops::RangeInclusive, +}; use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, + providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE, }; use delegate::delegate; -use itertools::{izip, Itertools}; +use itertools::{izip, zip, Itertools}; use services::{ + fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -23,6 +29,7 @@ mod websocket; pub use alloy::primitives::Address; pub use aws::*; use fuel_block_committer_encoding::blob::{self, generate_sidecar}; +use static_assertions::const_assert; pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; #[derive(Debug, Copy, Clone)] @@ -186,13 +193,101 @@ impl services::state_committer::port::l1::Api for WebsocketClient { } } +impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { + async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + const REWARD_PERCENTILE: f64 = + alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; + + // so that a alloy version bump doesn't surprise us + const_assert!(REWARD_PERCENTILE == 20.0,); + + let mut fees = vec![]; + + // TODO: segfault see when this can be None + // TODO: check edgecases + let mut current_height = height_range.clone().min().unwrap(); + while current_height <= *height_range.end() { + // There is a comment in alloy about not doing more than 1024 blocks at a time + const RPC_LIMIT: u64 = 1024; + + let upper_bound = min( + current_height.saturating_add(RPC_LIMIT), + *height_range.end(), + ); + + let history = self + .fees( + current_height..=upper_bound, + std::slice::from_ref(&REWARD_PERCENTILE), + ) + .await + .unwrap(); + + fees.push(history); + + current_height = upper_bound.saturating_add(1); + } + + let new_fees = fees + .into_iter() + .flat_map(|fees| { + // TODO: segfault check if the vector is ever going to have less than 2 elements, maybe + // for block count 0? + eprintln!("received {fees:?}"); + let number_of_blocks = fees.base_fee_per_blob_gas.len().checked_sub(1).unwrap(); + let rewards = fees + .reward + .unwrap() + .into_iter() + .map(|mut perc| perc.pop().unwrap()) + .collect_vec(); + + let oldest_block = fees.oldest_block; + + debug_assert_eq!(rewards.len(), number_of_blocks); + + izip!( + (oldest_block..), + fees.base_fee_per_gas.into_iter(), + fees.base_fee_per_blob_gas.into_iter(), + rewards + ) + .take(number_of_blocks) + .map( + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, + }, + ) + }) + .collect_vec(); + + eprintln!("converted into {new_fees:?}"); + + new_fees.try_into().unwrap() + } + + async fn current_block_height(&self) -> u64 { + self._get_block_number().await.unwrap() + } +} + #[cfg(test)] mod test { + use std::time::Duration; + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; - use services::block_bundler::port::l1::FragmentEncoder; + use services::{ + block_bundler::port::l1::FragmentEncoder, block_committer::port::l1::Api, + fee_analytics::port::l1::FeesProvider, + }; - use crate::BlobEncoder; + use crate::{BlobEncoder, Signer, Signers}; #[test] fn gas_usage_correctly_calculated() { @@ -207,4 +302,35 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } + + #[tokio::test] + async fn can_connect_to_eth_mainnet() { + let signers = Signers::for_keys(crate::L1Keys { + main: crate::L1Key::Private( + "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), + ), + blob: None, + }) + .await + .unwrap(); + + let client = crate::WebsocketClient::connect( + "wss://ethereum-rpc.publicnode.com".parse().unwrap(), + Default::default(), + signers, + 10, + crate::TxConfig { + tx_max_fee: u128::MAX, + send_tx_request_timeout: Duration::MAX, + }, + ) + .await + .unwrap(); + + let current_height = client._get_block_number().await.unwrap(); + + let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; + + panic!("{:?}", fees); + } } diff --git a/packages/adapters/eth/src/websocket.rs b/packages/adapters/eth/src/websocket.rs index 9ccd8091..8b64038b 100644 --- a/packages/adapters/eth/src/websocket.rs +++ b/packages/adapters/eth/src/websocket.rs @@ -1,10 +1,11 @@ -use std::{num::NonZeroU32, str::FromStr, time::Duration}; +use std::{num::NonZeroU32, ops::RangeInclusive, str::FromStr, time::Duration}; use ::metrics::{prometheus::core::Collector, HealthChecker, RegistersMetrics}; use alloy::{ consensus::SignableTransaction, network::TxSigner, primitives::{Address, ChainId, B256}, + rpc::types::FeeHistory, signers::{local::PrivateKeySigner, Signature}, }; use serde::Deserialize; @@ -225,6 +226,14 @@ impl WebsocketClient { Ok(self.inner.get_transaction_response(tx_hash).await?) } + pub(crate) async fn fees( + &self, + height_range: RangeInclusive, + rewards_percentile: &[f64], + ) -> Result { + Ok(self.inner.fees(height_range, rewards_percentile).await?) + } + pub(crate) async fn is_squeezed_out(&self, tx_hash: [u8; 32]) -> Result { Ok(self.inner.is_squeezed_out(tx_hash).await?) } diff --git a/packages/adapters/eth/src/websocket/connection.rs b/packages/adapters/eth/src/websocket/connection.rs index 8a718b9f..65318b49 100644 --- a/packages/adapters/eth/src/websocket/connection.rs +++ b/packages/adapters/eth/src/websocket/connection.rs @@ -1,6 +1,7 @@ use std::{ cmp::{max, min}, num::NonZeroU32, + ops::RangeInclusive, time::Duration, }; @@ -14,7 +15,7 @@ use alloy::{ primitives::{Address, U256}, providers::{utils::Eip1559Estimation, Provider, ProviderBuilder, SendableTx, WsConnect}, pubsub::PubSubFrontend, - rpc::types::{TransactionReceipt, TransactionRequest}, + rpc::types::{FeeHistory, TransactionReceipt, TransactionRequest}, sol, }; use itertools::Itertools; @@ -212,6 +213,19 @@ impl EthApi for WsConnection { Ok(submission_tx) } + async fn fees( + &self, + height_range: RangeInclusive, + reward_percentiles: &[f64], + ) -> Result { + let max = *height_range.end(); + let count = height_range.clone().count() as u64; + Ok(self + .provider + .get_fee_history(count, BlockNumberOrTag::Number(max), reward_percentiles) + .await?) + } + async fn get_block_number(&self) -> Result { let response = self.provider.get_block_number().await?; Ok(response) @@ -385,7 +399,9 @@ impl WsConnection { let contract_address = Address::from_slice(contract_address.as_ref()); let contract = FuelStateContract::new(contract_address, provider.clone()); - let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; + // TODO: segfault revert this + // let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; + let interval_u256 = 1u32; let commit_interval = u32::try_from(interval_u256) .map_err(|e| Error::Other(e.to_string())) diff --git a/packages/adapters/eth/src/websocket/health_tracking_middleware.rs b/packages/adapters/eth/src/websocket/health_tracking_middleware.rs index 21f5c3aa..93045485 100644 --- a/packages/adapters/eth/src/websocket/health_tracking_middleware.rs +++ b/packages/adapters/eth/src/websocket/health_tracking_middleware.rs @@ -1,8 +1,9 @@ -use std::num::NonZeroU32; +use std::{num::NonZeroU32, ops::RangeInclusive}; use ::metrics::{ prometheus::core::Collector, ConnectionHealthTracker, HealthChecker, RegistersMetrics, }; +use alloy::rpc::types::FeeHistory; use delegate::delegate; use services::types::{Address, BlockSubmissionTx, Fragment, NonEmpty, TransactionResponse, U256}; @@ -15,6 +16,11 @@ use crate::{ #[async_trait::async_trait] pub trait EthApi { async fn submit(&self, hash: [u8; 32], height: u32) -> Result; + async fn fees( + &self, + height_range: RangeInclusive, + reward_percentiles: &[f64], + ) -> Result; async fn get_block_number(&self) -> Result; async fn balance(&self, address: Address) -> Result; fn commit_interval(&self) -> NonZeroU32; @@ -117,6 +123,16 @@ where response } + async fn fees( + &self, + height_range: RangeInclusive, + reward_percentiles: &[f64], + ) -> Result { + let response = self.adapter.fees(height_range, reward_percentiles).await; + self.note_network_status(&response); + response + } + async fn is_squeezed_out(&self, tx_hash: [u8; 32]) -> Result { let response = self.adapter.is_squeezed_out(tx_hash).await; self.note_network_status(&response); diff --git a/packages/services/tests/historical_fees.rs b/packages/services/tests/fee_analytics.rs similarity index 100% rename from packages/services/tests/historical_fees.rs rename to packages/services/tests/fee_analytics.rs From c2f160143ab5bf51ffa94197fa803b1192c276a5 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sat, 14 Dec 2024 19:19:32 +0100 Subject: [PATCH 13/47] historical testing of algo --- Cargo.lock | 22 ++++ packages/adapters/eth/src/lib.rs | 73 ++++++------ packages/services/Cargo.toml | 1 + packages/services/src/fee_analytics.rs | 20 ++-- packages/services/src/state_committer.rs | 2 +- .../src/state_committer/fee_optimization.rs | 27 +++-- packages/services/tests/fee_analytics.rs | 111 +++++++++++++++++- 7 files changed, 197 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b64b27d..582429a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2146,6 +2146,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -5762,6 +5783,7 @@ dependencies = [ "async-trait", "bytesize", "clock", + "csv", "delegate", "eth", "fuel-block-committer-encoding", diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 2836fb33..17cdc6e4 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -2,6 +2,7 @@ use std::{ cmp::min, num::{NonZeroU32, NonZeroUsize}, ops::RangeInclusive, + time::Duration, }; use alloy::{ @@ -35,6 +36,29 @@ pub use websocket::{L1Key, L1Keys, Signer, Signers, TxConfig, WebsocketClient}; #[derive(Debug, Copy, Clone)] pub struct BlobEncoder; +pub async fn make_pub_eth_client() -> WebsocketClient { + let signers = Signers::for_keys(crate::L1Keys { + main: crate::L1Key::Private( + "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), + ), + blob: None, + }) + .await + .unwrap(); + + crate::WebsocketClient::connect( + "wss://ethereum-rpc.publicnode.com".parse().unwrap(), + Default::default(), + signers, + 10, + crate::TxConfig { + tx_max_fee: u128::MAX, + send_tx_request_timeout: Duration::MAX, + }, + ) + .await + .unwrap() +} impl BlobEncoder { #[cfg(feature = "test-helpers")] pub const FRAGMENT_SIZE: usize = BYTES_PER_BLOB; @@ -211,7 +235,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { const RPC_LIMIT: u64 = 1024; let upper_bound = min( - current_height.saturating_add(RPC_LIMIT), + current_height.saturating_add(RPC_LIMIT).saturating_sub(1), *height_range.end(), ); @@ -223,6 +247,11 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { .await .unwrap(); + assert_eq!( + history.reward.as_ref().unwrap().len(), + (current_height..=upper_bound).count() + ); + fees.push(history); current_height = upper_bound.saturating_add(1); @@ -233,7 +262,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { .flat_map(|fees| { // TODO: segfault check if the vector is ever going to have less than 2 elements, maybe // for block count 0? - eprintln!("received {fees:?}"); + // eprintln!("received {fees:?}"); let number_of_blocks = fees.base_fee_per_blob_gas.len().checked_sub(1).unwrap(); let rewards = fees .reward @@ -266,7 +295,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { }) .collect_vec(); - eprintln!("converted into {new_fees:?}"); + // eprintln!("converted into {new_fees:?}"); new_fees.try_into().unwrap() } @@ -303,34 +332,12 @@ mod test { assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } - #[tokio::test] - async fn can_connect_to_eth_mainnet() { - let signers = Signers::for_keys(crate::L1Keys { - main: crate::L1Key::Private( - "98d88144512cc5747fed20bdc81fb820c4785f7411bd65a88526f3b084dc931e".to_string(), - ), - blob: None, - }) - .await - .unwrap(); - - let client = crate::WebsocketClient::connect( - "wss://ethereum-rpc.publicnode.com".parse().unwrap(), - Default::default(), - signers, - 10, - crate::TxConfig { - tx_max_fee: u128::MAX, - send_tx_request_timeout: Duration::MAX, - }, - ) - .await - .unwrap(); - - let current_height = client._get_block_number().await.unwrap(); - - let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; - - panic!("{:?}", fees); - } + // #[tokio::test] + // async fn can_connect_to_eth_mainnet() { + // let current_height = client._get_block_number().await.unwrap(); + // + // let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; + // + // panic!("{:?}", fees); + // } } diff --git a/packages/services/Cargo.toml b/packages/services/Cargo.toml index d23a2313..194ecc6a 100644 --- a/packages/services/Cargo.toml +++ b/packages/services/Cargo.toml @@ -46,6 +46,7 @@ tai64 = { workspace = true } tokio = { workspace = true, features = ["macros"] } test-helpers = { workspace = true } rand = { workspace = true, features = ["small_rng", "std", "std_rng"] } +csv = "1.3" [features] test-helpers = ["dep:mockall", "dep:rand"] diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 50ae41c0..1894cae5 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -66,10 +66,11 @@ pub mod port { .tuple_windows() .all(|(l, r)| l.height + 1 == r.height); + let heights = fees.iter().map(|f| f.height).collect::>(); if !is_sequential { - return Err(InvalidSequence( - "blocks are not sequential by height".to_string(), - )); + return Err(InvalidSequence(format!( + "blocks are not sequential by height: {heights:?}" + ))); } Ok(Self { fees }) @@ -98,6 +99,7 @@ pub mod port { use super::{FeesProvider, SequentialBlockFees}; + #[derive(Debug, Clone)] pub struct TestFeesProvider { fees: BTreeMap, } @@ -150,6 +152,8 @@ pub mod port { pub mod service { + use std::ops::RangeInclusive; + use nonempty::NonEmpty; use super::{ @@ -170,14 +174,8 @@ pub mod port { // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees/save to db // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, last_n_blocks: u64) -> Fees { - let current_height = self.fees_provider.current_block_height().await; - - let starting_block = current_height.saturating_sub(last_n_blocks.saturating_sub(1)); - let fees = self - .fees_provider - .fees(starting_block..=current_height) - .await; + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { + let fees = self.fees_provider.fees(block_range).await; Self::mean(fees) } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 4395b606..5bdfd3e5 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,4 +1,4 @@ -mod fee_optimization; +pub mod fee_optimization; pub mod service { use std::{num::NonZeroUsize, time::Duration}; diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 06a9ee7c..3c691b5d 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -39,15 +39,17 @@ impl

SendOrWaitDecider

{ impl SendOrWaitDecider

{ // TODO: segfault validate blob number - pub async fn should_send_blob_tx(&self, num_blobs: u32) -> bool { + pub async fn should_send_blob_tx(&self, num_blobs: u32, at_block_height: u64) -> bool { let short_term_sma = self .fee_analytics - .calculate_sma(self.config.short_term_sma_num_blocks) + .calculate_sma( + at_block_height - self.config.short_term_sma_num_blocks..=at_block_height, + ) .await; let long_term_sma = self .fee_analytics - .calculate_sma(self.config.long_term_sma_num_blocks) + .calculate_sma(at_block_height - self.config.long_term_sma_num_blocks..=at_block_height) .await; let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); @@ -66,13 +68,13 @@ impl SendOrWaitDecider

{ // TODO: segfault proper type conversions, probably max(,,) - min(,,) <= delta let treshold = (long_term_tx_price as f64 * percentage) as u128; - eprintln!( - "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", - short_term_tx_price, - long_term_tx_price, - treshold, - short_term_tx_price.saturating_sub(treshold) - ); + // eprintln!( + // "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", + // short_term_tx_price, + // long_term_tx_price, + // treshold, + // short_term_tx_price.saturating_sub(treshold) + // ); short_term_tx_price < treshold } @@ -370,11 +372,14 @@ mod tests { ) { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = TestFeesProvider::new(fees); + let current_block_height = fees_provider.current_block_height().await; let analytics_service = FeeAnalytics::new(fees_provider); let sut = SendOrWaitDecider::new(analytics_service, config); - let should_send = sut.should_send_blob_tx(num_blobs).await; + let should_send = sut + .should_send_blob_tx(num_blobs, current_block_height) + .await; assert_eq!( should_send, expected_decision, diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 8f6c9d48..342994fd 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -1,4 +1,17 @@ -use services::fee_analytics::port::{l1::testing, service::FeeAnalytics}; +use std::{collections::HashMap, path::PathBuf}; + +use eth::make_pub_eth_client; +use services::{ + fee_analytics::{ + self, + port::{ + l1::testing::{self, TestFeesProvider}, + service::FeeAnalytics, + BlockFees, Fees, + }, + }, + state_committer::fee_optimization::SendOrWaitDecider, +}; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { @@ -8,7 +21,7 @@ async fn calculates_sma_correctly_for_last_1_block() { let last_n_blocks = 1; // when - let sma = fee_analytics.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(4..=4).await; // then assert_eq!(sma.base_fee_per_gas, 5); @@ -24,7 +37,7 @@ async fn calculates_sma_correctly_for_last_5_blocks() { let last_n_blocks = 5; // when - let sma = fee_analytics.calculate_sma(last_n_blocks).await; + let sma = fee_analytics.calculate_sma(0..=4).await; // then let mean = (5 + 4 + 3 + 2 + 1) / 5; @@ -32,3 +45,95 @@ async fn calculates_sma_correctly_for_last_5_blocks() { assert_eq!(sma.reward, mean); assert_eq!(sma.base_fee_per_blob_gas, mean); } + +fn calculate_tx_fee(fees: &Fees) -> u128 { + 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 +} + +fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + let mut csv_writer = + csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)).unwrap(); + csv_writer + .write_record(["height", "tx_fee"].iter()) + .unwrap(); + for (height, fee) in tx_fees { + csv_writer + .write_record([height.to_string(), fee.to_string()]) + .unwrap(); + } + csv_writer.flush().unwrap(); +} + +#[tokio::test] +async fn something() { + let client = make_pub_eth_client().await; + use services::fee_analytics::port::l1::FeesProvider; + + let current_block_height = 21402042; + let starting_block_height = current_block_height - 24 * 3600 / 12; + let data = client + .fees(starting_block_height..=current_block_height) + .await + .into_iter() + .collect::>(); + + let fee_lookup = data + .iter() + .map(|b| (b.height, b.fees)) + .collect::>(); + + let short_sma = 25u64; + let long_sma = 900; + + let current_tx_fees = data + .iter() + .map(|b| (b.height, calculate_tx_fee(&b.fees))) + .collect::>(); + + save_tx_fees(¤t_tx_fees, "current_fees.csv"); + + let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + let fee_analytics = FeeAnalytics::new(local_client.clone()); + + let mut short_sma_tx_fees = vec![]; + for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - short_sma..=height) + .await; + + let tx_fee = calculate_tx_fee(&fees); + + short_sma_tx_fees.push((height, tx_fee)); + } + save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + + let decider = SendOrWaitDecider::new( + FeeAnalytics::new(local_client.clone()), + services::state_committer::fee_optimization::Config { + sma_activation_fee_treshold: 1000, + short_term_sma_num_blocks: short_sma, + long_term_sma_num_blocks: long_sma, + comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) + }, + ); + + let mut decisions = vec![]; + let mut long_sma_tx_fees = vec![]; + + for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - long_sma..=height) + .await; + let tx_fee = calculate_tx_fee(&fees); + long_sma_tx_fees.push((height, tx_fee)); + + if decider.should_send_blob_tx(6, height).await { + let current_fees = fee_lookup.get(&height).unwrap(); + let current_tx_fee = calculate_tx_fee(current_fees); + decisions.push((height, current_tx_fee)); + } + } + + save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + save_tx_fees(&decisions, "decisions.csv"); +} From c732cd7f3d7abfe1cc203e56ea848d896c2eeb30 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 13:37:21 +0100 Subject: [PATCH 14/47] adding in the fee acceptance scaling for being late with posting --- .../src/state_committer/fee_optimization.rs | 156 +++++++++++------- packages/services/tests/fee_analytics.rs | 24 ++- 2 files changed, 118 insertions(+), 62 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 3c691b5d..899fb8b8 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,26 +1,27 @@ +use std::cmp::min; + use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; -// TODO: segfault validate percentages #[derive(Debug, Clone, Copy)] -pub enum ComparisonStrategy { - /// Short-term fee must be at most (1 - percentage) of the long-term fee. - /// For example, if percentage = 0.1 (10%), then: - /// short_term ≤ long_term * 0.9. - StrictlyLessOrEqualByPercent(f64), - - /// Short-term fee may be more expensive, but not by more than the given percentage. - /// For example, if percentage = 0.1 (10%), then: - /// short_term ≤ long_term * 1.1. - /// Short-term can be cheaper by any amount. - WithinVicinityOfPriceByPercent(f64), +pub struct SmaBlockNumPeriods { + pub short: u64, + pub long: u64, +} + +// TODO: segfault validate start discount is less than end premium and both are positive +#[derive(Debug, Clone, Copy)] +pub struct Feethresholds { + // TODO: segfault validate not 0 + pub max_l2_blocks_behind: u64, + pub start_discount_percentage: f64, + pub end_premium_percentage: f64, + pub always_acceptable_fee: u128, } #[derive(Debug, Clone, Copy)] pub struct Config { - pub sma_activation_fee_treshold: u128, - pub short_term_sma_num_blocks: u64, - pub long_term_sma_num_blocks: u64, - pub comparison_strategy: ComparisonStrategy, + pub sma_periods: SmaBlockNumPeriods, + pub fee_thresholds: Feethresholds, } pub struct SendOrWaitDecider

{ @@ -37,46 +38,75 @@ impl

SendOrWaitDecider

{ } } +#[derive(Debug, Clone, Copy)] +pub struct Context { + pub num_l2_blocks_behind: u64, + pub at_l1_height: u64, +} + impl SendOrWaitDecider

{ // TODO: segfault validate blob number - pub async fn should_send_blob_tx(&self, num_blobs: u32, at_block_height: u64) -> bool { + pub async fn should_send_blob_tx(&self, num_blobs: u32, context: Context) -> bool { + let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; + let short_term_sma = self .fee_analytics - .calculate_sma( - at_block_height - self.config.short_term_sma_num_blocks..=at_block_height, - ) + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await; let long_term_sma = self .fee_analytics - .calculate_sma(at_block_height - self.config.long_term_sma_num_blocks..=at_block_height) + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await; - let short_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + + let fee_always_acceptable = + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; - if short_term_tx_price < self.config.sma_activation_fee_treshold { + // TODO: segfault test this + let too_far_behind = + context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind; + + if fee_always_acceptable || too_far_behind { return true; } - let long_term_tx_price = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let max_upper_tx_fee = self.calculate_max_upper_fee(long_term_tx_fee, context); + + short_term_tx_fee < max_upper_tx_fee + } + + // TODO: segfault test this + fn calculate_max_upper_fee(&self, fee: u128, context: Context) -> u128 { + // Define percentages in Parts Per Million (PPM) for precision + // 1 PPM = 0.0001% + const PPM: u128 = 1_000_000; + let start_discount_ppm = + (self.config.fee_thresholds.start_discount_percentage * PPM as f64) as u128; + let end_premium_ppm = + (self.config.fee_thresholds.end_premium_percentage * PPM as f64) as u128; + + let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind as u128; - let percentage = match self.config.comparison_strategy { - ComparisonStrategy::StrictlyLessOrEqualByPercent(p) => 1.0 - p, - ComparisonStrategy::WithinVicinityOfPriceByPercent(p) => 1.0 + p, - }; + let blocks_behind = context.num_l2_blocks_behind; - // TODO: segfault proper type conversions, probably max(,,) - min(,,) <= delta - let treshold = (long_term_tx_price as f64 * percentage) as u128; + // TODO: segfault rename possibly + let ratio_ppm = (blocks_behind as u128 * PPM) / max_blocks_behind; - // eprintln!( - // "Short-term: {}, Long-term: {}, Allowed max: {}, diff: {}", - // short_term_tx_price, - // long_term_tx_price, - // treshold, - // short_term_tx_price.saturating_sub(treshold) - // ); + let initial_multiplier = PPM.saturating_sub(start_discount_ppm); - short_term_tx_price < treshold + let effect_of_being_late = (start_discount_ppm + end_premium_ppm) + .saturating_mul(ratio_ppm) + .saturating_div(PPM); + + let multiplier_ppm = initial_multiplier.saturating_add(effect_of_being_late); + + // TODO: segfault, for now just in case, but this should never happen + let multiplier_ppm = min(PPM + end_premium_ppm, multiplier_ppm); + + fee.saturating_mul(multiplier_ppm).saturating_div(PPM) } // TODO: Segfault maybe dont leak so much eth abstractions @@ -101,9 +131,9 @@ mod tests { fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, - (config.long_term_sma_num_blocks - config.short_term_sma_num_blocks) as usize, + (config.sma_periods.long - config.sma_periods.short) as usize, ); - let newer_fees = std::iter::repeat_n(new_fees, config.short_term_sma_num_blocks as usize); + let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); older_fees .chain(newer_fees) @@ -117,7 +147,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -130,7 +160,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -143,7 +173,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -156,7 +186,7 @@ mod tests { Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -169,7 +199,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -182,7 +212,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -195,7 +225,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -208,7 +238,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -221,7 +251,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -234,7 +264,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -247,7 +277,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -260,7 +290,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -273,7 +303,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -286,7 +316,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) @@ -300,7 +330,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.1) @@ -313,7 +343,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, // Strictly less or equal by 0% means must be cheaper or equal, which it's not @@ -328,7 +358,7 @@ mod tests { 6, Config { // Below threshold means we send anyway - sma_activation_fee_treshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, + sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) @@ -341,7 +371,7 @@ mod tests { Fees { base_fee_per_gas: 3200, reward: 5000, base_fee_per_blob_gas: 3200 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) @@ -354,7 +384,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_treshold: 0, + sma_activation_fee_threshold: 0, short_term_sma_num_blocks: 2, long_term_sma_num_blocks: 6, comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.0) @@ -378,7 +408,13 @@ mod tests { let sut = SendOrWaitDecider::new(analytics_service, config); let should_send = sut - .should_send_blob_tx(num_blobs, current_block_height) + .should_send_blob_tx( + num_blobs, + Context { + at_l1_height: current_block_height, + num_l2_blocks_behind: 0, + }, + ) .await; assert_eq!( diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 342994fd..d9cac8fc 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -10,7 +10,7 @@ use services::{ BlockFees, Fees, }, }, - state_committer::fee_optimization::SendOrWaitDecider, + state_committer::fee_optimization::{Context, SendOrWaitDecider}, }; #[tokio::test] @@ -83,6 +83,7 @@ async fn something() { .collect::>(); let short_sma = 25u64; + let middle_sma = 300; let long_sma = 900; let current_tx_fees = data @@ -107,6 +108,16 @@ async fn something() { } save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + let mut middle_sma_tx_fees = vec![]; + for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - middle_sma..=height) + .await; + let tx_fee = calculate_tx_fee(&fees); + middle_sma_tx_fees.push((height, tx_fee)); + } + save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); + let decider = SendOrWaitDecider::new( FeeAnalytics::new(local_client.clone()), services::state_committer::fee_optimization::Config { @@ -127,7 +138,16 @@ async fn something() { let tx_fee = calculate_tx_fee(&fees); long_sma_tx_fees.push((height, tx_fee)); - if decider.should_send_blob_tx(6, height).await { + if decider + .should_send_blob_tx( + 6, + Context { + at_l1_height: height, + num_l2_blocks_behind: 0, + }, + ) + .await + { let current_fees = fee_lookup.get(&height).unwrap(); let current_tx_fee = calculate_tx_fee(current_fees); decisions.push((height, current_tx_fee)); From 3a2506479c62ef7c12fc757537fede10ba6cffb4 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 14:34:11 +0100 Subject: [PATCH 15/47] checked tests validity, tweaked values to better represent the test scenario --- .../src/state_committer/fee_optimization.rs | 330 +++++++++++------- packages/services/tests/fee_analytics.rs | 186 +++++----- 2 files changed, 305 insertions(+), 211 deletions(-) diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 899fb8b8..54a91824 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -63,11 +63,14 @@ impl SendOrWaitDecider

{ let fee_always_acceptable = short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; + eprintln!("fee always acceptable: {}", fee_always_acceptable); // TODO: segfault test this let too_far_behind = context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind; + eprintln!("too far behind: {}", too_far_behind); + if fee_always_acceptable || too_far_behind { return true; } @@ -75,6 +78,32 @@ impl SendOrWaitDecider

{ let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); let max_upper_tx_fee = self.calculate_max_upper_fee(long_term_tx_fee, context); + let long_vs_max_delta_perc = + ((max_upper_tx_fee as f64 - long_term_tx_fee as f64) / long_term_tx_fee as f64 * 100.) + .abs(); + + let short_vs_max_delta_perc = ((max_upper_tx_fee as f64 - short_term_tx_fee as f64) + / short_term_tx_fee as f64 + * 100.) + .abs(); + + if long_term_tx_fee <= max_upper_tx_fee { + eprintln!("The max upper fee({max_upper_tx_fee}) is above the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + } else { + eprintln!("The max upper fee({max_upper_tx_fee}) is below the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + } + + if short_term_tx_fee <= max_upper_tx_fee { + eprintln!("The short term fee({short_term_tx_fee}) is below the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); + } else { + eprintln!("The short term fee({short_term_tx_fee}) is above the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); + } + + eprintln!( + "Short-term fee: {}, Long-term fee: {}, Max upper fee: {}", + short_term_tx_fee, long_term_tx_fee, max_upper_tx_fee + ); + short_term_tx_fee < max_upper_tx_fee } @@ -126,8 +155,8 @@ mod tests { use super::*; use crate::fee_analytics::port::{l1::testing::TestFeesProvider, Fees}; use test_case::test_case; + use tokio; - // Function to generate historical fees data fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, @@ -147,11 +176,15 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, }, + 0, // not behind at all true; "Should send because all short-term fees are lower than long-term" )] @@ -160,11 +193,15 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, }, + 0, false; "Should not send because all short-term fees are higher than long-term" )] @@ -173,24 +210,32 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + } }, + 0, true; - "Should send since we're below the activation fee threshold, even if all short-term fees are higher than long-term" + "Should send since short-term fee < always_acceptable_fee" )] #[test_case( Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because short-term base_fee_per_gas is lower" )] @@ -199,37 +244,49 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; "Should not send because short-term base_fee_per_gas is higher" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 900 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because short-term base_fee_per_blob_gas is lower" )] #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1100 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; "Should not send because short-term base_fee_per_blob_gas is higher" )] @@ -238,11 +295,15 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because short-term reward is lower" )] @@ -251,24 +312,33 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; "Should not send because short-term reward is higher" )] #[test_case( + // Multiple short-term fees are lower Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; "Should send because multiple short-term fees are lower" )] @@ -277,120 +347,144 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; - "Should not send because all fees are identical" + "Should not send because all fees are identical and no tolerance" )] #[test_case( + // Zero blobs scenario: blob fee differences don't matter Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; - "Zero blobs but short-term base_fee_per_gas and reward are lower" + "Zero blobs: short-term base_fee_per_gas and reward are lower, send" )] #[test_case( + // Zero blobs but short-term reward is higher Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, false; - "Zero blobs but short-term reward is higher" + "Zero blobs: short-term reward is higher, don't send" )] #[test_case( + // Zero blobs don't care about higher short-term base_fee_per_blob_gas Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) - }, - true; - "Zero blobs dont care about higher short-term base_fee_per_blob_gas" - )] - // New Tests (Introducing other Comparison Strategies) - #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - 6, - Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.1) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + } }, + 0, true; - "Should send (cheaper) with StrictlyLessOrEqualByPercent(10%)" + "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" )] + // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, + // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - // Strictly less or equal by 0% means must be cheaper or equal, which it's not - comparison_strategy: ComparisonStrategy::StrictlyLessOrEqualByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + }, }, + 0, false; - "Should not send (more expensive) with StrictlyLessOrEqualByPercent(0%)" + "Early: short-term expensive, not send" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, + // At max_l2_blocks_behind, send regardless + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, Config { - // Below threshold means we send anyway - sma_activation_fee_threshold: 21_000 * 5000 + 6 * 131_072 * 5000 + 5000 + 1, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + } }, + 100, true; - "Below activation threshold, send anyway for WithinVicinityOfPriceByPercent(10%)" + "Later: after max wait, send regardless" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 3200, reward: 5000, base_fee_per_blob_gas: 3200 }, - 6, + // Partway: at 80 blocks behind, tolerance might have increased enough to accept + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.1) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + }, }, + 65, true; - "Within vicinity of price by 10%: short_term slightly more expensive but allowed" + "Mid-wait: increased tolerance allows acceptance" )] #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, + // Short-term fee is huge, but always_acceptable_fee is large, so send immediately + Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, + Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, + 1, Config { - sma_activation_fee_threshold: 0, - short_term_sma_num_blocks: 2, - long_term_sma_num_blocks: 6, - comparison_strategy: ComparisonStrategy::WithinVicinityOfPriceByPercent(0.0) + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 26_000_000_000, + }, }, - false; - "Within vicinity with 0% means must not exceed long-term, not sending" + 0, + true; + "Always acceptable fee triggers immediate send" )] #[tokio::test] async fn parameterized_send_or_wait_tests( @@ -398,6 +492,7 @@ mod tests { new_fees: Fees, num_blobs: u32, config: Config, + num_l2_blocks_behind: u64, expected_decision: bool, ) { let fees = generate_fees(config, old_fees, new_fees); @@ -412,15 +507,14 @@ mod tests { num_blobs, Context { at_l1_height: current_block_height, - num_l2_blocks_behind: 0, + num_l2_blocks_behind, }, ) .await; assert_eq!( should_send, expected_decision, - "For num_blobs={num_blobs}, config={:?}: Expected decision: {}, got: {}", - config, expected_decision, should_send + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", ); } } diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index d9cac8fc..ffa5acc2 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -64,96 +64,96 @@ fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { csv_writer.flush().unwrap(); } -#[tokio::test] -async fn something() { - let client = make_pub_eth_client().await; - use services::fee_analytics::port::l1::FeesProvider; - - let current_block_height = 21402042; - let starting_block_height = current_block_height - 24 * 3600 / 12; - let data = client - .fees(starting_block_height..=current_block_height) - .await - .into_iter() - .collect::>(); - - let fee_lookup = data - .iter() - .map(|b| (b.height, b.fees)) - .collect::>(); - - let short_sma = 25u64; - let middle_sma = 300; - let long_sma = 900; - - let current_tx_fees = data - .iter() - .map(|b| (b.height, calculate_tx_fee(&b.fees))) - .collect::>(); - - save_tx_fees(¤t_tx_fees, "current_fees.csv"); - - let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - let fee_analytics = FeeAnalytics::new(local_client.clone()); - - let mut short_sma_tx_fees = vec![]; - for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - short_sma..=height) - .await; - - let tx_fee = calculate_tx_fee(&fees); - - short_sma_tx_fees.push((height, tx_fee)); - } - save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - - let mut middle_sma_tx_fees = vec![]; - for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - middle_sma..=height) - .await; - let tx_fee = calculate_tx_fee(&fees); - middle_sma_tx_fees.push((height, tx_fee)); - } - save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); - - let decider = SendOrWaitDecider::new( - FeeAnalytics::new(local_client.clone()), - services::state_committer::fee_optimization::Config { - sma_activation_fee_treshold: 1000, - short_term_sma_num_blocks: short_sma, - long_term_sma_num_blocks: long_sma, - comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) - }, - ); - - let mut decisions = vec![]; - let mut long_sma_tx_fees = vec![]; - - for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - long_sma..=height) - .await; - let tx_fee = calculate_tx_fee(&fees); - long_sma_tx_fees.push((height, tx_fee)); - - if decider - .should_send_blob_tx( - 6, - Context { - at_l1_height: height, - num_l2_blocks_behind: 0, - }, - ) - .await - { - let current_fees = fee_lookup.get(&height).unwrap(); - let current_tx_fee = calculate_tx_fee(current_fees); - decisions.push((height, current_tx_fee)); - } - } - - save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - save_tx_fees(&decisions, "decisions.csv"); -} +// #[tokio::test] +// async fn something() { +// let client = make_pub_eth_client().await; +// use services::fee_analytics::port::l1::FeesProvider; +// +// let current_block_height = 21402042; +// let starting_block_height = current_block_height - 24 * 3600 / 12; +// let data = client +// .fees(starting_block_height..=current_block_height) +// .await +// .into_iter() +// .collect::>(); +// +// let fee_lookup = data +// .iter() +// .map(|b| (b.height, b.fees)) +// .collect::>(); +// +// let short_sma = 25u64; +// let middle_sma = 300; +// let long_sma = 900; +// +// let current_tx_fees = data +// .iter() +// .map(|b| (b.height, calculate_tx_fee(&b.fees))) +// .collect::>(); +// +// save_tx_fees(¤t_tx_fees, "current_fees.csv"); +// +// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); +// let fee_analytics = FeeAnalytics::new(local_client.clone()); +// +// let mut short_sma_tx_fees = vec![]; +// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - short_sma..=height) +// .await; +// +// let tx_fee = calculate_tx_fee(&fees); +// +// short_sma_tx_fees.push((height, tx_fee)); +// } +// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); +// +// let mut middle_sma_tx_fees = vec![]; +// for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - middle_sma..=height) +// .await; +// let tx_fee = calculate_tx_fee(&fees); +// middle_sma_tx_fees.push((height, tx_fee)); +// } +// save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); +// +// let decider = SendOrWaitDecider::new( +// FeeAnalytics::new(local_client.clone()), +// services::state_committer::fee_optimization::Config { +// sma_activation_fee_treshold: 1000, +// short_term_sma_num_blocks: short_sma, +// long_term_sma_num_blocks: long_sma, +// comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) +// }, +// ); +// +// let mut decisions = vec![]; +// let mut long_sma_tx_fees = vec![]; +// +// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - long_sma..=height) +// .await; +// let tx_fee = calculate_tx_fee(&fees); +// long_sma_tx_fees.push((height, tx_fee)); +// +// if decider +// .should_send_blob_tx( +// 6, +// Context { +// at_l1_height: height, +// num_l2_blocks_behind: 0, +// }, +// ) +// .await +// { +// let current_fees = fee_lookup.get(&height).unwrap(); +// let current_tx_fee = calculate_tx_fee(current_fees); +// decisions.push((height, current_tx_fee)); +// } +// } +// +// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); +// save_tx_fees(&decisions, "decisions.csv"); +// } From c1d482496d5270b43e84d04647bcfcfdbce1c37e Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 14:51:52 +0100 Subject: [PATCH 16/47] analyzing algo config --- packages/services/tests/fee_analytics.rs | 183 +++++++++++------------ 1 file changed, 89 insertions(+), 94 deletions(-) diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index ffa5acc2..f8c00e0a 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -10,7 +10,7 @@ use services::{ BlockFees, Fees, }, }, - state_committer::fee_optimization::{Context, SendOrWaitDecider}, + state_committer::fee_optimization::{Context, Feethresholds, SendOrWaitDecider}, }; #[tokio::test] @@ -64,96 +64,91 @@ fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { csv_writer.flush().unwrap(); } -// #[tokio::test] -// async fn something() { -// let client = make_pub_eth_client().await; -// use services::fee_analytics::port::l1::FeesProvider; -// -// let current_block_height = 21402042; -// let starting_block_height = current_block_height - 24 * 3600 / 12; -// let data = client -// .fees(starting_block_height..=current_block_height) -// .await -// .into_iter() -// .collect::>(); -// -// let fee_lookup = data -// .iter() -// .map(|b| (b.height, b.fees)) -// .collect::>(); -// -// let short_sma = 25u64; -// let middle_sma = 300; -// let long_sma = 900; -// -// let current_tx_fees = data -// .iter() -// .map(|b| (b.height, calculate_tx_fee(&b.fees))) -// .collect::>(); -// -// save_tx_fees(¤t_tx_fees, "current_fees.csv"); -// -// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); -// let fee_analytics = FeeAnalytics::new(local_client.clone()); -// -// let mut short_sma_tx_fees = vec![]; -// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - short_sma..=height) -// .await; -// -// let tx_fee = calculate_tx_fee(&fees); -// -// short_sma_tx_fees.push((height, tx_fee)); -// } -// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); -// -// let mut middle_sma_tx_fees = vec![]; -// for height in (starting_block_height..=current_block_height).skip(middle_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - middle_sma..=height) -// .await; -// let tx_fee = calculate_tx_fee(&fees); -// middle_sma_tx_fees.push((height, tx_fee)); -// } -// save_tx_fees(&middle_sma_tx_fees, "middle_sma_fees.csv"); -// -// let decider = SendOrWaitDecider::new( -// FeeAnalytics::new(local_client.clone()), -// services::state_committer::fee_optimization::Config { -// sma_activation_fee_treshold: 1000, -// short_term_sma_num_blocks: short_sma, -// long_term_sma_num_blocks: long_sma, -// comparison_strategy: services::state_committer::fee_optimization::ComparisonStrategy::StrictlyLessOrEqualByPercent(0.01) -// }, -// ); -// -// let mut decisions = vec![]; -// let mut long_sma_tx_fees = vec![]; -// -// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - long_sma..=height) -// .await; -// let tx_fee = calculate_tx_fee(&fees); -// long_sma_tx_fees.push((height, tx_fee)); -// -// if decider -// .should_send_blob_tx( -// 6, -// Context { -// at_l1_height: height, -// num_l2_blocks_behind: 0, -// }, -// ) -// .await -// { -// let current_fees = fee_lookup.get(&height).unwrap(); -// let current_tx_fee = calculate_tx_fee(current_fees); -// decisions.push((height, current_tx_fee)); -// } -// } -// -// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); -// save_tx_fees(&decisions, "decisions.csv"); -// } +#[tokio::test] +async fn something() { + let client = make_pub_eth_client().await; + use services::fee_analytics::port::l1::FeesProvider; + + let current_block_height = 21408300; + let starting_block_height = current_block_height - 48 * 3600 / 12; + let data = client + .fees(starting_block_height..=current_block_height) + .await + .into_iter() + .collect::>(); + + let fee_lookup = data + .iter() + .map(|b| (b.height, b.fees)) + .collect::>(); + + let short_sma = 25u64; + let long_sma = 900; + + let current_tx_fees = data + .iter() + .map(|b| (b.height, calculate_tx_fee(&b.fees))) + .collect::>(); + + save_tx_fees(¤t_tx_fees, "current_fees.csv"); + + let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + let fee_analytics = FeeAnalytics::new(local_client.clone()); + + let mut short_sma_tx_fees = vec![]; + for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - short_sma..=height) + .await; + + let tx_fee = calculate_tx_fee(&fees); + + short_sma_tx_fees.push((height, tx_fee)); + } + save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + + let decider = SendOrWaitDecider::new( + FeeAnalytics::new(local_client.clone()), + services::state_committer::fee_optimization::Config { + sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + short: short_sma, + long: long_sma, + }, + fee_thresholds: Feethresholds { + max_l2_blocks_behind: 43200 * 3, + start_discount_percentage: 0.2, + end_premium_percentage: 0.2, + always_acceptable_fee: 1000000000000000u128, + }, + }, + ); + + let mut decisions = vec![]; + let mut long_sma_tx_fees = vec![]; + + for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + let fees = fee_analytics + .calculate_sma(height - long_sma..=height) + .await; + let tx_fee = calculate_tx_fee(&fees); + long_sma_tx_fees.push((height, tx_fee)); + + if decider + .should_send_blob_tx( + 6, + Context { + at_l1_height: height, + num_l2_blocks_behind: (height - starting_block_height) * 12, + }, + ) + .await + { + let current_fees = fee_lookup.get(&height).unwrap(); + let current_tx_fee = calculate_tx_fee(current_fees); + decisions.push((height, current_tx_fee)); + } + } + + save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + save_tx_fees(&decisions, "decisions.csv"); +} From 6d74ce3fe597ff968f71ad3ef40ca353926a811d Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Sun, 15 Dec 2024 19:23:16 +0100 Subject: [PATCH 17/47] committer compiles, unit tests fixed --- committer/src/main.rs | 4 + committer/src/setup.rs | 3 + packages/services/src/fee_analytics.rs | 89 ++++----- packages/services/src/state_committer.rs | 15 +- .../src/state_committer/fee_optimization.rs | 9 +- packages/services/tests/fee_analytics.rs | 178 +++++++++--------- packages/services/tests/state_committer.rs | 8 + packages/services/tests/state_listener.rs | 3 + packages/test-helpers/src/lib.rs | 4 + 9 files changed, 176 insertions(+), 137 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 0473d0c7..3e1714f4 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,6 +7,7 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; +use services::fee_analytics; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; @@ -72,12 +73,15 @@ async fn main() -> Result<()> { &metrics_registry, ); + let fee_analytics = fee_analytics::service::FeeAnalytics::new(ethereum_rpc.clone()); + let state_committer_handle = setup::state_committer( fuel_adapter.clone(), ethereum_rpc.clone(), storage.clone(), cancel_token.clone(), &config, + fee_analytics, ); let state_importer_handle = diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 3101e8fe..96cec1c4 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,6 +9,7 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, + fee_analytics::{self, service::FeeAnalytics}, state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, @@ -117,6 +118,7 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, + fee_analytics: FeeAnalytics, ) -> tokio::task::JoinHandle<()> { let state_committer = services::StateCommitter::new( l1, @@ -130,6 +132,7 @@ pub fn state_committer( tx_max_fee: config.app.tx_max_fee as u128, }, SystemClock, + fee_analytics, ); schedule_polling( diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 1894cae5..f1e88a8a 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -149,55 +149,55 @@ pub mod port { } } } +} - pub mod service { +pub mod service { - use std::ops::RangeInclusive; + use std::ops::RangeInclusive; - use nonempty::NonEmpty; + use nonempty::NonEmpty; - use super::{ - l1::{FeesProvider, SequentialBlockFees}, - BlockFees, Fees, - }; + use super::port::{ + l1::{FeesProvider, SequentialBlockFees}, + Fees, + }; - pub struct FeeAnalytics

{ - fees_provider: P, - } - impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } + pub struct FeeAnalytics

{ + fees_provider: P, + } + impl

FeeAnalytics

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } } + } - impl FeeAnalytics

{ - // TODO: segfault fail or signal if missing blocks/holes present - // TODO: segfault cache fees/save to db - // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { - let fees = self.fees_provider.fees(block_range).await; + impl FeeAnalytics

{ + // TODO: segfault fail or signal if missing blocks/holes present + // TODO: segfault cache fees/save to db + // TODO: segfault job to update fees in the background + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { + let fees = self.fees_provider.fees(block_range).await; - Self::mean(fees) - } + Self::mean(fees) + } - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, - }); - - // TODO: segfault should we round to nearest here? - Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), - } + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + + let total = fees + .into_iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), } } } @@ -205,6 +205,7 @@ pub mod port { #[cfg(test)] mod tests { + use itertools::Itertools; use port::{l1::SequentialBlockFees, BlockFees, Fees}; use super::*; @@ -294,7 +295,7 @@ mod tests { ); assert_eq!( result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height\")" + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" ); } @@ -328,8 +329,12 @@ mod tests { let iterated_fees: Vec = sequential_fees.into_iter().collect(); // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); assert_eq!( - iterated_fees, block_fees, + iterated_fees, expectation, "Expected iterator to yield the same block fees" ); } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 5bdfd3e5..ad86c1de 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -4,6 +4,7 @@ pub mod service { use std::{num::NonZeroUsize, time::Duration}; use crate::{ + fee_analytics::service::FeeAnalytics, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Result, Runner, }; @@ -35,16 +36,17 @@ pub mod service { } /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, + fee_analytics: FeeAnalytics, } - impl StateCommitter + impl StateCommitter where Clock: crate::state_committer::port::Clock, { @@ -55,6 +57,7 @@ pub mod service { storage: Db, config: Config, clock: Clock, + fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); Self { @@ -64,16 +67,18 @@ pub mod service { config, clock, startup_time, + fee_analytics, } } } - impl StateCommitter + impl StateCommitter where L1: crate::state_committer::port::l1::Api, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, + FeeProvider: crate::fee_analytics::port::l1::FeesProvider, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -234,12 +239,14 @@ pub mod service { } } - impl Runner for StateCommitter + impl Runner + for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, + FeeProvider: crate::fee_analytics::port::l1::FeesProvider + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 54a91824..19859ac4 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -1,6 +1,9 @@ use std::cmp::min; -use crate::fee_analytics::port::{l1::FeesProvider, service::FeeAnalytics, Fees}; +use crate::fee_analytics::{ + port::{l1::FeesProvider, Fees}, + service::FeeAnalytics, +}; #[derive(Debug, Clone, Copy)] pub struct SmaBlockNumPeriods { @@ -46,6 +49,8 @@ pub struct Context { impl SendOrWaitDecider

{ // TODO: segfault validate blob number + // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes + // (once that is implemented) pub async fn should_send_blob_tx(&self, num_blobs: u32, context: Context) -> bool { let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; @@ -479,7 +484,7 @@ mod tests { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, - always_acceptable_fee: 26_000_000_000, + always_acceptable_fee: 1_781_000_000_000 }, }, 0, diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index f8c00e0a..c8ddec6a 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -6,9 +6,9 @@ use services::{ self, port::{ l1::testing::{self, TestFeesProvider}, - service::FeeAnalytics, BlockFees, Fees, }, + service::FeeAnalytics, }, state_committer::fee_optimization::{Context, Feethresholds, SendOrWaitDecider}, }; @@ -64,91 +64,91 @@ fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { csv_writer.flush().unwrap(); } -#[tokio::test] -async fn something() { - let client = make_pub_eth_client().await; - use services::fee_analytics::port::l1::FeesProvider; - - let current_block_height = 21408300; - let starting_block_height = current_block_height - 48 * 3600 / 12; - let data = client - .fees(starting_block_height..=current_block_height) - .await - .into_iter() - .collect::>(); - - let fee_lookup = data - .iter() - .map(|b| (b.height, b.fees)) - .collect::>(); - - let short_sma = 25u64; - let long_sma = 900; - - let current_tx_fees = data - .iter() - .map(|b| (b.height, calculate_tx_fee(&b.fees))) - .collect::>(); - - save_tx_fees(¤t_tx_fees, "current_fees.csv"); - - let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - let fee_analytics = FeeAnalytics::new(local_client.clone()); - - let mut short_sma_tx_fees = vec![]; - for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - short_sma..=height) - .await; - - let tx_fee = calculate_tx_fee(&fees); - - short_sma_tx_fees.push((height, tx_fee)); - } - save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - - let decider = SendOrWaitDecider::new( - FeeAnalytics::new(local_client.clone()), - services::state_committer::fee_optimization::Config { - sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { - short: short_sma, - long: long_sma, - }, - fee_thresholds: Feethresholds { - max_l2_blocks_behind: 43200 * 3, - start_discount_percentage: 0.2, - end_premium_percentage: 0.2, - always_acceptable_fee: 1000000000000000u128, - }, - }, - ); - - let mut decisions = vec![]; - let mut long_sma_tx_fees = vec![]; - - for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - let fees = fee_analytics - .calculate_sma(height - long_sma..=height) - .await; - let tx_fee = calculate_tx_fee(&fees); - long_sma_tx_fees.push((height, tx_fee)); - - if decider - .should_send_blob_tx( - 6, - Context { - at_l1_height: height, - num_l2_blocks_behind: (height - starting_block_height) * 12, - }, - ) - .await - { - let current_fees = fee_lookup.get(&height).unwrap(); - let current_tx_fee = calculate_tx_fee(current_fees); - decisions.push((height, current_tx_fee)); - } - } - - save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - save_tx_fees(&decisions, "decisions.csv"); -} +// #[tokio::test] +// async fn something() { +// let client = make_pub_eth_client().await; +// use services::fee_analytics::port::l1::FeesProvider; +// +// let current_block_height = 21408300; +// let starting_block_height = current_block_height - 48 * 3600 / 12; +// let data = client +// .fees(starting_block_height..=current_block_height) +// .await +// .into_iter() +// .collect::>(); +// +// let fee_lookup = data +// .iter() +// .map(|b| (b.height, b.fees)) +// .collect::>(); +// +// let short_sma = 25u64; +// let long_sma = 900; +// +// let current_tx_fees = data +// .iter() +// .map(|b| (b.height, calculate_tx_fee(&b.fees))) +// .collect::>(); +// +// save_tx_fees(¤t_tx_fees, "current_fees.csv"); +// +// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); +// let fee_analytics = FeeAnalytics::new(local_client.clone()); +// +// let mut short_sma_tx_fees = vec![]; +// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - short_sma..=height) +// .await; +// +// let tx_fee = calculate_tx_fee(&fees); +// +// short_sma_tx_fees.push((height, tx_fee)); +// } +// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); +// +// let decider = SendOrWaitDecider::new( +// FeeAnalytics::new(local_client.clone()), +// services::state_committer::fee_optimization::Config { +// sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { +// short: short_sma, +// long: long_sma, +// }, +// fee_thresholds: Feethresholds { +// max_l2_blocks_behind: 43200 * 3, +// start_discount_percentage: 0.2, +// end_premium_percentage: 0.2, +// always_acceptable_fee: 1000000000000000u128, +// }, +// }, +// ); +// +// let mut decisions = vec![]; +// let mut long_sma_tx_fees = vec![]; +// +// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { +// let fees = fee_analytics +// .calculate_sma(height - long_sma..=height) +// .await; +// let tx_fee = calculate_tx_fee(&fees); +// long_sma_tx_fees.push((height, tx_fee)); +// +// if decider +// .should_send_blob_tx( +// 6, +// Context { +// at_l1_height: height, +// num_l2_blocks_behind: (height - starting_block_height) * 12, +// }, +// ) +// .await +// { +// let current_fees = fee_lookup.get(&height).unwrap(); +// let current_tx_fee = calculate_tx_fee(current_fees); +// decisions.push((height, current_tx_fee)); +// } +// } +// +// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); +// save_tx_fees(&decisions, "decisions.csv"); +// } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 831dbeb5..26acb9d1 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,4 +1,5 @@ use services::{ + fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -33,6 +34,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // when @@ -73,6 +75,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time beyond the timeout @@ -108,6 +111,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time less than the timeout @@ -150,6 +154,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // when @@ -193,6 +198,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time to exceed the timeout since last finalized fragment @@ -236,6 +242,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Advance time beyond the timeout from startup @@ -291,6 +298,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Submit the initial fragments diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index b60460cc..35f02e65 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,6 +3,7 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ + fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, @@ -447,6 +448,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Orig tx @@ -544,6 +546,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index f73fc475..52788924 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,6 +8,8 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; +use services::fee_analytics::port::l1::testing::TestFeesProvider; +use services::fee_analytics::service::FeeAnalytics; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, }; @@ -550,6 +552,7 @@ impl Setup { tx_max_fee: 1_000_000_000, }, self.test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ) .run() .await @@ -584,6 +587,7 @@ impl Setup { tx_max_fee: 1_000_000_000, }, self.test_clock.clone(), + FeeAnalytics::new(TestFeesProvider::new(vec![])), ); committer.run().await.unwrap(); From f7b44cfd8a18d51428dc37441fc76ce776293ccc Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 14:32:05 +0100 Subject: [PATCH 18/47] tracks fragment age, tests fixed --- ...e42fee96b0925eb2c30a0b98cf9f79c6ed76.json} | 10 +++- ...f99a2bfb27dd1e51d0f877f939e29b7f3a52.json} | 10 +++- committer/src/setup.rs | 2 +- packages/adapters/eth/src/lib.rs | 4 ++ .../adapters/storage/src/mappings/tables.rs | 13 ++++- packages/adapters/storage/src/postgres.rs | 16 +++--- packages/services/src/fee_analytics.rs | 31 +++++++++-- packages/services/src/state_committer.rs | 50 ++++++++++++++++-- .../src/state_committer/fee_optimization.rs | 44 ++++++++-------- packages/services/src/types/storage.rs | 1 + packages/services/tests/fee_analytics.rs | 8 +-- packages/services/tests/state_committer.rs | 52 ++++++++++++++----- packages/services/tests/state_listener.rs | 28 ++++++++-- packages/test-helpers/src/lib.rs | 38 +++++++++----- 14 files changed, 227 insertions(+), 80 deletions(-) rename .sqlx/{query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json => query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json} (58%) rename .sqlx/{query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json => query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json} (51%) diff --git a/.sqlx/query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json b/.sqlx/query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json similarity index 58% rename from .sqlx/query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json rename to .sqlx/query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json index 0b8b6451..86a298a0 100644 --- a/.sqlx/query-126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147.json +++ b/.sqlx/query-ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT f.*\n FROM l1_fragments f\n JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n WHERE t.hash = $1\n ", + "query": "\n SELECT\n f.*,\n b.start_height\n FROM l1_fragments f\n JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n JOIN bundles b ON b.id = f.bundle_id\n WHERE t.hash = $1\n ", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "bundle_id", "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "start_height", + "type_info": "Int8" } ], "parameters": { @@ -45,8 +50,9 @@ false, false, false, + false, false ] }, - "hash": "126284fed623566f0551d4e6a343ddbd8800dd6c27165f89fc72970fe8a89147" + "hash": "ddc1a18d0d257b9065830b46a10ce42fee96b0925eb2c30a0b98cf9f79c6ed76" } diff --git a/.sqlx/query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json b/.sqlx/query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json similarity index 51% rename from .sqlx/query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json rename to .sqlx/query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json index 2fd79840..9fe76b57 100644 --- a/.sqlx/query-11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40.json +++ b/.sqlx/query-ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n sub.id,\n sub.idx,\n sub.bundle_id,\n sub.data,\n sub.unused_bytes,\n sub.total_bytes\n FROM (\n SELECT DISTINCT ON (f.id)\n f.*,\n b.start_height\n FROM l1_fragments f\n JOIN bundles b ON b.id = f.bundle_id\n WHERE\n b.end_height >= $2\n AND NOT EXISTS (\n SELECT 1\n FROM l1_transaction_fragments tf\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n WHERE tf.fragment_id = f.id\n AND t.state <> $1\n )\n ORDER BY\n f.id,\n b.start_height ASC,\n f.idx ASC\n ) AS sub\n ORDER BY\n sub.start_height ASC,\n sub.idx ASC\n LIMIT $3;\n", + "query": "SELECT\n sub.id,\n sub.idx,\n sub.bundle_id,\n sub.data,\n sub.unused_bytes,\n sub.total_bytes,\n sub.start_height\n FROM (\n SELECT DISTINCT ON (f.id)\n f.*,\n b.start_height\n FROM l1_fragments f\n JOIN bundles b ON b.id = f.bundle_id\n WHERE\n b.end_height >= $2\n AND NOT EXISTS (\n SELECT 1\n FROM l1_transaction_fragments tf\n JOIN l1_blob_transaction t ON t.id = tf.transaction_id\n WHERE tf.fragment_id = f.id\n AND t.state <> $1\n )\n ORDER BY\n f.id,\n b.start_height ASC,\n f.idx ASC\n ) AS sub\n ORDER BY\n sub.start_height ASC,\n sub.idx ASC\n LIMIT $3;\n", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "total_bytes", "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "start_height", + "type_info": "Int8" } ], "parameters": { @@ -47,8 +52,9 @@ false, false, false, + false, false ] }, - "hash": "11c3dc9c06523c39e928bfc1c2947309b2f92155b5d2198e39b42f687cc58f40" + "hash": "ed56ffeb0264867943f7891de21ff99a2bfb27dd1e51d0f877f939e29b7f3a52" } diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 96cec1c4..209d049f 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -129,7 +129,7 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - tx_max_fee: config.app.tx_max_fee as u128, + price_algo: todo!(), }, SystemClock, fee_analytics, diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 17cdc6e4..f5113eec 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -206,6 +206,10 @@ impl services::block_committer::port::l1::Api for WebsocketClient { } impl services::state_committer::port::l1::Api for WebsocketClient { + async fn current_height(&self) -> Result { + self._get_block_number().await + } + delegate! { to (*self) { async fn submit_state_fragments( diff --git a/packages/adapters/storage/src/mappings/tables.rs b/packages/adapters/storage/src/mappings/tables.rs index 9729f382..566e68ae 100644 --- a/packages/adapters/storage/src/mappings/tables.rs +++ b/packages/adapters/storage/src/mappings/tables.rs @@ -1,4 +1,4 @@ -use std::num::NonZeroU32; +use std::{i64, num::NonZeroU32}; use num_bigint::BigInt; use services::types::{ @@ -193,6 +193,7 @@ pub struct BundleFragment { pub data: Vec, pub unused_bytes: i64, pub total_bytes: i64, + pub start_height: i64, } impl TryFrom for services::types::storage::BundleFragment { @@ -261,10 +262,18 @@ impl TryFrom for services::types::storage::BundleFragment { total_bytes, }; + let start_height = value.start_height.try_into().map_err(|e| { + crate::error::Error::Conversion(format!( + "Invalid db `start_height` ({}). Reason: {e}", + value.start_height + )) + })?; + Ok(Self { id, - idx, bundle_id, + idx, + oldest_block_in_bundle: start_height, fragment, }) } diff --git a/packages/adapters/storage/src/postgres.rs b/packages/adapters/storage/src/postgres.rs index 9d09a945..b836e4d9 100644 --- a/packages/adapters/storage/src/postgres.rs +++ b/packages/adapters/storage/src/postgres.rs @@ -277,7 +277,8 @@ impl Postgres { sub.bundle_id, sub.data, sub.unused_bytes, - sub.total_bytes + sub.total_bytes, + sub.start_height FROM ( SELECT DISTINCT ON (f.id) f.*, @@ -323,11 +324,14 @@ impl Postgres { let fragments = sqlx::query_as!( tables::BundleFragment, r#" - SELECT f.* - FROM l1_fragments f - JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id - JOIN l1_blob_transaction t ON t.id = tf.transaction_id - WHERE t.hash = $1 + SELECT + f.*, + b.start_height + FROM l1_fragments f + JOIN l1_transaction_fragments tf ON tf.fragment_id = f.id + JOIN l1_blob_transaction t ON t.id = tf.transaction_id + JOIN bundles b ON b.id = f.bundle_id + WHERE t.hash = $1 "#, tx_hash.as_slice() ) diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index f1e88a8a..79bef51f 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -99,12 +99,37 @@ pub mod port { use super::{FeesProvider, SequentialBlockFees}; + #[derive(Debug, Clone, Copy)] + pub struct ConstantFeesProvider { + fees: Fees, + } + + impl ConstantFeesProvider { + pub fn new(fees: Fees) -> Self { + Self { fees } + } + } + + impl FeesProvider for ConstantFeesProvider { + async fn fees(&self, _height_range: RangeInclusive) -> SequentialBlockFees { + let fees = BlockFees { + height: self.current_block_height().await, + fees: self.fees, + }; + + vec![fees].try_into().unwrap() + } + async fn current_block_height(&self) -> u64 { + 0 + } + } + #[derive(Debug, Clone)] - pub struct TestFeesProvider { + pub struct PreconfiguredFeesProvider { fees: BTreeMap, } - impl FeesProvider for TestFeesProvider { + impl FeesProvider for PreconfiguredFeesProvider { async fn current_block_height(&self) -> u64 { *self.fees.keys().last().unwrap() } @@ -125,7 +150,7 @@ pub mod port { } } - impl TestFeesProvider { + impl PreconfiguredFeesProvider { pub fn new(blocks: impl IntoIterator) -> Self { Self { fees: blocks.into_iter().collect(), diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index ad86c1de..569ceacc 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -5,12 +5,15 @@ pub mod service { use crate::{ fee_analytics::service::FeeAnalytics, + state_committer::fee_optimization::Context, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Result, Runner, + Error, Result, Runner, }; use itertools::Itertools; use tracing::info; + use super::fee_optimization::{FeeThresholds, SendOrWaitDecider, SmaBlockNumPeriods}; + // src/config.rs #[derive(Debug, Clone)] pub struct Config { @@ -19,7 +22,7 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, - pub tx_max_fee: u128, + pub price_algo: crate::state_committer::fee_optimization::Config, } #[cfg(feature = "test-helpers")] @@ -30,7 +33,15 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - tx_max_fee: 1_000_000_000, + price_algo: crate::state_committer::fee_optimization::Config { + sma_periods: SmaBlockNumPeriods { short: 1, long: 2 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100, + start_discount_percentage: 0., + end_premium_percentage: 0., + always_acceptable_fee: u128::MAX, + }, + }, } } } @@ -43,7 +54,7 @@ pub mod service { config: Config, clock: Clock, startup_time: DateTime, - fee_analytics: FeeAnalytics, + decider: SendOrWaitDecider, } impl StateCommitter @@ -60,6 +71,7 @@ pub mod service { fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); + let price_algo = config.price_algo; Self { l1_adapter, fuel_api, @@ -67,7 +79,7 @@ pub mod service { config, clock, startup_time, - fee_analytics, + decider: SendOrWaitDecider::new(fee_analytics, price_algo), } } } @@ -104,6 +116,33 @@ pub mod service { ) -> Result<()> { info!("about to send at most {} fragments", fragments.len()); + // TODO: segfault proper type conversion + let l1_height = self.l1_adapter.current_height().await?; + // TODO: segfault test this + let l2_height = self.fuel_api.latest_height().await?; + + let oldest_l2_block_in_fragments = fragments + .maximum_by_key(|b| b.oldest_block_in_bundle) + .oldest_block_in_bundle; + + let behind_on_l2 = l2_height.saturating_sub(oldest_l2_block_in_fragments); + + let should_send = self + .decider + .should_send_blob_tx( + fragments.len() as u32, + Context { + num_l2_blocks_behind: behind_on_l2 as u64, + at_l1_height: l1_height, + }, + ) + .await; + + if !should_send { + // TODO: segfault log here + return Ok(()); + } + let data = fragments.clone().map(|f| f.fragment); match self @@ -287,6 +326,7 @@ pub mod port { #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] pub trait Api { + async fn current_height(&self) -> Result; async fn submit_state_fragments( &self, fragments: NonEmpty, diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_optimization.rs index 19859ac4..cfb51eb7 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_optimization.rs @@ -13,7 +13,7 @@ pub struct SmaBlockNumPeriods { // TODO: segfault validate start discount is less than end premium and both are positive #[derive(Debug, Clone, Copy)] -pub struct Feethresholds { +pub struct FeeThresholds { // TODO: segfault validate not 0 pub max_l2_blocks_behind: u64, pub start_discount_percentage: f64, @@ -24,7 +24,7 @@ pub struct Feethresholds { #[derive(Debug, Clone, Copy)] pub struct Config { pub sma_periods: SmaBlockNumPeriods, - pub fee_thresholds: Feethresholds, + pub fee_thresholds: FeeThresholds, } pub struct SendOrWaitDecider

{ @@ -158,7 +158,7 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { use super::*; - use crate::fee_analytics::port::{l1::testing::TestFeesProvider, Fees}; + use crate::fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}; use test_case::test_case; use tokio; @@ -182,7 +182,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -199,7 +199,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -216,7 +216,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100, start_discount_percentage: 0.0, @@ -233,7 +233,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -250,7 +250,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -267,7 +267,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -284,7 +284,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -301,7 +301,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -318,7 +318,7 @@ mod tests { 5, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -336,7 +336,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -353,7 +353,7 @@ mod tests { 6, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -371,7 +371,7 @@ mod tests { 0, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -389,7 +389,7 @@ mod tests { 0, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -407,7 +407,7 @@ mod tests { 0, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.0, end_premium_percentage: 0.0, @@ -426,7 +426,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -444,7 +444,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -462,7 +462,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -480,7 +480,7 @@ mod tests { 1, Config { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, - fee_thresholds: Feethresholds { + fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100, start_discount_percentage: 0.20, end_premium_percentage: 0.20, @@ -501,7 +501,7 @@ mod tests { expected_decision: bool, ) { let fees = generate_fees(config, old_fees, new_fees); - let fees_provider = TestFeesProvider::new(fees); + let fees_provider = PreconfiguredFeesProvider::new(fees); let current_block_height = fees_provider.current_block_height().await; let analytics_service = FeeAnalytics::new(fees_provider); diff --git a/packages/services/src/types/storage.rs b/packages/services/src/types/storage.rs index 3f93e8ca..368c24f3 100644 --- a/packages/services/src/types/storage.rs +++ b/packages/services/src/types/storage.rs @@ -16,6 +16,7 @@ pub struct BundleFragment { pub id: NonNegative, pub idx: NonNegative, pub bundle_id: NonNegative, + pub oldest_block_in_bundle: u32, pub fragment: Fragment, } diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index c8ddec6a..7435f329 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -5,18 +5,18 @@ use services::{ fee_analytics::{ self, port::{ - l1::testing::{self, TestFeesProvider}, + l1::testing::{self, PreconfiguredFeesProvider}, BlockFees, Fees, }, service::FeeAnalytics, }, - state_committer::fee_optimization::{Context, Feethresholds, SendOrWaitDecider}, + state_committer::fee_optimization::{Context, SendOrWaitDecider}, }; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { // given - let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 1; @@ -32,7 +32,7 @@ async fn calculates_sma_correctly_for_last_1_block() { #[tokio::test] async fn calculates_sma_correctly_for_last_5_blocks() { // given - let fees_provider = testing::TestFeesProvider::new(testing::incrementing_fees(5)); + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); let last_n_blocks = 5; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 26acb9d1..8d17a148 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,5 +1,9 @@ use services::{ - fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, + fee_analytics::{ + port::{l1::testing::ConstantFeesProvider, Fees}, + service::FeeAnalytics, + }, + state_committer::port::l1::Api, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -13,7 +17,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { let fragments = setup.insert_fragments(0, 4).await; let tx_hash = [0; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { hash: tx_hash, @@ -21,6 +25,9 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( @@ -34,7 +41,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -54,7 +61,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { let fragments = setup.insert_fragments(0, 5).await; // Only 5 fragments, less than required let tx_hash = [1; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { hash: tx_hash, @@ -63,6 +70,9 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( l1_mock_submit, @@ -75,7 +85,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout @@ -111,7 +121,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time less than the timeout @@ -133,7 +143,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { let fragments = setup.insert_fragments(0, 5).await; let tx_hash = [3; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments).unwrap()), L1Tx { hash: tx_hash, @@ -141,6 +151,9 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( @@ -154,7 +167,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -177,7 +190,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { let fragments_to_submit = setup.insert_fragments(1, 2).await; let tx_hash = [4; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments_to_submit).unwrap()), L1Tx { hash: tx_hash, @@ -185,6 +198,9 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(1) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(1); let mut state_committer = StateCommitter::new( @@ -198,7 +214,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time to exceed the timeout since last finalized fragment @@ -221,7 +237,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> let fragments = setup.insert_fragments(0, 5).await; // Only 5 fragments, less than required let tx_hash = [5; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { hash: tx_hash, @@ -231,6 +247,9 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> )]); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(1) })); let mut state_committer = StateCommitter::new( l1_mock_submit, fuel_mock, @@ -242,7 +261,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout from startup @@ -266,7 +285,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { let tx_hash_1 = [6; 32]; let tx_hash_2 = [7; 32]; - let l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([ + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([ ( Some(NonEmpty::from_vec(fragments.clone()).unwrap()), L1Tx { @@ -285,6 +304,11 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ), ]); + l1_mock_submit.expect_current_height().returning(|| { + eprintln!("I was called"); + Box::pin(async { Ok(0) }) + }); + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( l1_mock_submit, @@ -298,7 +322,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Submit the initial fragments diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 35f02e65..9ddfe409 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,7 +3,13 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ - fee_analytics::{port::l1::testing::TestFeesProvider, service::FeeAnalytics}, + fee_analytics::{ + port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, + }, + service::FeeAnalytics, + }, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, @@ -439,8 +445,14 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { nonce, ..Default::default() }; + let mut l1_mock = + mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]); + l1_mock + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); + let mut committer = StateCommitter::new( - mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), StateCommitterConfig { @@ -448,7 +460,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx @@ -537,8 +549,14 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }; + let mut l1_mock = + mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]); + l1_mock + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); + let mut committer = StateCommitter::new( - mocks::l1::expects_state_submissions(vec![(None, orig_tx), (None, replacement_tx)]), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), crate::StateCommitterConfig { @@ -546,7 +564,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 52788924..34538031 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,8 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_analytics::port::l1::testing::TestFeesProvider; +use services::fee_analytics::port::l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}; +use services::fee_analytics::port::Fees; use services::fee_analytics::service::FeeAnalytics; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, @@ -533,15 +534,20 @@ impl Setup { } pub async fn send_fragments(&self, eth_tx: [u8; 32], eth_nonce: u32) { + let mut l1_mock = mocks::l1::expects_state_submissions(vec![( + None, + L1Tx { + hash: eth_tx, + nonce: eth_nonce, + ..Default::default() + }, + )]); + l1_mock + .expect_current_height() + .return_once(move || Box::pin(async { Ok(0) })); + StateCommitter::new( - mocks::l1::expects_state_submissions(vec![( - None, - L1Tx { - hash: eth_tx, - nonce: eth_nonce, - ..Default::default() - }, - )]), + l1_mock, mocks::fuel::latest_height_is(0), self.db(), services::StateCommitterConfig { @@ -549,10 +555,10 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - tx_max_fee: 1_000_000_000, + ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ) .run() .await @@ -566,7 +572,7 @@ impl Setup { pub async fn commit_block_bundle(&self, eth_tx: [u8; 32], eth_nonce: u32, height: u32) { self.insert_fragments(height, 6).await; - let l1_mock = mocks::l1::expects_state_submissions(vec![( + let mut l1_mock = mocks::l1::expects_state_submissions(vec![( None, L1Tx { hash: eth_tx, @@ -574,6 +580,10 @@ impl Setup { ..Default::default() }, )]); + l1_mock + .expect_current_height() + .return_once(move || Box::pin(async { Ok(0) })); + let fuel_mock = mocks::fuel::latest_height_is(height); let mut committer = StateCommitter::new( l1_mock, @@ -584,10 +594,10 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - tx_max_fee: 1_000_000_000, + ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(TestFeesProvider::new(vec![])), + FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); committer.run().await.unwrap(); From ac72a02237bfd2593513e8438924c9d2bc5e893a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 14:41:32 +0100 Subject: [PATCH 19/47] pull up the config to state committer level so it can be exported out without revealing the fee algo --- committer/src/setup.rs | 4 +- packages/adapters/eth/src/lib.rs | 12 +- packages/services/src/fee_analytics.rs | 11 +- packages/services/src/state_committer.rs | 42 +++++-- .../{fee_optimization.rs => fee_algo.rs} | 111 ++++++++---------- packages/services/tests/fee_analytics.rs | 14 +-- packages/services/tests/state_committer.rs | 1 - packages/services/tests/state_listener.rs | 2 +- packages/test-helpers/src/lib.rs | 2 +- 9 files changed, 96 insertions(+), 103 deletions(-) rename packages/services/src/state_committer/{fee_optimization.rs => fee_algo.rs} (89%) diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 209d049f..f493f227 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,7 +9,7 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_analytics::{self, service::FeeAnalytics}, + fee_analytics::service::FeeAnalytics, state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, @@ -129,7 +129,7 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - price_algo: todo!(), + fee_algo: todo!(), }, SystemClock, fee_analytics, diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index f5113eec..fa609e1e 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -9,10 +9,9 @@ use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, - providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE, }; use delegate::delegate; -use itertools::{izip, zip, Itertools}; +use itertools::{izip, Itertools}; use services::{ fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, types::{ @@ -311,16 +310,13 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { #[cfg(test)] mod test { - use std::time::Duration; + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; - use services::{ - block_bundler::port::l1::FragmentEncoder, block_committer::port::l1::Api, - fee_analytics::port::l1::FeesProvider, - }; + use services::block_bundler::port::l1::FragmentEncoder; - use crate::{BlobEncoder, Signer, Signers}; + use crate::BlobEncoder; #[test] fn gas_usage_correctly_calculated() { diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 79bef51f..206896f5 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -16,7 +16,7 @@ pub mod port { use std::ops::RangeInclusive; use itertools::Itertools; - use nonempty::NonEmpty; + use super::BlockFees; @@ -90,12 +90,9 @@ pub mod port { use std::{collections::BTreeMap, ops::RangeInclusive}; use itertools::Itertools; - use nonempty::NonEmpty; + - use crate::{ - fee_analytics::port::{BlockFees, Fees}, - types::CollectNonEmpty, - }; + use crate::fee_analytics::port::{BlockFees, Fees}; use super::{FeesProvider, SequentialBlockFees}; @@ -180,7 +177,7 @@ pub mod service { use std::ops::RangeInclusive; - use nonempty::NonEmpty; + use super::port::{ l1::{FeesProvider, SequentialBlockFees}, diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 569ceacc..48026c42 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,18 +1,42 @@ -pub mod fee_optimization; +mod fee_algo; pub mod service { - use std::{num::NonZeroUsize, time::Duration}; + use std::{ + num::{NonZeroU64, NonZeroUsize}, + time::Duration, + }; use crate::{ fee_analytics::service::FeeAnalytics, - state_committer::fee_optimization::Context, + state_committer::fee_algo::Context, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Error, Result, Runner, + Result, Runner, }; use itertools::Itertools; use tracing::info; - use super::fee_optimization::{FeeThresholds, SendOrWaitDecider, SmaBlockNumPeriods}; + use super::fee_algo::SendOrWaitDecider; + + #[derive(Debug, Clone, Copy)] + pub struct SmaBlockNumPeriods { + pub short: u64, + pub long: u64, + } + + // TODO: segfault validate start discount is less than end premium and both are positive + #[derive(Debug, Clone, Copy)] + pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU64, + pub start_discount_percentage: f64, + pub end_premium_percentage: f64, + pub always_acceptable_fee: u128, + } + + #[derive(Debug, Clone, Copy)] + pub struct FeeAlgoConfig { + pub sma_periods: SmaBlockNumPeriods, + pub fee_thresholds: FeeThresholds, + } // src/config.rs #[derive(Debug, Clone)] @@ -22,7 +46,7 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, - pub price_algo: crate::state_committer::fee_optimization::Config, + pub fee_algo: FeeAlgoConfig, } #[cfg(feature = "test-helpers")] @@ -33,10 +57,10 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - price_algo: crate::state_committer::fee_optimization::Config { + fee_algo: FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 1, long: 2 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0., end_premium_percentage: 0., always_acceptable_fee: u128::MAX, @@ -71,7 +95,7 @@ pub mod service { fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); - let price_algo = config.price_algo; + let price_algo = config.fee_algo; Self { l1_adapter, fuel_api, diff --git a/packages/services/src/state_committer/fee_optimization.rs b/packages/services/src/state_committer/fee_algo.rs similarity index 89% rename from packages/services/src/state_committer/fee_optimization.rs rename to packages/services/src/state_committer/fee_algo.rs index cfb51eb7..bb2a5b37 100644 --- a/packages/services/src/state_committer/fee_optimization.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -5,35 +5,15 @@ use crate::fee_analytics::{ service::FeeAnalytics, }; -#[derive(Debug, Clone, Copy)] -pub struct SmaBlockNumPeriods { - pub short: u64, - pub long: u64, -} - -// TODO: segfault validate start discount is less than end premium and both are positive -#[derive(Debug, Clone, Copy)] -pub struct FeeThresholds { - // TODO: segfault validate not 0 - pub max_l2_blocks_behind: u64, - pub start_discount_percentage: f64, - pub end_premium_percentage: f64, - pub always_acceptable_fee: u128, -} - -#[derive(Debug, Clone, Copy)] -pub struct Config { - pub sma_periods: SmaBlockNumPeriods, - pub fee_thresholds: FeeThresholds, -} +use super::service::FeeAlgoConfig; pub struct SendOrWaitDecider

{ fee_analytics: FeeAnalytics

, - config: Config, + config: FeeAlgoConfig, } impl

SendOrWaitDecider

{ - pub fn new(fee_analytics: FeeAnalytics

, config: Config) -> Self { + pub fn new(fee_analytics: FeeAnalytics

, config: FeeAlgoConfig) -> Self { Self { fee_analytics, config, @@ -72,7 +52,7 @@ impl SendOrWaitDecider

{ // TODO: segfault test this let too_far_behind = - context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind; + context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get(); eprintln!("too far behind: {}", too_far_behind); @@ -122,7 +102,7 @@ impl SendOrWaitDecider

{ let end_premium_ppm = (self.config.fee_thresholds.end_premium_percentage * PPM as f64) as u128; - let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind as u128; + let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind.get() as u128; let blocks_behind = context.num_l2_blocks_behind; @@ -158,11 +138,14 @@ impl SendOrWaitDecider

{ #[cfg(test)] mod tests { use super::*; - use crate::fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}; + use crate::{ + fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}, + state_committer::service::{FeeThresholds, SmaBlockNumPeriods}, + }; use test_case::test_case; use tokio; - fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + fn generate_fees(config: FeeAlgoConfig, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, (config.sma_periods.long - config.sma_periods.short) as usize, @@ -180,10 +163,10 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -197,10 +180,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -214,11 +197,11 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, } @@ -231,10 +214,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -248,10 +231,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -265,10 +248,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -282,10 +265,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -299,10 +282,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -316,10 +299,10 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -334,10 +317,10 @@ mod tests { Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -351,10 +334,10 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -369,10 +352,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -387,10 +370,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -405,10 +388,10 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -424,10 +407,10 @@ mod tests { Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, @@ -442,10 +425,10 @@ mod tests { Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, @@ -460,10 +443,10 @@ mod tests { Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, @@ -478,10 +461,10 @@ mod tests { Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, 1, - Config { + FeeAlgoConfig { sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100, + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 1_781_000_000_000 @@ -496,7 +479,7 @@ mod tests { old_fees: Fees, new_fees: Fees, num_blobs: u32, - config: Config, + config: FeeAlgoConfig, num_l2_blocks_behind: u64, expected_decision: bool, ) { diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 7435f329..39513e98 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -1,17 +1,11 @@ -use std::{collections::HashMap, path::PathBuf}; +use std::path::PathBuf; -use eth::make_pub_eth_client; -use services::{ - fee_analytics::{ - self, +use services::fee_analytics::{ port::{ - l1::testing::{self, PreconfiguredFeesProvider}, - BlockFees, Fees, + l1::testing::{self}, Fees, }, service::FeeAnalytics, - }, - state_committer::fee_optimization::{Context, SendOrWaitDecider}, -}; + }; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 8d17a148..5647c481 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -3,7 +3,6 @@ use services::{ port::{l1::testing::ConstantFeesProvider, Fees}, service::FeeAnalytics, }, - state_committer::port::l1::Api, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 9ddfe409..b62108bf 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -5,7 +5,7 @@ use mockall::predicate::eq; use services::{ fee_analytics::{ port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + l1::testing::ConstantFeesProvider, Fees, }, service::FeeAnalytics, diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 34538031..ac357e66 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,7 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_analytics::port::l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}; +use services::fee_analytics::port::l1::testing::ConstantFeesProvider; use services::fee_analytics::port::Fees; use services::fee_analytics::service::FeeAnalytics; use services::types::{ From b8d66319fc6d29205ea6d8288ed7cadbaaa81b72 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 14:56:27 +0100 Subject: [PATCH 20/47] added tests to state committer showing that the price algo is consulted --- packages/services/tests/state_committer.rs | 441 ++++++++++++++++++++- 1 file changed, 439 insertions(+), 2 deletions(-) diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 5647c481..d29649a1 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,12 +1,16 @@ use services::{ fee_analytics::{ - port::{l1::testing::ConstantFeesProvider, Fees}, + port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, + }, service::FeeAnalytics, }, + state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaBlockNumPeriods}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use std::time::Duration; +use std::{num::NonZeroU64, time::Duration}; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -338,3 +342,436 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { // Mocks validate that the fragments have been sent again Ok(()) } + +#[tokio::test] +async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { + // Given + let setup = test_helpers::Setup::init().await; + + let fee_sequence = vec![ + ( + 1, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + }; + + // Insert enough fragments to meet the accumulation threshold + let fragments = setup.insert_fragments(0, 6).await; + + // Expect a state submission + let tx_hash = [0; 32]; + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + Some(NonEmpty::from_vec(fragments.clone()).unwrap()), + L1Tx { + hash: tx_hash, + nonce: 0, + ..Default::default() + }, + )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(6) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); + let mut state_committer = StateCommitter::new( + l1_mock_submit, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // When + state_committer.run().await?; + + // Then + // Mocks validate that the fragments have been sent + Ok(()) +} + +#[tokio::test] +async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<()> { + // given + let setup = test_helpers::Setup::init().await; + + // Define fee sequence: last 2 blocks have higher fees than the long-term average + let fee_sequence = vec![ + ( + 1, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 3000, + reward: 3000, + base_fee_per_blob_gas: 3000, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + }; + + // Insert enough fragments to meet the accumulation threshold + let _fragments = setup.insert_fragments(0, 6).await; + + let mut l1_mock = test_helpers::mocks::l1::expects_state_submissions([]); + l1_mock + .expect_current_height() + .returning(|| Box::pin(async { Ok(6) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); + let mut state_committer = StateCommitter::new( + l1_mock, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // when + state_committer.run().await?; + + // then + // Mocks validate that no fragments have been sent + Ok(()) +} + +#[tokio::test] +async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { + // given + let setup = test_helpers::Setup::init().await; + + // Define fee sequence with high fees to ensure that without the behind condition, it wouldn't send + let fee_sequence = vec![ + ( + 1, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 2, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 3, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 4, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 5, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ( + 6, + Fees { + base_fee_per_gas: 7000, + reward: 7000, + base_fee_per_blob_gas: 7000, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(50).unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + }; + + // Insert enough fragments to meet the accumulation threshold + let fragments = setup.insert_fragments(0, 6).await; + + // Expect a state submission despite high fees because blocks behind exceed max + let tx_hash = [0; 32]; + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + Some(NonEmpty::from_vec(fragments.clone()).unwrap()), + L1Tx { + hash: tx_hash, + nonce: 0, + ..Default::default() + }, + )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(6) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 + let mut state_committer = StateCommitter::new( + l1_mock_submit, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // when + state_committer.run().await?; + + // then + // Mocks validate that the fragments have been sent despite high fees + Ok(()) +} + +#[tokio::test] +async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_tolerance() -> Result<()> { + // given + let setup = test_helpers::Setup::init().await; + + let fee_sequence = vec![ + ( + 95, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 96, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 97, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 98, + Fees { + base_fee_per_gas: 5000, + reward: 5000, + base_fee_per_blob_gas: 5000, + }, + ), + ( + 99, + Fees { + base_fee_per_gas: 5800, + reward: 5800, + base_fee_per_blob_gas: 5800, + }, + ), + ( + 100, + Fees { + base_fee_per_gas: 5800, + reward: 5800, + base_fee_per_blob_gas: 5800, + }, + ), + ]; + + let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); + let fee_analytics = FeeAnalytics::new(fees_provider); + + let fee_algo_config = FeeAlgoConfig { + sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + start_discount_percentage: 0.20, + end_premium_percentage: 0.20, + always_acceptable_fee: 0, + }, + }; + + let fragments = setup.insert_fragments(0, 6).await; + + // Expect a state submission due to nearing max blocks behind and increased tolerance + let tx_hash = [0; 32]; + let mut l1_mock_submit = test_helpers::mocks::l1::expects_state_submissions([( + Some(NonEmpty::from_vec(fragments.clone()).unwrap()), + L1Tx { + hash: tx_hash, + nonce: 0, + ..Default::default() + }, + )]); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(100) })); + + let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); + + let mut state_committer = StateCommitter::new( + l1_mock_submit, + fuel_mock, + setup.db(), + StateCommitterConfig { + lookback_window: 1000, + fragment_accumulation_timeout: Duration::from_secs(60), + fragments_to_accumulate: 6.try_into().unwrap(), + fee_algo: fee_algo_config, + ..Default::default() + }, + setup.test_clock(), + fee_analytics, + ); + + // when + state_committer.run().await?; + + // then + // Mocks validate that the fragments have been sent due to increased tolerance from nearing max blocks behind + Ok(()) +} From fb6c016bdca0956d8c01dc829491e5d9ba4bc2b7 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 18:45:58 +0100 Subject: [PATCH 21/47] all tests passing, including e2e --- committer/src/config.rs | 26 ++++++++++++- committer/src/setup.rs | 18 ++++++++- e2e/src/committer.rs | 10 +++++ e2e/src/whole_stack.rs | 2 +- .../adapters/eth/src/websocket/connection.rs | 4 +- packages/services/src/state_committer.rs | 6 +-- .../services/src/state_committer/fee_algo.rs | 38 +++++++++---------- packages/services/tests/state_committer.rs | 10 ++--- run_tests.sh | 2 +- 9 files changed, 81 insertions(+), 35 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index fbc6cf6b..734e92a6 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -1,6 +1,6 @@ use std::{ net::Ipv4Addr, - num::{NonZeroU32, NonZeroUsize}, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, str::FromStr, time::Duration, }; @@ -111,6 +111,30 @@ pub struct App { /// How often to run state pruner #[serde(deserialize_with = "human_readable_duration")] pub state_pruner_run_interval: Duration, + /// Configuration for the fee algorithm used by the StateCommitter + pub fee_algo: FeeAlgoConfig, +} + +/// Configuration for the fee algorithm used by the StateCommitter +#[derive(Debug, Clone, Deserialize)] +pub struct FeeAlgoConfig { + /// Short-term period for Simple Moving Average (SMA) in block numbers + pub short_sma_blocks: u64, + + /// Long-term period for Simple Moving Average (SMA) in block numbers + pub long_sma_blocks: u64, + + /// Maximum number of unposted L2 blocks before sending a transaction regardless of fees + pub max_l2_blocks_behind: NonZeroU64, + + /// Starting discount percentage applied we try to achieve if we're 0 l2 blocks behind + pub start_discount_percentage: f64, + + /// Premium percentage we're willing to pay if we're max_l2_blocks_behind - 1 blocks behind + pub end_premium_percentage: f64, + + /// A fee that is always acceptable regardless of other conditions + pub always_acceptable_fee: u64, } /// Configuration settings for managing fuel block bundling and fragment submission operations. diff --git a/committer/src/setup.rs b/committer/src/setup.rs index f493f227..b320ee40 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -10,7 +10,10 @@ use metrics::{ use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, fee_analytics::service::FeeAnalytics, - state_committer::port::Storage, + state_committer::{ + port::Storage, + service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, + }, state_listener::service::StateListener, state_pruner::service::StatePruner, wallet_balance_tracker::service::WalletBalanceTracker, @@ -129,7 +132,18 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - fee_algo: todo!(), + fee_algo: FeeAlgoConfig { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config.app.fee_algo.start_discount_percentage, + end_premium_percentage: config.app.fee_algo.end_premium_percentage, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }, }, SystemClock, fee_analytics, diff --git a/e2e/src/committer.rs b/e2e/src/committer.rs index ada77606..4cdc21e2 100644 --- a/e2e/src/committer.rs +++ b/e2e/src/committer.rs @@ -121,6 +121,16 @@ impl Committer { "COMMITTER__APP__STATE_PRUNER_RUN_INTERVAL", get_field!(state_pruner_run_interval), ) + .env("COMMITTER__APP__FEE_ALGO__SHORT_SMA_BLOCKS", "1") + .env("COMMITTER__APP__FEE_ALGO__LONG_SMA_BLOCKS", "1") + .env("COMMITTER__APP__FEE_ALGO__MAX_L2_BLOCKS_BEHIND", "1") + .env("COMMITTER__APP__FEE_ALGO__START_DISCOUNT_PERCENTAGE", "0") + .env("COMMITTER__APP__FEE_ALGO__END_PREMIUM_PERCENTAGE", "0") + // we're basically disabling the fee algo here + .env( + "COMMITTER__APP__FEE_ALGO__ALWAYS_ACCEPTABLE_FEE", + u64::MAX.to_string(), + ) .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap()) .kill_on_drop(true); diff --git a/e2e/src/whole_stack.rs b/e2e/src/whole_stack.rs index 59e8e779..1a3e257d 100644 --- a/e2e/src/whole_stack.rs +++ b/e2e/src/whole_stack.rs @@ -60,7 +60,7 @@ impl WholeStack { let db = start_db().await?; let committer = start_committer( - logs, + true, blob_support, db.clone(), ð_node, diff --git a/packages/adapters/eth/src/websocket/connection.rs b/packages/adapters/eth/src/websocket/connection.rs index 65318b49..f6729d21 100644 --- a/packages/adapters/eth/src/websocket/connection.rs +++ b/packages/adapters/eth/src/websocket/connection.rs @@ -399,9 +399,7 @@ impl WsConnection { let contract_address = Address::from_slice(contract_address.as_ref()); let contract = FuelStateContract::new(contract_address, provider.clone()); - // TODO: segfault revert this - // let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; - let interval_u256 = 1u32; + let interval_u256 = contract.BLOCKS_PER_COMMIT_INTERVAL().call().await?._0; let commit_interval = u32::try_from(interval_u256) .map_err(|e| Error::Other(e.to_string())) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 48026c42..38387532 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -18,7 +18,7 @@ pub mod service { use super::fee_algo::SendOrWaitDecider; #[derive(Debug, Clone, Copy)] - pub struct SmaBlockNumPeriods { + pub struct SmaPeriods { pub short: u64, pub long: u64, } @@ -34,7 +34,7 @@ pub mod service { #[derive(Debug, Clone, Copy)] pub struct FeeAlgoConfig { - pub sma_periods: SmaBlockNumPeriods, + pub sma_periods: SmaPeriods, pub fee_thresholds: FeeThresholds, } @@ -58,7 +58,7 @@ pub mod service { fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), fee_algo: FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 1, long: 2 }, + sma_periods: SmaPeriods { short: 1, long: 2 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0., diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index bb2a5b37..edc3186f 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -140,7 +140,7 @@ mod tests { use super::*; use crate::{ fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}, - state_committer::service::{FeeThresholds, SmaBlockNumPeriods}, + state_committer::service::{FeeThresholds, SmaPeriods}, }; use test_case::test_case; use tokio; @@ -164,7 +164,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -181,7 +181,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -198,7 +198,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100.try_into().unwrap(), @@ -215,7 +215,7 @@ mod tests { Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -232,7 +232,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -249,7 +249,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -266,7 +266,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -283,7 +283,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -300,7 +300,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -318,7 +318,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -335,7 +335,7 @@ mod tests { Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -353,7 +353,7 @@ mod tests { Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -371,7 +371,7 @@ mod tests { Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -389,7 +389,7 @@ mod tests { Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, @@ -408,7 +408,7 @@ mod tests { Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -426,7 +426,7 @@ mod tests { Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -444,7 +444,7 @@ mod tests { Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -462,7 +462,7 @@ mod tests { Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, 1, FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index d29649a1..fcd50a98 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -6,7 +6,7 @@ use services::{ }, service::FeeAnalytics, }, - state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaBlockNumPeriods}, + state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -403,7 +403,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), start_discount_percentage: 0.0, @@ -514,7 +514,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), start_discount_percentage: 0.0, @@ -616,7 +616,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(50).unwrap(), start_discount_percentage: 0.0, @@ -726,7 +726,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fee_analytics = FeeAnalytics::new(fees_provider); let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaBlockNumPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), start_discount_percentage: 0.20, diff --git a/run_tests.sh b/run_tests.sh index 3280743e..9c074973 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,4 +9,4 @@ cargo test --manifest-path "$workspace_cargo_manifest" --workspace --exclude e2e # So that we may have a binary in `target/release` cargo build --release --manifest-path "$workspace_cargo_manifest" --bin fuel-block-committer -PATH="$script_location/target/release:$PATH" cargo test --manifest-path "$workspace_cargo_manifest" --package e2e -- --test-threads=1 +PATH="$script_location/target/release:$PATH" cargo test --manifest-path "$workspace_cargo_manifest" --package e2e -- --test-threads=1 --nocapture From c7aa8fa52cadcd1615241cb6e7e0b3cd7af22897 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 19:58:31 +0100 Subject: [PATCH 22/47] cleanup --- Cargo.lock | 1 + packages/services/Cargo.toml | 1 + .../services/src/state_committer/fee_algo.rs | 79 ++++++++++++++----- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 582429a1..84196586 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5796,6 +5796,7 @@ dependencies = [ "mockall", "nonempty", "pretty_assertions", + "proptest", "rand", "rayon", "serde", diff --git a/packages/services/Cargo.toml b/packages/services/Cargo.toml index 194ecc6a..644c6b78 100644 --- a/packages/services/Cargo.toml +++ b/packages/services/Cargo.toml @@ -47,6 +47,7 @@ tokio = { workspace = true, features = ["macros"] } test-helpers = { workspace = true } rand = { workspace = true, features = ["small_rng", "std", "std_rng"] } csv = "1.3" +proptest = { workspace = true, features = ["default"] } [features] test-helpers = ["dep:mockall", "dep:rand"] diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index edc3186f..9a3c1d40 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -94,33 +94,63 @@ impl SendOrWaitDecider

{ // TODO: segfault test this fn calculate_max_upper_fee(&self, fee: u128, context: Context) -> u128 { - // Define percentages in Parts Per Million (PPM) for precision - // 1 PPM = 0.0001% - const PPM: u128 = 1_000_000; - let start_discount_ppm = - (self.config.fee_thresholds.start_discount_percentage * PPM as f64) as u128; - let end_premium_ppm = - (self.config.fee_thresholds.end_premium_percentage * PPM as f64) as u128; + const PPM: u128 = 1_000_000; // 100% in PPM - let max_blocks_behind = self.config.fee_thresholds.max_l2_blocks_behind.get() as u128; + let max_blocks_behind = u128::from(self.config.fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(context.num_l2_blocks_behind); - let blocks_behind = context.num_l2_blocks_behind; + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({})", + blocks_behind, + max_blocks_behind + ); - // TODO: segfault rename possibly - let ratio_ppm = (blocks_behind as u128 * PPM) / max_blocks_behind; + let start_discount_ppm = + percentage_to_ppm(self.config.fee_thresholds.start_discount_percentage); + let end_premium_ppm = percentage_to_ppm(self.config.fee_thresholds.end_premium_percentage); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = self.calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); - let initial_multiplier = PPM.saturating_sub(start_discount_ppm); + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + PPM + end_premium_ppm, + ); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm).saturating_div(PPM) + } - let effect_of_being_late = (start_discount_ppm + end_premium_ppm) - .saturating_mul(ratio_ppm) - .saturating_div(PPM); + fn calculate_premium_increment( + &self, + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, + ) -> u128 { + const PPM: u128 = 1_000_000; // 100% in PPM - let multiplier_ppm = initial_multiplier.saturating_add(effect_of_being_late); + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - // TODO: segfault, for now just in case, but this should never happen - let multiplier_ppm = min(PPM + end_premium_ppm, multiplier_ppm); + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(PPM) + .saturating_div(max_blocks_behind) + }; - fee.saturating_mul(multiplier_ppm).saturating_div(PPM) + total_ppm.saturating_mul(proportion).saturating_div(PPM) } // TODO: Segfault maybe dont leak so much eth abstractions @@ -135,11 +165,20 @@ impl SendOrWaitDecider

{ } } +fn percentage_to_ppm(percentage: f64) -> u128 { + (percentage * 1_000_000.0) as u128 +} + #[cfg(test)] mod tests { + use std::num::NonZeroU64; + use super::*; use crate::{ - fee_analytics::port::{l1::testing::PreconfiguredFeesProvider, Fees}, + fee_analytics::port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, + }, state_committer::service::{FeeThresholds, SmaPeriods}, }; use test_case::test_case; From 79ee93629b9c78353dc67e23c6e443ee094a343d Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 20:27:18 +0100 Subject: [PATCH 23/47] add tests for max fee calculation --- .../services/src/state_committer/fee_algo.rs | 136 +++++++++++++++--- 1 file changed, 118 insertions(+), 18 deletions(-) diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 9a3c1d40..629015fa 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -5,7 +5,7 @@ use crate::fee_analytics::{ service::FeeAnalytics, }; -use super::service::FeeAlgoConfig; +use super::service::{FeeAlgoConfig, FeeThresholds}; pub struct SendOrWaitDecider

{ fee_analytics: FeeAnalytics

, @@ -61,7 +61,8 @@ impl SendOrWaitDecider

{ } let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - let max_upper_tx_fee = self.calculate_max_upper_fee(long_term_tx_fee, context); + let max_upper_tx_fee = + Self::calculate_max_upper_fee(&self.config.fee_thresholds, long_term_tx_fee, context); let long_vs_max_delta_perc = ((max_upper_tx_fee as f64 - long_term_tx_fee as f64) / long_term_tx_fee as f64 * 100.) @@ -92,11 +93,14 @@ impl SendOrWaitDecider

{ short_term_tx_fee < max_upper_tx_fee } - // TODO: segfault test this - fn calculate_max_upper_fee(&self, fee: u128, context: Context) -> u128 { + fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + context: Context, + ) -> u128 { const PPM: u128 = 1_000_000; // 100% in PPM - let max_blocks_behind = u128::from(self.config.fee_thresholds.max_l2_blocks_behind.get()); + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); let blocks_behind = u128::from(context.num_l2_blocks_behind); debug_assert!( @@ -106,15 +110,14 @@ impl SendOrWaitDecider

{ max_blocks_behind ); - let start_discount_ppm = - percentage_to_ppm(self.config.fee_thresholds.start_discount_percentage); - let end_premium_ppm = percentage_to_ppm(self.config.fee_thresholds.end_premium_percentage); + let start_discount_ppm = percentage_to_ppm(fee_thresholds.start_discount_percentage); + let end_premium_ppm = percentage_to_ppm(fee_thresholds.end_premium_percentage); // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% let base_multiplier = PPM.saturating_sub(start_discount_ppm); // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = self.calculate_premium_increment( + let premium_increment = Self::calculate_premium_increment( start_discount_ppm, end_premium_ppm, blocks_behind, @@ -132,7 +135,6 @@ impl SendOrWaitDecider

{ } fn calculate_premium_increment( - &self, start_discount_ppm: u128, end_premium_ppm: u128, blocks_behind: u128, @@ -171,16 +173,13 @@ fn percentage_to_ppm(percentage: f64) -> u128 { #[cfg(test)] mod tests { - use std::num::NonZeroU64; - use super::*; - use crate::{ - fee_analytics::port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - Fees, - }, - state_committer::service::{FeeThresholds, SmaPeriods}, + use crate::fee_analytics::port::{ + l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + Fees, }; + use crate::state_committer::service::{FeeThresholds, SmaPeriods}; + use std::num::NonZeroU64; use test_case::test_case; use tokio; @@ -544,4 +543,105 @@ mod tests { "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", ); } + + /// Helper function to convert a percentage to Parts Per Million (PPM) + fn percentage_to_ppm_test_helper(percentage: f64) -> u128 { + (percentage * 1_000_000.0) as u128 + } + + #[test_case( + // Test Case 1: No blocks behind, no discount or premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 0, + at_l1_height: 0, + }, + 1000; + "No blocks behind, multiplier should be 100%" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20, + end_premium_percentage: 0.25, + always_acceptable_fee: 0, + }, + 2000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 2050; + "Half blocks behind with discount and premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25, + end_premium_percentage: 0.0, + always_acceptable_fee: 0, + }, + 800, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 700; + "Start discount only, no premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.0, + end_premium_percentage: 0.30, + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 1150; + "End premium only, no discount" + )] + #[test_case( + // Test Case 8: High fee with premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10, // 100,000 PPM + end_premium_percentage: 0.20, // 200,000 PPM + always_acceptable_fee: 0, + }, + 10_000, + Context { + num_l2_blocks_behind: 99, + at_l1_height: 0, + }, + 11970; + "High fee with premium" + )] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + context: Context, + expected_max_upper_fee: u128, + ) { + let max_upper_fee = SendOrWaitDecider::::calculate_max_upper_fee( + &fee_thresholds, + fee, + context, + ); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } } From d4785a0158425def99d7415a7d8c70000b4a1e81 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 16 Dec 2024 20:55:32 +0100 Subject: [PATCH 24/47] some todos --- committer/src/config.rs | 2 +- packages/adapters/eth/src/lib.rs | 22 ++------- packages/services/src/fee_analytics.rs | 46 +++++++++++-------- packages/services/src/state_committer.rs | 37 ++++++++------- .../services/src/state_committer/fee_algo.rs | 24 ++++++---- packages/services/tests/fee_analytics.rs | 17 ++++--- packages/services/tests/state_committer.rs | 8 ++-- 7 files changed, 77 insertions(+), 79 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index 734e92a6..06d8a94d 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -125,7 +125,7 @@ pub struct FeeAlgoConfig { pub long_sma_blocks: u64, /// Maximum number of unposted L2 blocks before sending a transaction regardless of fees - pub max_l2_blocks_behind: NonZeroU64, + pub max_l2_blocks_behind: NonZeroU32, /// Starting discount percentage applied we try to achieve if we're 0 l2 blocks behind pub start_discount_percentage: f64, diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index fa609e1e..27bba643 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -221,7 +221,7 @@ impl services::state_committer::port::l1::Api for WebsocketClient { } impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { - async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; @@ -232,7 +232,7 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { // TODO: segfault see when this can be None // TODO: check edgecases - let mut current_height = height_range.clone().min().unwrap(); + let mut current_height = *height_range.start(); while current_height <= *height_range.end() { // There is a comment in alloy about not doing more than 1024 blocks at a time const RPC_LIMIT: u64 = 1024; @@ -298,19 +298,16 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { }) .collect_vec(); - // eprintln!("converted into {new_fees:?}"); - - new_fees.try_into().unwrap() + Ok(new_fees.try_into().unwrap()) } - async fn current_block_height(&self) -> u64 { - self._get_block_number().await.unwrap() + async fn current_block_height(&self) -> Result { + self._get_block_number().await } } #[cfg(test)] mod test { - use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; @@ -331,13 +328,4 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } - - // #[tokio::test] - // async fn can_connect_to_eth_mainnet() { - // let current_height = client._get_block_number().await.unwrap(); - // - // let fees = FeesProvider::fees(&client, current_height - 1026..=current_height).await; - // - // panic!("{:?}", fees); - // } } diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs index 206896f5..79426574 100644 --- a/packages/services/src/fee_analytics.rs +++ b/packages/services/src/fee_analytics.rs @@ -16,7 +16,6 @@ pub mod port { use std::ops::RangeInclusive; use itertools::Itertools; - use super::BlockFees; @@ -81,8 +80,11 @@ pub mod port { #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] pub trait FeesProvider { - async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees; - async fn current_block_height(&self) -> u64; + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result; + async fn current_block_height(&self) -> crate::Result; } #[cfg(feature = "test-helpers")] @@ -90,7 +92,6 @@ pub mod port { use std::{collections::BTreeMap, ops::RangeInclusive}; use itertools::Itertools; - use crate::fee_analytics::port::{BlockFees, Fees}; @@ -108,16 +109,20 @@ pub mod port { } impl FeesProvider for ConstantFeesProvider { - async fn fees(&self, _height_range: RangeInclusive) -> SequentialBlockFees { + async fn fees( + &self, + _height_range: RangeInclusive, + ) -> crate::Result { let fees = BlockFees { - height: self.current_block_height().await, + height: self.current_block_height().await?, fees: self.fees, }; - vec![fees].try_into().unwrap() + Ok(vec![fees].try_into().unwrap()) } - async fn current_block_height(&self) -> u64 { - 0 + + async fn current_block_height(&self) -> crate::Result { + Ok(0) } } @@ -127,11 +132,18 @@ pub mod port { } impl FeesProvider for PreconfiguredFeesProvider { - async fn current_block_height(&self) -> u64 { - *self.fees.keys().last().unwrap() + async fn current_block_height(&self) -> crate::Result { + Ok(*self + .fees + .keys() + .last() + .expect("no fees registered with PreconfiguredFeesProvider")) } - async fn fees(&self, height_range: RangeInclusive) -> SequentialBlockFees { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { let fees = self .fees .iter() @@ -143,7 +155,7 @@ pub mod port { }) .collect_vec(); - fees.try_into().unwrap() + Ok(fees.try_into().expect("block fees not sequential")) } } @@ -177,8 +189,6 @@ pub mod service { use std::ops::RangeInclusive; - - use super::port::{ l1::{FeesProvider, SequentialBlockFees}, Fees, @@ -197,10 +207,8 @@ pub mod service { // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees/save to db // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> Fees { - let fees = self.fees_provider.fees(block_range).await; - - Self::mean(fees) + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + self.fees_provider.fees(block_range).await.map(Self::mean) } fn mean(fees: SequentialBlockFees) -> Fees { diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 38387532..7b9f226c 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -2,7 +2,7 @@ mod fee_algo; pub mod service { use std::{ - num::{NonZeroU64, NonZeroUsize}, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, time::Duration, }; @@ -26,7 +26,7 @@ pub mod service { // TODO: segfault validate start discount is less than end premium and both are positive #[derive(Debug, Clone, Copy)] pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU64, + pub max_l2_blocks_behind: NonZeroU32, pub start_discount_percentage: f64, pub end_premium_percentage: f64, pub always_acceptable_fee: u128, @@ -133,40 +133,39 @@ pub mod service { Ok(std_elapsed >= self.config.fragment_accumulation_timeout) } - async fn submit_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<()> { - info!("about to send at most {} fragments", fragments.len()); - - // TODO: segfault proper type conversion + async fn should_send_tx(&self, fragments: &NonEmpty) -> Result { let l1_height = self.l1_adapter.current_height().await?; - // TODO: segfault test this let l2_height = self.fuel_api.latest_height().await?; let oldest_l2_block_in_fragments = fragments .maximum_by_key(|b| b.oldest_block_in_bundle) .oldest_block_in_bundle; - let behind_on_l2 = l2_height.saturating_sub(oldest_l2_block_in_fragments); + let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); - let should_send = self - .decider + self.decider .should_send_blob_tx( - fragments.len() as u32, + u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), Context { - num_l2_blocks_behind: behind_on_l2 as u64, + num_l2_blocks_behind, at_l1_height: l1_height, }, ) - .await; + .await + } - if !should_send { - // TODO: segfault log here + async fn submit_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<()> { + if !self.should_send_tx(&fragments).await? { + info!("decided against sending fragments"); return Ok(()); } + info!("about to send at most {} fragments", fragments.len()); + let data = fragments.clone().map(|f| f.fragment); match self diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 629015fa..c6043b97 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -23,7 +23,7 @@ impl

SendOrWaitDecider

{ #[derive(Debug, Clone, Copy)] pub struct Context { - pub num_l2_blocks_behind: u64, + pub num_l2_blocks_behind: u32, pub at_l1_height: u64, } @@ -31,18 +31,22 @@ impl SendOrWaitDecider

{ // TODO: segfault validate blob number // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes // (once that is implemented) - pub async fn should_send_blob_tx(&self, num_blobs: u32, context: Context) -> bool { + pub async fn should_send_blob_tx( + &self, + num_blobs: u32, + context: Context, + ) -> crate::Result { let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; let short_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await; + .await?; let long_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await; + .await?; let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); @@ -50,14 +54,13 @@ impl SendOrWaitDecider

{ short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; eprintln!("fee always acceptable: {}", fee_always_acceptable); - // TODO: segfault test this let too_far_behind = context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get(); eprintln!("too far behind: {}", too_far_behind); if fee_always_acceptable || too_far_behind { - return true; + return Ok(true); } let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); @@ -90,7 +93,7 @@ impl SendOrWaitDecider

{ short_term_tx_fee, long_term_tx_fee, max_upper_tx_fee ); - short_term_tx_fee < max_upper_tx_fee + Ok(short_term_tx_fee < max_upper_tx_fee) } fn calculate_max_upper_fee( @@ -518,12 +521,12 @@ mod tests { new_fees: Fees, num_blobs: u32, config: FeeAlgoConfig, - num_l2_blocks_behind: u64, + num_l2_blocks_behind: u32, expected_decision: bool, ) { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = PreconfiguredFeesProvider::new(fees); - let current_block_height = fees_provider.current_block_height().await; + let current_block_height = fees_provider.current_block_height().await.unwrap(); let analytics_service = FeeAnalytics::new(fees_provider); let sut = SendOrWaitDecider::new(analytics_service, config); @@ -536,7 +539,8 @@ mod tests { num_l2_blocks_behind, }, ) - .await; + .await + .unwrap(); assert_eq!( should_send, expected_decision, diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs index 39513e98..a89f52ce 100644 --- a/packages/services/tests/fee_analytics.rs +++ b/packages/services/tests/fee_analytics.rs @@ -1,21 +1,21 @@ use std::path::PathBuf; use services::fee_analytics::{ - port::{ - l1::testing::{self}, Fees, - }, - service::FeeAnalytics, - }; + port::{ + l1::testing::{self}, + Fees, + }, + service::FeeAnalytics, +}; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { // given let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); - let last_n_blocks = 1; // when - let sma = fee_analytics.calculate_sma(4..=4).await; + let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); // then assert_eq!(sma.base_fee_per_gas, 5); @@ -28,10 +28,9 @@ async fn calculates_sma_correctly_for_last_5_blocks() { // given let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); let fee_analytics = FeeAnalytics::new(fees_provider); - let last_n_blocks = 5; // when - let sma = fee_analytics.calculate_sma(0..=4).await; + let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); // then let mean = (5 + 4 + 3 + 2 + 1) / 5; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index fcd50a98..501ccbf4 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -405,7 +405,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -516,7 +516,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -618,7 +618,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(50).unwrap(), + max_l2_blocks_behind: 50.try_into().unwrap(), start_discount_percentage: 0.0, end_premium_percentage: 0.0, always_acceptable_fee: 0, @@ -728,7 +728,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { - max_l2_blocks_behind: NonZeroU64::new(100).unwrap(), + max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, always_acceptable_fee: 0, From 7326055ff9c7d1abb85988777db5c886b29a2fed Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 09:59:31 +0100 Subject: [PATCH 25/47] add tests for converting fees --- committer/src/config.rs | 2 +- packages/adapters/eth/src/lib.rs | 600 ++++++++++++++++-- .../adapters/storage/src/mappings/tables.rs | 2 +- packages/services/src/lib.rs | 2 +- 4 files changed, 534 insertions(+), 72 deletions(-) diff --git a/committer/src/config.rs b/committer/src/config.rs index 06d8a94d..5b3b6b03 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -1,6 +1,6 @@ use std::{ net::Ipv4Addr, - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + num::{NonZeroU32, NonZeroUsize}, str::FromStr, time::Duration, }; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 27bba643..99c1d94c 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -1,5 +1,4 @@ use std::{ - cmp::min, num::{NonZeroU32, NonZeroUsize}, ops::RangeInclusive, time::Duration, @@ -9,8 +8,10 @@ use alloy::{ consensus::BlobTransactionSidecar, eips::eip4844::{BYTES_PER_BLOB, DATA_GAS_PER_BLOB}, primitives::U256, + rpc::types::FeeHistory, }; use delegate::delegate; +use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, @@ -224,96 +225,132 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; - // so that a alloy version bump doesn't surprise us const_assert!(REWARD_PERCENTILE == 20.0,); - let mut fees = vec![]; + // There is a comment in alloy about not doing more than 1024 blocks at a time + const RPC_LIMIT: u64 = 1024; - // TODO: segfault see when this can be None - // TODO: check edgecases - let mut current_height = *height_range.start(); - while current_height <= *height_range.end() { - // There is a comment in alloy about not doing more than 1024 blocks at a time - const RPC_LIMIT: u64 = 1024; + let fees: Vec = stream::iter(chunk_range_inclusive(height_range, RPC_LIMIT)) + .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) + .try_collect() + .await?; - let upper_bound = min( - current_height.saturating_add(RPC_LIMIT).saturating_sub(1), - *height_range.end(), - ); + let mut unpacked_fees = vec![]; + for fee in fees { + unpacked_fees.extend(unpack_fee_history(fee)?); + } - let history = self - .fees( - current_height..=upper_bound, - std::slice::from_ref(&REWARD_PERCENTILE), - ) - .await - .unwrap(); + unpacked_fees + .try_into() + .map_err(|e| services::Error::Other(format!("{e}"))) + } - assert_eq!( - history.reward.as_ref().unwrap().len(), - (current_height..=upper_bound).count() - ); + async fn current_block_height(&self) -> Result { + self._get_block_number().await + } +} - fees.push(history); +fn unpack_fee_history(fees: FeeHistory) -> Result> { + let number_of_blocks = if fees.base_fee_per_gas.is_empty() { + 0 + } else { + // We subtract 1 because the last element is the expected fee for the next block + fees.base_fee_per_gas + .len() + .checked_sub(1) + .expect("checked not 0") + }; + + if number_of_blocks == 0 { + return Ok(vec![]); + } - current_height = upper_bound.saturating_add(1); - } + let Some(nested_rewards) = fees.reward.as_ref() else { + return Err(services::Error::Other(format!( + "missing rewards field: {fees:?}" + ))); + }; + + if number_of_blocks != nested_rewards.len() + || number_of_blocks != fees.base_fee_per_blob_gas.len() - 1 + { + return Err(services::Error::Other(format!( + "discrepancy in lengths of fee fields: {fees:?}" + ))); + } - let new_fees = fees - .into_iter() - .flat_map(|fees| { - // TODO: segfault check if the vector is ever going to have less than 2 elements, maybe - // for block count 0? - // eprintln!("received {fees:?}"); - let number_of_blocks = fees.base_fee_per_blob_gas.len().checked_sub(1).unwrap(); - let rewards = fees - .reward - .unwrap() - .into_iter() - .map(|mut perc| perc.pop().unwrap()) - .collect_vec(); - - let oldest_block = fees.oldest_block; - - debug_assert_eq!(rewards.len(), number_of_blocks); - - izip!( - (oldest_block..), - fees.base_fee_per_gas.into_iter(), - fees.base_fee_per_blob_gas.into_iter(), - rewards - ) - .take(number_of_blocks) - .map( - |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { - height, - fees: Fees { - base_fee_per_gas, - reward, - base_fee_per_blob_gas, - }, - }, + let rewards: Vec<_> = nested_rewards + .iter() + .map(|perc| { + perc.last().copied().ok_or_else(|| { + crate::error::Error::Other( + "should have had at least one reward percentile".to_string(), ) }) - .collect_vec(); + }) + .try_collect()?; + + let values = izip!( + (fees.oldest_block..), + fees.base_fee_per_gas.into_iter(), + fees.base_fee_per_blob_gas.into_iter(), + rewards + ) + .take(number_of_blocks) + .map( + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, + }, + ) + .collect(); + + Ok(values) +} - Ok(new_fees.try_into().unwrap()) +fn chunk_range_inclusive( + initial_range: RangeInclusive, + chunk_size: u64, +) -> Vec> { + let mut ranges = Vec::new(); + + if chunk_size == 0 { + return ranges; } - async fn current_block_height(&self) -> Result { - self._get_block_number().await + let start = *initial_range.start(); + let end = *initial_range.end(); + + let mut current = start; + while current <= end { + // Calculate the end of the current chunk. + let chunk_end = (current + chunk_size - 1).min(end); + + ranges.push(current..=chunk_end); + + current = chunk_end + 1; } + + ranges } #[cfg(test)] mod test { - - use alloy::eips::eip4844::DATA_GAS_PER_BLOB; + use super::chunk_range_inclusive; + use alloy::{eips::eip4844::DATA_GAS_PER_BLOB, rpc::types::FeeHistory}; use fuel_block_committer_encoding::blob; - use services::block_bundler::port::l1::FragmentEncoder; + use services::{ + block_bundler::port::l1::FragmentEncoder, + fee_analytics::port::{BlockFees, Fees}, + }; + use std::ops::RangeInclusive; - use crate::BlobEncoder; + use crate::{unpack_fee_history, BlobEncoder}; #[test] fn gas_usage_correctly_calculated() { @@ -328,4 +365,429 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } + + #[test] + fn test_chunk_size_zero() { + // given + let initial_range = 1..=10; + let chunk_size = 0; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected: Vec> = vec![]; + assert_eq!( + result, expected, + "Expected empty vector when chunk_size is zero" + ); + } + + #[test] + fn test_chunk_size_larger_than_range() { + // given + let initial_range = 1..=5; + let chunk_size = 10; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=5]; + assert_eq!( + result, expected, + "Expected single chunk when chunk_size exceeds range length" + ); + } + + #[test] + fn test_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 2; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; + assert_eq!(result, expected, "Chunks should exactly divide the range"); + } + + #[test] + fn test_non_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 3; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; + assert_eq!( + result, expected, + "Last chunk should contain the remaining elements" + ); + } + + #[test] + fn test_single_element_range() { + // given + let initial_range = 5..=5; + let chunk_size = 1; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![5..=5]; + assert_eq!( + result, expected, + "Single element range should return one chunk with that element" + ); + } + + #[test] + fn test_start_equals_end_with_large_chunk_size() { + // given + let initial_range = 100..=100; + let chunk_size = 50; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![100..=100]; + assert_eq!( + result, expected, + "Single element range should return one chunk regardless of chunk_size" + ); + } + + #[test] + fn test_chunk_size_one() { + // given + let initial_range = 10..=15; + let chunk_size = 1; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; + assert_eq!( + result, expected, + "Each number should be its own chunk when chunk_size is one" + ); + } + + #[test] + fn test_full_range_chunk() { + // given + let initial_range = 20..=30; + let chunk_size = 11; + + // when + let result = chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![20..=30]; + assert_eq!( + result, expected, + "Whole range should be a single chunk when chunk_size equals range size" + ); + } + + #[test] + fn test_unpack_fee_history_empty_base_fee() { + // given + let fees = FeeHistory { + oldest_block: 100, + base_fee_per_gas: vec![], + base_fee_per_blob_gas: vec![], + reward: Some(vec![]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected: Vec = vec![]; + assert_eq!( + result.unwrap(), + expected, + "Expected empty vector when base_fee_per_gas is empty" + ); + } + + #[test] + fn test_unpack_fee_history_missing_rewards() { + // given + let fees = FeeHistory { + oldest_block: 200, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: None, + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to missing rewards field" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_base_fee_rewards() { + // given + let fees = FeeHistory { + oldest_block: 300, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250, 350], + reward: Some(vec![vec![10]]), // Should have 2 rewards for 2 blocks + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in lengths of fee fields" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_blob_gas() { + // given + let fees = FeeHistory { + oldest_block: 400, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250], // Should have 3 elements + reward: Some(vec![vec![10], vec![20]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in base_fee_per_blob_gas lengths" + ); + } + + #[test] + fn test_unpack_fee_history_empty_reward_percentile() { + // given + let fees = FeeHistory { + oldest_block: 500, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![]]), // Empty percentile + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other("should have had at least one reward percentile".to_string()); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to empty reward percentile" + ); + } + + #[test] + fn test_unpack_fee_history_single_block() { + // given + let fees = FeeHistory { + oldest_block: 600, + base_fee_per_gas: vec![100, 200], // number_of_blocks =1 + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![10]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected = vec![BlockFees { + height: 600, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }]; + assert_eq!( + result.unwrap(), + expected, + "Expected one BlockFees entry for a single block" + ); + } + + #[test] + fn test_unpack_fee_history_multiple_blocks() { + // given + let fees = FeeHistory { + oldest_block: 700, + base_fee_per_gas: vec![100, 200, 300, 400], // number_of_blocks =3 + base_fee_per_blob_gas: vec![150, 250, 350, 450], + reward: Some(vec![vec![10], vec![20], vec![30]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 700, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }, + BlockFees { + height: 701, + fees: Fees { + base_fee_per_gas: 200, + reward: 20, + base_fee_per_blob_gas: 250, + }, + }, + BlockFees { + height: 702, + fees: Fees { + base_fee_per_gas: 300, + reward: 30, + base_fee_per_blob_gas: 350, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected three BlockFees entries for three blocks" + ); + } + + #[test] + fn test_unpack_fee_history_large_values() { + // given + let fees = FeeHistory { + oldest_block: u64::MAX - 2, + base_fee_per_gas: vec![u128::MAX - 2, u128::MAX - 1, u128::MAX], + base_fee_per_blob_gas: vec![u128::MAX - 3, u128::MAX - 2, u128::MAX - 1], + reward: Some(vec![vec![u128::MAX - 4], vec![u128::MAX - 3]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees.clone()); + + // then + let expected = vec![ + BlockFees { + height: u64::MAX - 2, + fees: Fees { + base_fee_per_gas: u128::MAX - 2, + reward: u128::MAX - 4, + base_fee_per_blob_gas: u128::MAX - 3, + }, + }, + BlockFees { + height: u64::MAX - 1, + fees: Fees { + base_fee_per_gas: u128::MAX - 1, + reward: u128::MAX - 3, + base_fee_per_blob_gas: u128::MAX - 2, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries with large u64 values" + ); + } + + #[test] + fn test_unpack_fee_history_full_range_chunk() { + // given + let fees = FeeHistory { + oldest_block: 800, + base_fee_per_gas: vec![500, 600, 700, 800, 900], // number_of_blocks =4 + base_fee_per_blob_gas: vec![550, 650, 750, 850, 950], + reward: Some(vec![vec![50], vec![60], vec![70], vec![80]]), + ..Default::default() + }; + + // when + let result = unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 800, + fees: Fees { + base_fee_per_gas: 500, + reward: 50, + base_fee_per_blob_gas: 550, + }, + }, + BlockFees { + height: 801, + fees: Fees { + base_fee_per_gas: 600, + reward: 60, + base_fee_per_blob_gas: 650, + }, + }, + BlockFees { + height: 802, + fees: Fees { + base_fee_per_gas: 700, + reward: 70, + base_fee_per_blob_gas: 750, + }, + }, + BlockFees { + height: 803, + fees: Fees { + base_fee_per_gas: 800, + reward: 80, + base_fee_per_blob_gas: 850, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries matching the full range chunk" + ); + } } diff --git a/packages/adapters/storage/src/mappings/tables.rs b/packages/adapters/storage/src/mappings/tables.rs index 566e68ae..ea419785 100644 --- a/packages/adapters/storage/src/mappings/tables.rs +++ b/packages/adapters/storage/src/mappings/tables.rs @@ -1,4 +1,4 @@ -use std::{i64, num::NonZeroU32}; +use std::num::NonZeroU32; use num_bigint::BigInt; use services::types::{ diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index 69e04048..aa807be8 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -23,7 +23,7 @@ pub use block_bundler::{ pub use state_committer::service::{Config as StateCommitterConfig, StateCommitter}; use types::InvalidL1Height; -#[derive(thiserror::Error, Debug)] +#[derive(thiserror::Error, Debug, PartialEq)] pub enum Error { #[error("{0}")] Other(String), From c123862b801ee6874730a164794653224c99f096 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 10:03:16 +0100 Subject: [PATCH 26/47] move fee conversion to separate mod --- packages/adapters/eth/src/fee_conversion.rs | 532 +++++++++++++++++ packages/adapters/eth/src/lib.rs | 540 +----------------- packages/services/src/state_committer.rs | 2 +- .../services/src/state_committer/fee_algo.rs | 2 +- packages/services/tests/state_committer.rs | 2 +- 5 files changed, 548 insertions(+), 530 deletions(-) create mode 100644 packages/adapters/eth/src/fee_conversion.rs diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs new file mode 100644 index 00000000..eeb80995 --- /dev/null +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -0,0 +1,532 @@ +use std::ops::RangeInclusive; + +use alloy::rpc::types::FeeHistory; +use itertools::{izip, Itertools}; +use services::Result; + +use services::fee_analytics::port::Fees; + +use services::fee_analytics::port::BlockFees; + +pub fn unpack_fee_history(fees: FeeHistory) -> Result> { + let number_of_blocks = if fees.base_fee_per_gas.is_empty() { + 0 + } else { + // We subtract 1 because the last element is the expected fee for the next block + fees.base_fee_per_gas + .len() + .checked_sub(1) + .expect("checked not 0") + }; + + if number_of_blocks == 0 { + return Ok(vec![]); + } + + let Some(nested_rewards) = fees.reward.as_ref() else { + return Err(services::Error::Other(format!( + "missing rewards field: {fees:?}" + ))); + }; + + if number_of_blocks != nested_rewards.len() + || number_of_blocks != fees.base_fee_per_blob_gas.len() - 1 + { + return Err(services::Error::Other(format!( + "discrepancy in lengths of fee fields: {fees:?}" + ))); + } + + let rewards: Vec<_> = nested_rewards + .iter() + .map(|perc| { + perc.last().copied().ok_or_else(|| { + crate::error::Error::Other( + "should have had at least one reward percentile".to_string(), + ) + }) + }) + .try_collect()?; + + let values = izip!( + (fees.oldest_block..), + fees.base_fee_per_gas.into_iter(), + fees.base_fee_per_blob_gas.into_iter(), + rewards + ) + .take(number_of_blocks) + .map( + |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { + height, + fees: Fees { + base_fee_per_gas, + reward, + base_fee_per_blob_gas, + }, + }, + ) + .collect(); + + Ok(values) +} + +pub fn chunk_range_inclusive( + initial_range: RangeInclusive, + chunk_size: u64, +) -> Vec> { + let mut ranges = Vec::new(); + + if chunk_size == 0 { + return ranges; + } + + let start = *initial_range.start(); + let end = *initial_range.end(); + + let mut current = start; + while current <= end { + // Calculate the end of the current chunk. + let chunk_end = (current + chunk_size - 1).min(end); + + ranges.push(current..=chunk_end); + + current = chunk_end + 1; + } + + ranges +} + +#[cfg(test)] +mod test { + use alloy::rpc::types::FeeHistory; + + use services::fee_analytics::port::{BlockFees, Fees}; + use std::ops::RangeInclusive; + + use crate::fee_conversion::{self}; + + #[test] + fn test_chunk_size_zero() { + // given + let initial_range = 1..=10; + let chunk_size = 0; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected: Vec> = vec![]; + assert_eq!( + result, expected, + "Expected empty vector when chunk_size is zero" + ); + } + + #[test] + fn test_chunk_size_larger_than_range() { + // given + let initial_range = 1..=5; + let chunk_size = 10; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=5]; + assert_eq!( + result, expected, + "Expected single chunk when chunk_size exceeds range length" + ); + } + + #[test] + fn test_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 2; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; + assert_eq!(result, expected, "Chunks should exactly divide the range"); + } + + #[test] + fn test_non_exact_multiples() { + // given + let initial_range = 1..=10; + let chunk_size = 3; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; + assert_eq!( + result, expected, + "Last chunk should contain the remaining elements" + ); + } + + #[test] + fn test_single_element_range() { + // given + let initial_range = 5..=5; + let chunk_size = 1; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![5..=5]; + assert_eq!( + result, expected, + "Single element range should return one chunk with that element" + ); + } + + #[test] + fn test_start_equals_end_with_large_chunk_size() { + // given + let initial_range = 100..=100; + let chunk_size = 50; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![100..=100]; + assert_eq!( + result, expected, + "Single element range should return one chunk regardless of chunk_size" + ); + } + + #[test] + fn test_chunk_size_one() { + // given + let initial_range = 10..=15; + let chunk_size = 1; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; + assert_eq!( + result, expected, + "Each number should be its own chunk when chunk_size is one" + ); + } + + #[test] + fn test_full_range_chunk() { + // given + let initial_range = 20..=30; + let chunk_size = 11; + + // when + let result = fee_conversion::chunk_range_inclusive(initial_range, chunk_size); + + // then + let expected = vec![20..=30]; + assert_eq!( + result, expected, + "Whole range should be a single chunk when chunk_size equals range size" + ); + } + + #[test] + fn test_unpack_fee_history_empty_base_fee() { + // given + let fees = FeeHistory { + oldest_block: 100, + base_fee_per_gas: vec![], + base_fee_per_blob_gas: vec![], + reward: Some(vec![]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected: Vec = vec![]; + assert_eq!( + result.unwrap(), + expected, + "Expected empty vector when base_fee_per_gas is empty" + ); + } + + #[test] + fn test_unpack_fee_history_missing_rewards() { + // given + let fees = FeeHistory { + oldest_block: 200, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: None, + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to missing rewards field" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_base_fee_rewards() { + // given + let fees = FeeHistory { + oldest_block: 300, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250, 350], + reward: Some(vec![vec![10]]), // Should have 2 rewards for 2 blocks + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in lengths of fee fields" + ); + } + + #[test] + fn test_unpack_fee_history_discrepancy_in_lengths_blob_gas() { + // given + let fees = FeeHistory { + oldest_block: 400, + base_fee_per_gas: vec![100, 200, 300], + base_fee_per_blob_gas: vec![150, 250], // Should have 3 elements + reward: Some(vec![vec![10], vec![20]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to discrepancy in base_fee_per_blob_gas lengths" + ); + } + + #[test] + fn test_unpack_fee_history_empty_reward_percentile() { + // given + let fees = FeeHistory { + oldest_block: 500, + base_fee_per_gas: vec![100, 200], + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![]]), // Empty percentile + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected_error = + services::Error::Other("should have had at least one reward percentile".to_string()); + assert_eq!( + result.unwrap_err(), + expected_error, + "Expected error due to empty reward percentile" + ); + } + + #[test] + fn test_unpack_fee_history_single_block() { + // given + let fees = FeeHistory { + oldest_block: 600, + base_fee_per_gas: vec![100, 200], // number_of_blocks =1 + base_fee_per_blob_gas: vec![150, 250], + reward: Some(vec![vec![10]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected = vec![BlockFees { + height: 600, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }]; + assert_eq!( + result.unwrap(), + expected, + "Expected one BlockFees entry for a single block" + ); + } + + #[test] + fn test_unpack_fee_history_multiple_blocks() { + // given + let fees = FeeHistory { + oldest_block: 700, + base_fee_per_gas: vec![100, 200, 300, 400], // number_of_blocks =3 + base_fee_per_blob_gas: vec![150, 250, 350, 450], + reward: Some(vec![vec![10], vec![20], vec![30]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 700, + fees: Fees { + base_fee_per_gas: 100, + reward: 10, + base_fee_per_blob_gas: 150, + }, + }, + BlockFees { + height: 701, + fees: Fees { + base_fee_per_gas: 200, + reward: 20, + base_fee_per_blob_gas: 250, + }, + }, + BlockFees { + height: 702, + fees: Fees { + base_fee_per_gas: 300, + reward: 30, + base_fee_per_blob_gas: 350, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected three BlockFees entries for three blocks" + ); + } + + #[test] + fn test_unpack_fee_history_large_values() { + // given + let fees = FeeHistory { + oldest_block: u64::MAX - 2, + base_fee_per_gas: vec![u128::MAX - 2, u128::MAX - 1, u128::MAX], + base_fee_per_blob_gas: vec![u128::MAX - 3, u128::MAX - 2, u128::MAX - 1], + reward: Some(vec![vec![u128::MAX - 4], vec![u128::MAX - 3]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees.clone()); + + // then + let expected = vec![ + BlockFees { + height: u64::MAX - 2, + fees: Fees { + base_fee_per_gas: u128::MAX - 2, + reward: u128::MAX - 4, + base_fee_per_blob_gas: u128::MAX - 3, + }, + }, + BlockFees { + height: u64::MAX - 1, + fees: Fees { + base_fee_per_gas: u128::MAX - 1, + reward: u128::MAX - 3, + base_fee_per_blob_gas: u128::MAX - 2, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries with large u64 values" + ); + } + + #[test] + fn test_unpack_fee_history_full_range_chunk() { + // given + let fees = FeeHistory { + oldest_block: 800, + base_fee_per_gas: vec![500, 600, 700, 800, 900], // number_of_blocks =4 + base_fee_per_blob_gas: vec![550, 650, 750, 850, 950], + reward: Some(vec![vec![50], vec![60], vec![70], vec![80]]), + ..Default::default() + }; + + // when + let result = fee_conversion::unpack_fee_history(fees); + + // then + let expected = vec![ + BlockFees { + height: 800, + fees: Fees { + base_fee_per_gas: 500, + reward: 50, + base_fee_per_blob_gas: 550, + }, + }, + BlockFees { + height: 801, + fees: Fees { + base_fee_per_gas: 600, + reward: 60, + base_fee_per_blob_gas: 650, + }, + }, + BlockFees { + height: 802, + fees: Fees { + base_fee_per_gas: 700, + reward: 70, + base_fee_per_blob_gas: 750, + }, + }, + BlockFees { + height: 803, + fees: Fees { + base_fee_per_gas: 800, + reward: 80, + base_fee_per_blob_gas: 850, + }, + }, + ]; + assert_eq!( + result.unwrap(), + expected, + "Expected BlockFees entries matching the full range chunk" + ); + } +} diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 99c1d94c..e0df5cd7 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - fee_analytics::port::{l1::SequentialBlockFees, BlockFees, Fees}, + fee_analytics::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -24,6 +24,7 @@ use services::{ mod aws; mod error; +mod fee_conversion; mod metrics; mod websocket; @@ -231,14 +232,17 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { // There is a comment in alloy about not doing more than 1024 blocks at a time const RPC_LIMIT: u64 = 1024; - let fees: Vec = stream::iter(chunk_range_inclusive(height_range, RPC_LIMIT)) - .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) - .try_collect() - .await?; + let fees: Vec = stream::iter(fee_conversion::chunk_range_inclusive( + height_range, + RPC_LIMIT, + )) + .then(|range| self.fees(range, std::slice::from_ref(&REWARD_PERCENTILE))) + .try_collect() + .await?; let mut unpacked_fees = vec![]; for fee in fees { - unpacked_fees.extend(unpack_fee_history(fee)?); + unpacked_fees.extend(fee_conversion::unpack_fee_history(fee)?); } unpacked_fees @@ -251,106 +255,13 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { } } -fn unpack_fee_history(fees: FeeHistory) -> Result> { - let number_of_blocks = if fees.base_fee_per_gas.is_empty() { - 0 - } else { - // We subtract 1 because the last element is the expected fee for the next block - fees.base_fee_per_gas - .len() - .checked_sub(1) - .expect("checked not 0") - }; - - if number_of_blocks == 0 { - return Ok(vec![]); - } - - let Some(nested_rewards) = fees.reward.as_ref() else { - return Err(services::Error::Other(format!( - "missing rewards field: {fees:?}" - ))); - }; - - if number_of_blocks != nested_rewards.len() - || number_of_blocks != fees.base_fee_per_blob_gas.len() - 1 - { - return Err(services::Error::Other(format!( - "discrepancy in lengths of fee fields: {fees:?}" - ))); - } - - let rewards: Vec<_> = nested_rewards - .iter() - .map(|perc| { - perc.last().copied().ok_or_else(|| { - crate::error::Error::Other( - "should have had at least one reward percentile".to_string(), - ) - }) - }) - .try_collect()?; - - let values = izip!( - (fees.oldest_block..), - fees.base_fee_per_gas.into_iter(), - fees.base_fee_per_blob_gas.into_iter(), - rewards - ) - .take(number_of_blocks) - .map( - |(height, base_fee_per_gas, base_fee_per_blob_gas, reward)| BlockFees { - height, - fees: Fees { - base_fee_per_gas, - reward, - base_fee_per_blob_gas, - }, - }, - ) - .collect(); - - Ok(values) -} - -fn chunk_range_inclusive( - initial_range: RangeInclusive, - chunk_size: u64, -) -> Vec> { - let mut ranges = Vec::new(); - - if chunk_size == 0 { - return ranges; - } - - let start = *initial_range.start(); - let end = *initial_range.end(); - - let mut current = start; - while current <= end { - // Calculate the end of the current chunk. - let chunk_end = (current + chunk_size - 1).min(end); - - ranges.push(current..=chunk_end); - - current = chunk_end + 1; - } - - ranges -} - #[cfg(test)] mod test { - use super::chunk_range_inclusive; - use alloy::{eips::eip4844::DATA_GAS_PER_BLOB, rpc::types::FeeHistory}; + use alloy::eips::eip4844::DATA_GAS_PER_BLOB; use fuel_block_committer_encoding::blob; - use services::{ - block_bundler::port::l1::FragmentEncoder, - fee_analytics::port::{BlockFees, Fees}, - }; - use std::ops::RangeInclusive; + use services::block_bundler::port::l1::FragmentEncoder; - use crate::{unpack_fee_history, BlobEncoder}; + use crate::BlobEncoder; #[test] fn gas_usage_correctly_calculated() { @@ -365,429 +276,4 @@ mod test { // then assert_eq!(gas_usage, 4 * DATA_GAS_PER_BLOB); } - - #[test] - fn test_chunk_size_zero() { - // given - let initial_range = 1..=10; - let chunk_size = 0; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected: Vec> = vec![]; - assert_eq!( - result, expected, - "Expected empty vector when chunk_size is zero" - ); - } - - #[test] - fn test_chunk_size_larger_than_range() { - // given - let initial_range = 1..=5; - let chunk_size = 10; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![1..=5]; - assert_eq!( - result, expected, - "Expected single chunk when chunk_size exceeds range length" - ); - } - - #[test] - fn test_exact_multiples() { - // given - let initial_range = 1..=10; - let chunk_size = 2; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![1..=2, 3..=4, 5..=6, 7..=8, 9..=10]; - assert_eq!(result, expected, "Chunks should exactly divide the range"); - } - - #[test] - fn test_non_exact_multiples() { - // given - let initial_range = 1..=10; - let chunk_size = 3; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![1..=3, 4..=6, 7..=9, 10..=10]; - assert_eq!( - result, expected, - "Last chunk should contain the remaining elements" - ); - } - - #[test] - fn test_single_element_range() { - // given - let initial_range = 5..=5; - let chunk_size = 1; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![5..=5]; - assert_eq!( - result, expected, - "Single element range should return one chunk with that element" - ); - } - - #[test] - fn test_start_equals_end_with_large_chunk_size() { - // given - let initial_range = 100..=100; - let chunk_size = 50; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![100..=100]; - assert_eq!( - result, expected, - "Single element range should return one chunk regardless of chunk_size" - ); - } - - #[test] - fn test_chunk_size_one() { - // given - let initial_range = 10..=15; - let chunk_size = 1; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![10..=10, 11..=11, 12..=12, 13..=13, 14..=14, 15..=15]; - assert_eq!( - result, expected, - "Each number should be its own chunk when chunk_size is one" - ); - } - - #[test] - fn test_full_range_chunk() { - // given - let initial_range = 20..=30; - let chunk_size = 11; - - // when - let result = chunk_range_inclusive(initial_range, chunk_size); - - // then - let expected = vec![20..=30]; - assert_eq!( - result, expected, - "Whole range should be a single chunk when chunk_size equals range size" - ); - } - - #[test] - fn test_unpack_fee_history_empty_base_fee() { - // given - let fees = FeeHistory { - oldest_block: 100, - base_fee_per_gas: vec![], - base_fee_per_blob_gas: vec![], - reward: Some(vec![]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected: Vec = vec![]; - assert_eq!( - result.unwrap(), - expected, - "Expected empty vector when base_fee_per_gas is empty" - ); - } - - #[test] - fn test_unpack_fee_history_missing_rewards() { - // given - let fees = FeeHistory { - oldest_block: 200, - base_fee_per_gas: vec![100, 200], - base_fee_per_blob_gas: vec![150, 250], - reward: None, - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = services::Error::Other(format!("missing rewards field: {:?}", fees)); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to missing rewards field" - ); - } - - #[test] - fn test_unpack_fee_history_discrepancy_in_lengths_base_fee_rewards() { - // given - let fees = FeeHistory { - oldest_block: 300, - base_fee_per_gas: vec![100, 200, 300], - base_fee_per_blob_gas: vec![150, 250, 350], - reward: Some(vec![vec![10]]), // Should have 2 rewards for 2 blocks - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = - services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to discrepancy in lengths of fee fields" - ); - } - - #[test] - fn test_unpack_fee_history_discrepancy_in_lengths_blob_gas() { - // given - let fees = FeeHistory { - oldest_block: 400, - base_fee_per_gas: vec![100, 200, 300], - base_fee_per_blob_gas: vec![150, 250], // Should have 3 elements - reward: Some(vec![vec![10], vec![20]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = - services::Error::Other(format!("discrepancy in lengths of fee fields: {:?}", fees)); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to discrepancy in base_fee_per_blob_gas lengths" - ); - } - - #[test] - fn test_unpack_fee_history_empty_reward_percentile() { - // given - let fees = FeeHistory { - oldest_block: 500, - base_fee_per_gas: vec![100, 200], - base_fee_per_blob_gas: vec![150, 250], - reward: Some(vec![vec![]]), // Empty percentile - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected_error = - services::Error::Other("should have had at least one reward percentile".to_string()); - assert_eq!( - result.unwrap_err(), - expected_error, - "Expected error due to empty reward percentile" - ); - } - - #[test] - fn test_unpack_fee_history_single_block() { - // given - let fees = FeeHistory { - oldest_block: 600, - base_fee_per_gas: vec![100, 200], // number_of_blocks =1 - base_fee_per_blob_gas: vec![150, 250], - reward: Some(vec![vec![10]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected = vec![BlockFees { - height: 600, - fees: Fees { - base_fee_per_gas: 100, - reward: 10, - base_fee_per_blob_gas: 150, - }, - }]; - assert_eq!( - result.unwrap(), - expected, - "Expected one BlockFees entry for a single block" - ); - } - - #[test] - fn test_unpack_fee_history_multiple_blocks() { - // given - let fees = FeeHistory { - oldest_block: 700, - base_fee_per_gas: vec![100, 200, 300, 400], // number_of_blocks =3 - base_fee_per_blob_gas: vec![150, 250, 350, 450], - reward: Some(vec![vec![10], vec![20], vec![30]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected = vec![ - BlockFees { - height: 700, - fees: Fees { - base_fee_per_gas: 100, - reward: 10, - base_fee_per_blob_gas: 150, - }, - }, - BlockFees { - height: 701, - fees: Fees { - base_fee_per_gas: 200, - reward: 20, - base_fee_per_blob_gas: 250, - }, - }, - BlockFees { - height: 702, - fees: Fees { - base_fee_per_gas: 300, - reward: 30, - base_fee_per_blob_gas: 350, - }, - }, - ]; - assert_eq!( - result.unwrap(), - expected, - "Expected three BlockFees entries for three blocks" - ); - } - - #[test] - fn test_unpack_fee_history_large_values() { - // given - let fees = FeeHistory { - oldest_block: u64::MAX - 2, - base_fee_per_gas: vec![u128::MAX - 2, u128::MAX - 1, u128::MAX], - base_fee_per_blob_gas: vec![u128::MAX - 3, u128::MAX - 2, u128::MAX - 1], - reward: Some(vec![vec![u128::MAX - 4], vec![u128::MAX - 3]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees.clone()); - - // then - let expected = vec![ - BlockFees { - height: u64::MAX - 2, - fees: Fees { - base_fee_per_gas: u128::MAX - 2, - reward: u128::MAX - 4, - base_fee_per_blob_gas: u128::MAX - 3, - }, - }, - BlockFees { - height: u64::MAX - 1, - fees: Fees { - base_fee_per_gas: u128::MAX - 1, - reward: u128::MAX - 3, - base_fee_per_blob_gas: u128::MAX - 2, - }, - }, - ]; - assert_eq!( - result.unwrap(), - expected, - "Expected BlockFees entries with large u64 values" - ); - } - - #[test] - fn test_unpack_fee_history_full_range_chunk() { - // given - let fees = FeeHistory { - oldest_block: 800, - base_fee_per_gas: vec![500, 600, 700, 800, 900], // number_of_blocks =4 - base_fee_per_blob_gas: vec![550, 650, 750, 850, 950], - reward: Some(vec![vec![50], vec![60], vec![70], vec![80]]), - ..Default::default() - }; - - // when - let result = unpack_fee_history(fees); - - // then - let expected = vec![ - BlockFees { - height: 800, - fees: Fees { - base_fee_per_gas: 500, - reward: 50, - base_fee_per_blob_gas: 550, - }, - }, - BlockFees { - height: 801, - fees: Fees { - base_fee_per_gas: 600, - reward: 60, - base_fee_per_blob_gas: 650, - }, - }, - BlockFees { - height: 802, - fees: Fees { - base_fee_per_gas: 700, - reward: 70, - base_fee_per_blob_gas: 750, - }, - }, - BlockFees { - height: 803, - fees: Fees { - base_fee_per_gas: 800, - reward: 80, - base_fee_per_blob_gas: 850, - }, - }, - ]; - assert_eq!( - result.unwrap(), - expected, - "Expected BlockFees entries matching the full range chunk" - ); - } } diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 7b9f226c..5ce37565 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -2,7 +2,7 @@ mod fee_algo; pub mod service { use std::{ - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + num::{NonZeroU32, NonZeroUsize}, time::Duration, }; diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index c6043b97..ad255d76 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -182,7 +182,7 @@ mod tests { Fees, }; use crate::state_committer::service::{FeeThresholds, SmaPeriods}; - use std::num::NonZeroU64; + use test_case::test_case; use tokio; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 501ccbf4..5f655702 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -10,7 +10,7 @@ use services::{ types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; -use std::{num::NonZeroU64, time::Duration}; +use std::time::Duration; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { From 4506b80944eedd015f4b37298cdd0866dc747907 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 19:45:35 +0100 Subject: [PATCH 27/47] removing fee analytics as a service, making it a helper --- committer/src/main.rs | 4 - committer/src/setup.rs | 3 - packages/adapters/eth/src/fee_conversion.rs | 11 +- packages/adapters/eth/src/lib.rs | 9 +- packages/services/src/fee_analytics.rs | 371 ------------- packages/services/src/lib.rs | 1 - packages/services/src/state_committer.rs | 101 +++- .../services/src/state_committer/fee_algo.rs | 33 +- .../src/state_committer/fee_analytics.rs | 503 ++++++++++++++++++ packages/services/tests/fee_analytics.rs | 147 ----- packages/services/tests/state_committer.rs | 55 +- packages/services/tests/state_listener.rs | 14 +- packages/test-helpers/src/lib.rs | 11 +- 13 files changed, 630 insertions(+), 633 deletions(-) delete mode 100644 packages/services/src/fee_analytics.rs create mode 100644 packages/services/src/state_committer/fee_analytics.rs delete mode 100644 packages/services/tests/fee_analytics.rs diff --git a/committer/src/main.rs b/committer/src/main.rs index 3e1714f4..0473d0c7 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,7 +7,6 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; -use services::fee_analytics; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; @@ -73,15 +72,12 @@ async fn main() -> Result<()> { &metrics_registry, ); - let fee_analytics = fee_analytics::service::FeeAnalytics::new(ethereum_rpc.clone()); - let state_committer_handle = setup::state_committer( fuel_adapter.clone(), ethereum_rpc.clone(), storage.clone(), cancel_token.clone(), &config, - fee_analytics, ); let state_importer_handle = diff --git a/committer/src/setup.rs b/committer/src/setup.rs index b320ee40..b09449c6 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,7 +9,6 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_analytics::service::FeeAnalytics, state_committer::{ port::Storage, service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, @@ -121,7 +120,6 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, - fee_analytics: FeeAnalytics, ) -> tokio::task::JoinHandle<()> { let state_committer = services::StateCommitter::new( l1, @@ -146,7 +144,6 @@ pub fn state_committer( }, }, SystemClock, - fee_analytics, ); schedule_polling( diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index eeb80995..3063b321 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -2,11 +2,10 @@ use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; -use services::Result; - -use services::fee_analytics::port::Fees; - -use services::fee_analytics::port::BlockFees; +use services::{ + state_committer::port::l1::{BlockFees, Fees}, + Result, +}; pub fn unpack_fee_history(fees: FeeHistory) -> Result> { let number_of_blocks = if fees.base_fee_per_gas.is_empty() { @@ -99,8 +98,8 @@ pub fn chunk_range_inclusive( #[cfg(test)] mod test { use alloy::rpc::types::FeeHistory; + use services::state_committer::port::l1::{BlockFees, Fees}; - use services::fee_analytics::port::{BlockFees, Fees}; use std::ops::RangeInclusive; use crate::fee_conversion::{self}; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index e0df5cd7..2cb7fc31 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - fee_analytics::port::l1::SequentialBlockFees, + state_committer::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -220,9 +220,6 @@ impl services::state_committer::port::l1::Api for WebsocketClient { ) -> Result<(L1Tx, FragmentsSubmitted)>; } } -} - -impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; @@ -249,10 +246,6 @@ impl services::fee_analytics::port::l1::FeesProvider for WebsocketClient { .try_into() .map_err(|e| services::Error::Other(format!("{e}"))) } - - async fn current_block_height(&self) -> Result { - self._get_block_number().await - } } #[cfg(test)] diff --git a/packages/services/src/fee_analytics.rs b/packages/services/src/fee_analytics.rs deleted file mode 100644 index 79426574..00000000 --- a/packages/services/src/fee_analytics.rs +++ /dev/null @@ -1,371 +0,0 @@ -pub mod port { - #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] - pub struct Fees { - pub base_fee_per_gas: u128, - pub reward: u128, - pub base_fee_per_blob_gas: u128, - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub struct BlockFees { - pub height: u64, - pub fees: Fees, - } - - pub mod l1 { - use std::ops::RangeInclusive; - - use itertools::Itertools; - - use super::BlockFees; - - #[derive(Debug)] - pub struct SequentialBlockFees { - fees: Vec, - } - - impl IntoIterator for SequentialBlockFees { - type Item = BlockFees; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - self.fees.into_iter() - } - } - - // Cannot be empty - #[allow(clippy::len_without_is_empty)] - impl SequentialBlockFees { - pub fn len(&self) -> usize { - self.fees.len() - } - } - - #[derive(Debug)] - pub struct InvalidSequence(String); - - impl std::error::Error for InvalidSequence {} - - impl std::fmt::Display for InvalidSequence { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") - } - } - - impl TryFrom> for SequentialBlockFees { - type Error = InvalidSequence; - fn try_from(mut fees: Vec) -> Result { - if fees.is_empty() { - return Err(InvalidSequence("Input cannot be empty".to_string())); - } - - fees.sort_by_key(|f| f.height); - - let is_sequential = fees - .iter() - .tuple_windows() - .all(|(l, r)| l.height + 1 == r.height); - - let heights = fees.iter().map(|f| f.height).collect::>(); - if !is_sequential { - return Err(InvalidSequence(format!( - "blocks are not sequential by height: {heights:?}" - ))); - } - - Ok(Self { fees }) - } - } - - #[allow(async_fn_in_trait)] - #[trait_variant::make(Send)] - #[cfg_attr(feature = "test-helpers", mockall::automock)] - pub trait FeesProvider { - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result; - async fn current_block_height(&self) -> crate::Result; - } - - #[cfg(feature = "test-helpers")] - pub mod testing { - use std::{collections::BTreeMap, ops::RangeInclusive}; - - use itertools::Itertools; - - use crate::fee_analytics::port::{BlockFees, Fees}; - - use super::{FeesProvider, SequentialBlockFees}; - - #[derive(Debug, Clone, Copy)] - pub struct ConstantFeesProvider { - fees: Fees, - } - - impl ConstantFeesProvider { - pub fn new(fees: Fees) -> Self { - Self { fees } - } - } - - impl FeesProvider for ConstantFeesProvider { - async fn fees( - &self, - _height_range: RangeInclusive, - ) -> crate::Result { - let fees = BlockFees { - height: self.current_block_height().await?, - fees: self.fees, - }; - - Ok(vec![fees].try_into().unwrap()) - } - - async fn current_block_height(&self) -> crate::Result { - Ok(0) - } - } - - #[derive(Debug, Clone)] - pub struct PreconfiguredFeesProvider { - fees: BTreeMap, - } - - impl FeesProvider for PreconfiguredFeesProvider { - async fn current_block_height(&self) -> crate::Result { - Ok(*self - .fees - .keys() - .last() - .expect("no fees registered with PreconfiguredFeesProvider")) - } - - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let fees = self - .fees - .iter() - .skip_while(|(height, _)| !height_range.contains(height)) - .take_while(|(height, _)| height_range.contains(height)) - .map(|(height, fees)| BlockFees { - height: *height, - fees: *fees, - }) - .collect_vec(); - - Ok(fees.try_into().expect("block fees not sequential")) - } - } - - impl PreconfiguredFeesProvider { - pub fn new(blocks: impl IntoIterator) -> Self { - Self { - fees: blocks.into_iter().collect(), - } - } - } - - pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { - (0..num_blocks) - .map(|i| { - ( - i, - Fees { - base_fee_per_gas: i as u128 + 1, - reward: i as u128 + 1, - base_fee_per_blob_gas: i as u128 + 1, - }, - ) - }) - .collect() - } - } - } -} - -pub mod service { - - use std::ops::RangeInclusive; - - use super::port::{ - l1::{FeesProvider, SequentialBlockFees}, - Fees, - }; - - pub struct FeeAnalytics

{ - fees_provider: P, - } - impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } - } - - impl FeeAnalytics

{ - // TODO: segfault fail or signal if missing blocks/holes present - // TODO: segfault cache fees/save to db - // TODO: segfault job to update fees in the background - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - self.fees_provider.fees(block_range).await.map(Self::mean) - } - - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, - }); - - // TODO: segfault should we round to nearest here? - Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), - } - } - } -} - -#[cfg(test)] -mod tests { - use itertools::Itertools; - use port::{l1::SequentialBlockFees, BlockFees, Fees}; - - use super::*; - - #[test] - fn can_create_valid_sequential_fees() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees.clone()); - - // Then - assert!( - result.is_ok(), - "Expected SequentialBlockFees creation to succeed" - ); - let sequential_fees = result.unwrap(); - assert_eq!(sequential_fees.len(), block_fees.len()); - } - - #[test] - fn sequential_fees_cannot_be_empty() { - // Given - let block_fees: Vec = vec![]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for empty input" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"Input cannot be empty\")" - ); - } - - #[test] - fn fees_must_be_sequential() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 3, // Non-sequential height - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for non-sequential heights" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" - ); - } - - // TODO: segfault add more tests so that the in-order iteration invariant is properly tested - #[test] - fn produced_iterator_gives_correct_values() { - // Given - // notice the heights are out of order so that we validate that the returned sequence is in - // order - let block_fees = vec![ - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - ]; - let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); - - // When - let iterated_fees: Vec = sequential_fees.into_iter().collect(); - - // Then - let expectation = block_fees - .into_iter() - .sorted_by_key(|b| b.height) - .collect_vec(); - assert_eq!( - iterated_fees, expectation, - "Expected iterator to yield the same block fees" - ); - } -} diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index aa807be8..c6ceb616 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,7 +2,6 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; -pub mod fee_analytics; pub mod health_reporter; pub mod state_committer; pub mod state_listener; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 5ce37565..5d53539f 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,4 +1,5 @@ mod fee_algo; +mod fee_analytics; pub mod service { use std::{ @@ -7,7 +8,6 @@ pub mod service { }; use crate::{ - fee_analytics::service::FeeAnalytics, state_committer::fee_algo::Context, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Result, Runner, @@ -71,18 +71,19 @@ pub mod service { } /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, - decider: SendOrWaitDecider, + decider: SendOrWaitDecider, } - impl StateCommitter + impl StateCommitter where + L1: Clone + Send + Sync, Clock: crate::state_committer::port::Clock, { /// Creates a new `StateCommitter`. @@ -92,29 +93,27 @@ pub mod service { storage: Db, config: Config, clock: Clock, - fee_analytics: FeeAnalytics, ) -> Self { let startup_time = clock.now(); let price_algo = config.fee_algo; Self { - l1_adapter, + l1_adapter: l1_adapter.clone(), fuel_api, storage, config, clock, startup_time, - decider: SendOrWaitDecider::new(fee_analytics, price_algo), + decider: SendOrWaitDecider::new(l1_adapter, price_algo), } } } - impl StateCommitter + impl StateCommitter where - L1: crate::state_committer::port::l1::Api, + L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, - FeeProvider: crate::fee_analytics::port::l1::FeesProvider, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -301,14 +300,12 @@ pub mod service { } } - impl Runner - for StateCommitter + impl Runner for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, - FeeProvider: crate::fee_analytics::port::l1::FeesProvider + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -331,13 +328,15 @@ pub mod port { }; pub mod l1 { + use std::ops::RangeInclusive; + use nonempty::NonEmpty; + pub use crate::state_committer::fee_analytics::{BlockFees, Fees, SequentialBlockFees}; use crate::{ types::{BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx}, Result, }; - #[allow(async_fn_in_trait)] #[trait_variant::make(Send)] #[cfg_attr(feature = "test-helpers", mockall::automock)] @@ -355,6 +354,80 @@ pub mod port { fragments: NonEmpty, previous_tx: Option, ) -> Result<(L1Tx, FragmentsSubmitted)>; + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result; + } + + #[cfg(feature = "test-helpers")] + pub mod testing { + use std::{ops::RangeInclusive, sync::Arc}; + + use nonempty::NonEmpty; + + use crate::{ + state_committer::fee_analytics::{ + testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + FeesProvider, + }, + types::{FragmentsSubmitted, L1Tx}, + }; + + use super::{Api, Fees, MockApi, SequentialBlockFees}; + + #[derive(Clone)] + pub struct ApiMockWFees { + pub api: Arc, + fee_provider: Fees, + } + + impl ApiMockWFees { + pub fn new(api: MockApi) -> Self { + Self { + api: Arc::new(api), + fee_provider: ConstantFeesProvider::new(Fees::default()), + } + } + } + + impl ApiMockWFees { + pub fn w_preconfigured_fees( + self, + fees: impl IntoIterator, + ) -> ApiMockWFees { + ApiMockWFees { + api: self.api, + fee_provider: PreconfiguredFeesProvider::new(fees), + } + } + } + + impl Api for ApiMockWFees + where + T: FeesProvider + Send + Sync, + { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + FeesProvider::fees(&self.fee_provider, height_range).await + } + + async fn current_height(&self) -> crate::Result { + self.api.current_height().await + } + + async fn submit_state_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> crate::Result<(L1Tx, FragmentsSubmitted)> { + self.api + .submit_state_fragments(fragments, previous_tx) + .await + } + } } } diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index ad255d76..a247e670 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -1,21 +1,20 @@ use std::cmp::min; -use crate::fee_analytics::{ - port::{l1::FeesProvider, Fees}, - service::FeeAnalytics, +use super::{ + fee_analytics::{FeeAnalytics, FeesProvider}, + port::l1::Fees, + service::{FeeAlgoConfig, FeeThresholds}, }; -use super::service::{FeeAlgoConfig, FeeThresholds}; - pub struct SendOrWaitDecider

{ fee_analytics: FeeAnalytics

, config: FeeAlgoConfig, } impl

SendOrWaitDecider

{ - pub fn new(fee_analytics: FeeAnalytics

, config: FeeAlgoConfig) -> Self { + pub fn new(fee_provider: P, config: FeeAlgoConfig) -> Self { Self { - fee_analytics, + fee_analytics: FeeAnalytics::new(fee_provider), config, } } @@ -28,7 +27,6 @@ pub struct Context { } impl SendOrWaitDecider

{ - // TODO: segfault validate blob number // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes // (once that is implemented) pub async fn should_send_blob_tx( @@ -36,6 +34,8 @@ impl SendOrWaitDecider

{ num_blobs: u32, context: Context, ) -> crate::Result { + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; let short_term_sma = self @@ -177,12 +177,11 @@ fn percentage_to_ppm(percentage: f64) -> u128 { #[cfg(test)] mod tests { use super::*; - use crate::fee_analytics::port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - Fees, + use crate::state_committer::{ + fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + service::{FeeThresholds, SmaPeriods}, }; - use crate::state_committer::service::{FeeThresholds, SmaPeriods}; - + use test_case::test_case; use tokio; @@ -527,9 +526,8 @@ mod tests { let fees = generate_fees(config, old_fees, new_fees); let fees_provider = PreconfiguredFeesProvider::new(fees); let current_block_height = fees_provider.current_block_height().await.unwrap(); - let analytics_service = FeeAnalytics::new(fees_provider); - let sut = SendOrWaitDecider::new(analytics_service, config); + let sut = SendOrWaitDecider::new(fees_provider, config); let should_send = sut .should_send_blob_tx( @@ -548,11 +546,6 @@ mod tests { ); } - /// Helper function to convert a percentage to Parts Per Million (PPM) - fn percentage_to_ppm_test_helper(percentage: f64) -> u128 { - (percentage * 1_000_000.0) as u128 - } - #[test_case( // Test Case 1: No blocks behind, no discount or premium FeeThresholds { diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs new file mode 100644 index 00000000..29633b26 --- /dev/null +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -0,0 +1,503 @@ +use std::ops::RangeInclusive; + +use itertools::Itertools; + +use crate::state_committer::port::l1::Api; + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct Fees { + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlockFees { + pub height: u64, + pub fees: Fees, +} + +#[derive(Debug)] +pub struct SequentialBlockFees { + fees: Vec, +} + +#[allow(async_fn_in_trait)] +#[trait_variant::make(Send)] +#[cfg_attr(feature = "test-helpers", mockall::automock)] +pub trait FeesProvider { + async fn fees(&self, height_range: RangeInclusive) -> crate::Result; + async fn current_block_height(&self) -> crate::Result; +} + +impl FeesProvider for T { + async fn fees(&self, height_range: RangeInclusive) -> crate::Result { + Api::fees(self, height_range).await + } + + async fn current_block_height(&self) -> crate::Result { + Api::current_height(self).await + } +} + +impl IntoIterator for SequentialBlockFees { + type Item = BlockFees; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.fees.into_iter() + } +} + +// Cannot be empty +#[allow(clippy::len_without_is_empty)] +impl SequentialBlockFees { + pub fn len(&self) -> usize { + self.fees.len() + } +} + +#[derive(Debug)] +pub struct InvalidSequence(String); + +impl std::error::Error for InvalidSequence {} + +impl std::fmt::Display for InvalidSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl TryFrom> for SequentialBlockFees { + type Error = InvalidSequence; + fn try_from(mut fees: Vec) -> Result { + if fees.is_empty() { + return Err(InvalidSequence("Input cannot be empty".to_string())); + } + + fees.sort_by_key(|f| f.height); + + let is_sequential = fees + .iter() + .tuple_windows() + .all(|(l, r)| l.height + 1 == r.height); + + let heights = fees.iter().map(|f| f.height).collect::>(); + if !is_sequential { + return Err(InvalidSequence(format!( + "blocks are not sequential by height: {heights:?}" + ))); + } + + Ok(Self { fees }) + } +} + +#[cfg(feature = "test-helpers")] +pub mod testing { + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use itertools::Itertools; + + use crate::state_committer::port::l1::{BlockFees, Fees}; + + use super::{FeesProvider, SequentialBlockFees}; + + #[derive(Debug, Clone, Copy)] + pub struct ConstantFeesProvider { + fees: Fees, + } + + impl ConstantFeesProvider { + pub fn new(fees: Fees) -> Self { + Self { fees } + } + } + + impl FeesProvider for ConstantFeesProvider { + async fn fees( + &self, + _height_range: RangeInclusive, + ) -> crate::Result { + let fees = BlockFees { + height: self.current_block_height().await?, + fees: self.fees, + }; + + Ok(vec![fees].try_into().unwrap()) + } + + async fn current_block_height(&self) -> crate::Result { + Ok(0) + } + } + + #[derive(Debug, Clone)] + pub struct PreconfiguredFeesProvider { + fees: BTreeMap, + } + + impl FeesProvider for PreconfiguredFeesProvider { + async fn current_block_height(&self) -> crate::Result { + Ok(*self + .fees + .keys() + .last() + .expect("no fees registered with PreconfiguredFeesProvider")) + } + + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let fees = self + .fees + .iter() + .skip_while(|(height, _)| !height_range.contains(height)) + .take_while(|(height, _)| height_range.contains(height)) + .map(|(height, fees)| BlockFees { + height: *height, + fees: *fees, + }) + .collect_vec(); + + Ok(fees.try_into().expect("block fees not sequential")) + } + } + + impl PreconfiguredFeesProvider { + pub fn new(blocks: impl IntoIterator) -> Self { + Self { + fees: blocks.into_iter().collect(), + } + } + } + + pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { + (0..num_blocks) + .map(|i| { + ( + i, + Fees { + base_fee_per_gas: i as u128 + 1, + reward: i as u128 + 1, + base_fee_per_blob_gas: i as u128 + 1, + }, + ) + }) + .collect() + } +} + +pub struct FeeAnalytics

{ + fees_provider: P, +} +impl

FeeAnalytics

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } + } +} + +impl FeeAnalytics

{ + // TODO: segfault fail or signal if missing blocks/holes present + // TODO: segfault cache fees/save to db + // TODO: segfault job to update fees in the background + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + self.fees_provider.fees(block_range).await.map(Self::mean) + } + + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + + let total = fees + .into_iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + } + } +} + +#[cfg(test)] +mod tests { + use itertools::Itertools; + + use super::*; + + #[test] + fn can_create_valid_sequential_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn sequential_fees_cannot_be_empty() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn fees_must_be_sequential() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" + ); + } + + // TODO: segfault add more tests so that the in-order iteration invariant is properly tested + #[test] + fn produced_iterator_gives_correct_values() { + // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order + let block_fees = vec![ + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); + assert_eq!( + iterated_fees, expectation, + "Expected iterator to yield the same block fees" + ); + } + use std::path::PathBuf; + + #[tokio::test] + async fn calculates_sma_correctly_for_last_1_block() { + // given + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); + + // then + assert_eq!(sma.base_fee_per_gas, 5); + assert_eq!(sma.reward, 5); + assert_eq!(sma.base_fee_per_blob_gas, 5); + } + + #[tokio::test] + async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); + + // then + let mean = (5 + 4 + 3 + 2 + 1) / 5; + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); + } + + fn calculate_tx_fee(fees: &Fees) -> u128 { + 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + } + + fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + let mut csv_writer = + csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + .unwrap(); + csv_writer + .write_record(["height", "tx_fee"].iter()) + .unwrap(); + for (height, fee) in tx_fees { + csv_writer + .write_record([height.to_string(), fee.to_string()]) + .unwrap(); + } + csv_writer.flush().unwrap(); + } + + // #[tokio::test] + // async fn something() { + // let client = make_pub_eth_client().await; + // use services::fee_analytics::port::l1::FeesProvider; + // + // let current_block_height = 21408300; + // let starting_block_height = current_block_height - 48 * 3600 / 12; + // let data = client + // .fees(starting_block_height..=current_block_height) + // .await + // .into_iter() + // .collect::>(); + // + // let fee_lookup = data + // .iter() + // .map(|b| (b.height, b.fees)) + // .collect::>(); + // + // let short_sma = 25u64; + // let long_sma = 900; + // + // let current_tx_fees = data + // .iter() + // .map(|b| (b.height, calculate_tx_fee(&b.fees))) + // .collect::>(); + // + // save_tx_fees(¤t_tx_fees, "current_fees.csv"); + // + // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + // let fee_analytics = FeeAnalytics::new(local_client.clone()); + // + // let mut short_sma_tx_fees = vec![]; + // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - short_sma..=height) + // .await; + // + // let tx_fee = calculate_tx_fee(&fees); + // + // short_sma_tx_fees.push((height, tx_fee)); + // } + // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + // + // let decider = SendOrWaitDecider::new( + // FeeAnalytics::new(local_client.clone()), + // services::state_committer::fee_optimization::Config { + // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + // short: short_sma, + // long: long_sma, + // }, + // fee_thresholds: Feethresholds { + // max_l2_blocks_behind: 43200 * 3, + // start_discount_percentage: 0.2, + // end_premium_percentage: 0.2, + // always_acceptable_fee: 1000000000000000u128, + // }, + // }, + // ); + // + // let mut decisions = vec![]; + // let mut long_sma_tx_fees = vec![]; + // + // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - long_sma..=height) + // .await; + // let tx_fee = calculate_tx_fee(&fees); + // long_sma_tx_fees.push((height, tx_fee)); + // + // if decider + // .should_send_blob_tx( + // 6, + // Context { + // at_l1_height: height, + // num_l2_blocks_behind: (height - starting_block_height) * 12, + // }, + // ) + // .await + // { + // let current_fees = fee_lookup.get(&height).unwrap(); + // let current_tx_fee = calculate_tx_fee(current_fees); + // decisions.push((height, current_tx_fee)); + // } + // } + // + // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + // save_tx_fees(&decisions, "decisions.csv"); + // } +} diff --git a/packages/services/tests/fee_analytics.rs b/packages/services/tests/fee_analytics.rs deleted file mode 100644 index a89f52ce..00000000 --- a/packages/services/tests/fee_analytics.rs +++ /dev/null @@ -1,147 +0,0 @@ -use std::path::PathBuf; - -use services::fee_analytics::{ - port::{ - l1::testing::{self}, - Fees, - }, - service::FeeAnalytics, -}; - -#[tokio::test] -async fn calculates_sma_correctly_for_last_1_block() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); - - // then - assert_eq!(sma.base_fee_per_gas, 5); - assert_eq!(sma.reward, 5); - assert_eq!(sma.base_fee_per_blob_gas, 5); -} - -#[tokio::test] -async fn calculates_sma_correctly_for_last_5_blocks() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); - - // then - let mean = (5 + 4 + 3 + 2 + 1) / 5; - assert_eq!(sma.base_fee_per_gas, mean); - assert_eq!(sma.reward, mean); - assert_eq!(sma.base_fee_per_blob_gas, mean); -} - -fn calculate_tx_fee(fees: &Fees) -> u128 { - 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 -} - -fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - let mut csv_writer = - csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)).unwrap(); - csv_writer - .write_record(["height", "tx_fee"].iter()) - .unwrap(); - for (height, fee) in tx_fees { - csv_writer - .write_record([height.to_string(), fee.to_string()]) - .unwrap(); - } - csv_writer.flush().unwrap(); -} - -// #[tokio::test] -// async fn something() { -// let client = make_pub_eth_client().await; -// use services::fee_analytics::port::l1::FeesProvider; -// -// let current_block_height = 21408300; -// let starting_block_height = current_block_height - 48 * 3600 / 12; -// let data = client -// .fees(starting_block_height..=current_block_height) -// .await -// .into_iter() -// .collect::>(); -// -// let fee_lookup = data -// .iter() -// .map(|b| (b.height, b.fees)) -// .collect::>(); -// -// let short_sma = 25u64; -// let long_sma = 900; -// -// let current_tx_fees = data -// .iter() -// .map(|b| (b.height, calculate_tx_fee(&b.fees))) -// .collect::>(); -// -// save_tx_fees(¤t_tx_fees, "current_fees.csv"); -// -// let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); -// let fee_analytics = FeeAnalytics::new(local_client.clone()); -// -// let mut short_sma_tx_fees = vec![]; -// for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - short_sma..=height) -// .await; -// -// let tx_fee = calculate_tx_fee(&fees); -// -// short_sma_tx_fees.push((height, tx_fee)); -// } -// save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); -// -// let decider = SendOrWaitDecider::new( -// FeeAnalytics::new(local_client.clone()), -// services::state_committer::fee_optimization::Config { -// sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { -// short: short_sma, -// long: long_sma, -// }, -// fee_thresholds: Feethresholds { -// max_l2_blocks_behind: 43200 * 3, -// start_discount_percentage: 0.2, -// end_premium_percentage: 0.2, -// always_acceptable_fee: 1000000000000000u128, -// }, -// }, -// ); -// -// let mut decisions = vec![]; -// let mut long_sma_tx_fees = vec![]; -// -// for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { -// let fees = fee_analytics -// .calculate_sma(height - long_sma..=height) -// .await; -// let tx_fee = calculate_tx_fee(&fees); -// long_sma_tx_fees.push((height, tx_fee)); -// -// if decider -// .should_send_blob_tx( -// 6, -// Context { -// at_l1_height: height, -// num_l2_blocks_behind: (height - starting_block_height) * 12, -// }, -// ) -// .await -// { -// let current_fees = fee_lookup.get(&height).unwrap(); -// let current_tx_fee = calculate_tx_fee(current_fees); -// decisions.push((height, current_tx_fee)); -// } -// } -// -// save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); -// save_tx_fees(&decisions, "decisions.csv"); -// } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 5f655702..8f207a21 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,12 +1,8 @@ use services::{ - fee_analytics::{ - port::{ - l1::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - Fees, - }, - service::FeeAnalytics, + state_committer::{ + port::l1::{testing::ApiMockWFees, Fees}, + service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, }, - state_committer::service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; @@ -34,7 +30,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -44,7 +40,6 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -78,7 +73,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -88,7 +83,6 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout @@ -114,7 +108,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -124,7 +118,6 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time less than the timeout @@ -160,7 +153,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -170,7 +163,6 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // when @@ -207,7 +199,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(1); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -217,7 +209,6 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time to exceed the timeout since last finalized fragment @@ -254,7 +245,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> .expect_current_height() .returning(|| Box::pin(async { Ok(1) })); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -264,7 +255,6 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Advance time beyond the timeout from startup @@ -314,7 +304,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit), fuel_mock, setup.db(), StateCommitterConfig { @@ -325,7 +315,6 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Submit the initial fragments @@ -399,9 +388,6 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -431,7 +417,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -442,7 +428,6 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ..Default::default() }, setup.test_clock(), - fee_analytics, ); // When @@ -510,9 +495,6 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -533,7 +515,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -544,7 +526,6 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ..Default::default() }, setup.test_clock(), - fee_analytics, ); // when @@ -612,9 +593,6 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -644,7 +622,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -655,7 +633,6 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ..Default::default() }, setup.test_clock(), - fee_analytics, ); // when @@ -722,9 +699,6 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ), ]; - let fees_provider = PreconfiguredFeesProvider::new(fee_sequence); - let fee_analytics = FeeAnalytics::new(fees_provider); - let fee_algo_config = FeeAlgoConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { @@ -754,7 +728,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); let mut state_committer = StateCommitter::new( - l1_mock_submit, + ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, setup.db(), StateCommitterConfig { @@ -765,7 +739,6 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ..Default::default() }, setup.test_clock(), - fee_analytics, ); // when diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index b62108bf..20ac4b38 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,13 +3,7 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ - fee_analytics::{ - port::{ - l1::testing::ConstantFeesProvider, - Fees, - }, - service::FeeAnalytics, - }, + state_committer::port::l1::testing::ApiMockWFees, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, @@ -452,7 +446,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), mocks::fuel::latest_height_is(0), setup.db(), StateCommitterConfig { @@ -460,7 +454,6 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx @@ -556,7 +549,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), mocks::fuel::latest_height_is(0), setup.db(), crate::StateCommitterConfig { @@ -564,7 +557,6 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index ac357e66..c248ae64 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -1,5 +1,6 @@ #![deny(unused_crate_dependencies)] +use std::sync::Arc; use std::{ops::RangeInclusive, time::Duration}; use clock::TestClock; @@ -8,9 +9,7 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_analytics::port::l1::testing::ConstantFeesProvider; -use services::fee_analytics::port::Fees; -use services::fee_analytics::service::FeeAnalytics; +use services::state_committer::port::l1::testing::ApiMockWFees; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, }; @@ -547,7 +546,7 @@ impl Setup { .return_once(move || Box::pin(async { Ok(0) })); StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), mocks::fuel::latest_height_is(0), self.db(), services::StateCommitterConfig { @@ -558,7 +557,6 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ) .run() .await @@ -586,7 +584,7 @@ impl Setup { let fuel_mock = mocks::fuel::latest_height_is(height); let mut committer = StateCommitter::new( - l1_mock, + ApiMockWFees::new(l1_mock), fuel_mock, self.db(), services::StateCommitterConfig { @@ -597,7 +595,6 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), - FeeAnalytics::new(ConstantFeesProvider::new(Fees::default())), ); committer.run().await.unwrap(); From 4c42cb7453870031a9b801b246966a2950ddfae2 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 20:03:29 +0100 Subject: [PATCH 28/47] add check whether provider reported all requested fees --- .../src/state_committer/fee_analytics.rs | 73 ++++++++++++++----- packages/services/tests/state_committer.rs | 28 +++---- 2 files changed, 68 insertions(+), 33 deletions(-) diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 29633b26..2761ead3 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -54,6 +54,12 @@ impl SequentialBlockFees { pub fn len(&self) -> usize { self.fees.len() } + + pub fn height_range(&self) -> RangeInclusive { + let start = self.fees.first().expect("not empty").height; + let end = self.fees.last().expect("not empty").height; + start..=end + } } #[derive(Debug)] @@ -199,10 +205,18 @@ impl

FeeAnalytics

{ impl FeeAnalytics

{ // TODO: segfault fail or signal if missing blocks/holes present - // TODO: segfault cache fees/save to db - // TODO: segfault job to update fees in the background + // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - self.fees_provider.fees(block_range).await.map(Self::mean) + let fees = self.fees_provider.fees(block_range.clone()).await?; + + let received_height_range = fees.height_range(); + if received_height_range != block_range { + return Err(crate::Error::from(format!( + "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" + ))); + } + + Ok(Self::mean(fees)) } fn mean(fees: SequentialBlockFees) -> Fees { @@ -393,25 +407,46 @@ mod tests { assert_eq!(sma.base_fee_per_blob_gas, mean); } - fn calculate_tx_fee(fees: &Fees) -> u128 { - 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 - } + #[tokio::test] + async fn errors_out_if_returned_fees_are_not_complete() { + // given + let mut fees = testing::incrementing_fees(5); + fees.remove(&4); + let fees_provider = testing::PreconfiguredFeesProvider::new(fees); + let fee_analytics = FeeAnalytics::new(fees_provider); - fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - let mut csv_writer = - csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) - .unwrap(); - csv_writer - .write_record(["height", "tx_fee"].iter()) - .unwrap(); - for (height, fee) in tx_fees { - csv_writer - .write_record([height.to_string(), fee.to_string()]) - .unwrap(); - } - csv_writer.flush().unwrap(); + // when + let err = fee_analytics + .calculate_sma(0..=4) + .await + .expect_err("should have failed because returned fees are not complete"); + + // then + assert_eq!( + err.to_string(), + "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" + ); } + // fn calculate_tx_fee(fees: &Fees) -> u128 { + // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + // } + // + // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + // let mut csv_writer = + // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + // .unwrap(); + // csv_writer + // .write_record(["height", "tx_fee"].iter()) + // .unwrap(); + // for (height, fee) in tx_fees { + // csv_writer + // .write_record([height.to_string(), fee.to_string()]) + // .unwrap(); + // } + // csv_writer.flush().unwrap(); + // } + // #[tokio::test] // async fn something() { // let client = make_pub_eth_client().await; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 8f207a21..0c548651 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -339,7 +339,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fee_sequence = vec![ ( - 1, + 0, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -347,7 +347,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 2, + 1, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -355,7 +355,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 3, + 2, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -363,7 +363,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 4, + 3, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -371,7 +371,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 5, + 4, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -379,7 +379,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { }, ), ( - 6, + 5, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -413,7 +413,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { )]); l1_mock_submit .expect_current_height() - .returning(|| Box::pin(async { Ok(6) })); + .returning(|| Box::pin(async { Ok(5) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( @@ -446,7 +446,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( // Define fee sequence: last 2 blocks have higher fees than the long-term average let fee_sequence = vec![ ( - 1, + 0, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -454,7 +454,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 2, + 1, Fees { base_fee_per_gas: 3000, reward: 3000, @@ -462,7 +462,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 3, + 2, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -470,7 +470,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 4, + 3, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -478,7 +478,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 5, + 4, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -486,7 +486,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( }, ), ( - 6, + 5, Fees { base_fee_per_gas: 5000, reward: 5000, @@ -511,7 +511,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let mut l1_mock = test_helpers::mocks::l1::expects_state_submissions([]); l1_mock .expect_current_height() - .returning(|| Box::pin(async { Ok(6) })); + .returning(|| Box::pin(async { Ok(5) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( From 61c6fd35af29c39f172af9f5b7abbbae3afbdfdd Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 20:55:34 +0100 Subject: [PATCH 29/47] add caching and tests --- .../src/state_committer/fee_analytics.rs | 259 +++++++++++++++++- 1 file changed, 256 insertions(+), 3 deletions(-) diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 2761ead3..e29e4242 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -1,6 +1,7 @@ -use std::ops::RangeInclusive; +use std::{collections::BTreeMap, ops::RangeInclusive}; use itertools::Itertools; +use tokio::sync::RwLock; use crate::state_committer::port::l1::Api; @@ -17,7 +18,7 @@ pub struct BlockFees { pub fees: Fees, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct SequentialBlockFees { fees: Vec, } @@ -30,6 +31,90 @@ pub trait FeesProvider { async fn current_block_height(&self) -> crate::Result; } +#[derive(Debug)] +pub struct CachingFeesProvider

{ + fees_provider: P, + cache: RwLock>, + cache_limit: usize, +} + +impl

CachingFeesProvider

{ + pub fn new(fees_provider: P, cache_limit: usize) -> Self { + Self { + fees_provider, + cache: RwLock::new(BTreeMap::new()), + cache_limit, + } + } +} + +impl FeesProvider for CachingFeesProvider

{ + async fn fees(&self, height_range: RangeInclusive) -> crate::Result { + self.get_fees(height_range).await + } + + async fn current_block_height(&self) -> crate::Result { + self.fees_provider.current_block_height().await + } +} + +impl CachingFeesProvider

{ + pub async fn get_fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let mut missing_heights = vec![]; + + // Mind the scope to release the read lock + { + let cache = self.cache.read().await; + for height in height_range.clone() { + if !cache.contains_key(&height) { + missing_heights.push(height); + } + } + } + + if !missing_heights.is_empty() { + let fetched_fees = self + .fees_provider + .fees( + *missing_heights.first().expect("not empty") + ..=*missing_heights.last().expect("not empty"), + ) + .await?; + + let mut cache = self.cache.write().await; + for block_fee in fetched_fees { + cache.insert(block_fee.height, block_fee.fees); + } + } + + let fees: Vec<_> = { + let cache = self.cache.read().await; + height_range + .filter_map(|h| { + cache.get(&h).map(|f| BlockFees { + height: h, + fees: *f, + }) + }) + .collect() + }; + + self.shrink_cache().await; + + SequentialBlockFees::try_from(fees).map_err(|e| crate::Error::Other(e.to_string())) + } + + async fn shrink_cache(&self) { + let mut cache = self.cache.write().await; + while cache.len() > self.cache_limit { + cache.pop_first(); + } + } +} + impl FeesProvider for T { async fn fees(&self, height_range: RangeInclusive) -> crate::Result { Api::fees(self, height_range).await @@ -194,6 +279,7 @@ pub mod testing { } } +#[derive(Debug, Clone)] pub struct FeeAnalytics

{ fees_provider: P, } @@ -204,7 +290,6 @@ impl

FeeAnalytics

{ } impl FeeAnalytics

{ - // TODO: segfault fail or signal if missing blocks/holes present // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { let fees = self.fees_provider.fees(block_range.clone()).await?; @@ -243,6 +328,7 @@ impl FeeAnalytics

{ #[cfg(test)] mod tests { use itertools::Itertools; + use mockall::{predicate::eq, Sequence}; use super::*; @@ -428,6 +514,173 @@ mod tests { ); } + #[tokio::test] + async fn caching_provider_avoids_duplicate_requests() { + // given + let mut mock_provider = MockFeesProvider::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }); + + let provider = CachingFeesProvider::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // mock validates no extra calls made + } + + #[tokio::test] + async fn caching_provider_fetches_only_missing_blocks() { + // Given: A mock FeesProvider + let mut mock_provider = MockFeesProvider::new(); + + // Expectation: The provider will fetch blocks 3..=5, since 0..=2 are cached + let mut sequence = Sequence::new(); + mock_provider + .expect_fees() + .with(eq(0..=2)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + mock_provider + .expect_fees() + .with(eq(3..=5)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + let provider = CachingFeesProvider::new(mock_provider, 5); + let _ = provider.get_fees(0..=2).await.unwrap(); + + // when + let _ = provider.get_fees(2..=5).await.unwrap(); + + // then + // not called for the overlapping area + } + + fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { + SequentialBlockFees::try_from( + height_range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap() + } + + #[tokio::test] + async fn caching_provider_evicts_oldest_blocks() { + // given + let mut mock_provider = MockFeesProvider::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .times(2) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + mock_provider + .expect_fees() + .with(eq(5..=9)) + .times(1) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + let provider = CachingFeesProvider::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + let _ = provider.get_fees(5..=9).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // will refetch 0..=4 due to eviction + } + + #[tokio::test] + async fn caching_provider_handles_request_larger_than_cache() { + use mockall::predicate::*; + + // given + let mut mock_provider = MockFeesProvider::new(); + + let cache_limit = 5; + + mock_provider + .expect_fees() + .with(eq(0..=9)) + .times(1) + .returning(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); + + let provider = CachingFeesProvider::new(mock_provider, cache_limit); + + // when + let result = provider.get_fees(0..=9).await.unwrap(); + + assert_eq!(result, generate_sequential_fees(0..=9)); + } + // fn calculate_tx_fee(fees: &Fees) -> u128 { // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 // } From 8b6993b920b2f6dc385b81f27e3cdb21d8031fb8 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Tue, 17 Dec 2024 22:36:00 +0100 Subject: [PATCH 30/47] changed the last n block calculation, need to fix tests --- packages/services/src/state_committer.rs | 20 +++++++++++++++---- .../services/src/state_committer/fee_algo.rs | 11 ++++++---- .../src/state_committer/fee_analytics.rs | 6 ++++++ packages/services/tests/state_committer.rs | 19 ++++++++++-------- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 5d53539f..7cd45e14 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -8,7 +8,7 @@ pub mod service { }; use crate::{ - state_committer::fee_algo::Context, + state_committer::{fee_algo::Context, fee_analytics::CachingFeesProvider}, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Result, Runner, }; @@ -78,7 +78,7 @@ pub mod service { config: Config, clock: Clock, startup_time: DateTime, - decider: SendOrWaitDecider, + decider: SendOrWaitDecider>, } impl StateCommitter @@ -96,14 +96,20 @@ pub mod service { ) -> Self { let startup_time = clock.now(); let price_algo = config.fee_algo; + + // TODO: segfault, configure this cache + let decider = SendOrWaitDecider::new( + CachingFeesProvider::new(l1_adapter.clone(), 24 * 3600 / 12), + price_algo, + ); Self { - l1_adapter: l1_adapter.clone(), + l1_adapter, fuel_api, storage, config, clock, startup_time, - decider: SendOrWaitDecider::new(l1_adapter, price_algo), + decider, } } } @@ -141,6 +147,10 @@ pub mod service { .oldest_block_in_bundle; let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); + eprintln!( + "deciding whether to send tx with {} fragments", + fragments.len() + ); self.decider .should_send_blob_tx( @@ -159,9 +169,11 @@ pub mod service { previous_tx: Option, ) -> Result<()> { if !self.should_send_tx(&fragments).await? { + eprintln!("decided against sending fragments"); info!("decided against sending fragments"); return Ok(()); } + eprintln!("decided to send fragments"); info!("about to send at most {} fragments", fragments.len()); diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index a247e670..56c92806 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -36,17 +36,21 @@ impl SendOrWaitDecider

{ ) -> crate::Result { // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller // wants to send more than 6 blobs - let last_n_blocks = |n: u64| context.at_l1_height.saturating_sub(n)..=context.at_l1_height; + let last_n_blocks = |n: u64| { + context.at_l1_height.saturating_sub(n.saturating_sub(1))..=context.at_l1_height + }; let short_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await?; + eprintln!("short term sma: {:?}", short_term_sma); let long_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; + eprintln!("long term sma: {:?}", long_term_sma); let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); @@ -479,7 +483,6 @@ mod tests { "Later: after max wait, send regardless" )] #[test_case( - // Partway: at 80 blocks behind, tolerance might have increased enough to accept Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, @@ -492,7 +495,7 @@ mod tests { always_acceptable_fee: 0, }, }, - 65, + 80, true; "Mid-wait: increased tolerance allows acceptance" )] @@ -507,7 +510,7 @@ mod tests { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, end_premium_percentage: 0.20, - always_acceptable_fee: 1_781_000_000_000 + always_acceptable_fee: 2_700_000_000_000 }, }, 0, diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index e29e4242..36b0f14a 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -292,15 +292,21 @@ impl

FeeAnalytics

{ impl FeeAnalytics

{ // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + eprintln!("asking for fees"); let fees = self.fees_provider.fees(block_range.clone()).await?; + eprintln!("fees received"); + eprintln!("checking if fees are complete"); let received_height_range = fees.height_range(); + eprintln!("received height range: {:?}", received_height_range); if received_height_range != block_range { + eprintln!("not equeal {received_height_range:?} != {block_range:?}",); return Err(crate::Error::from(format!( "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" ))); } + eprintln!("calculating mean"); Ok(Self::mean(fees)) } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 0c548651..e407783e 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -544,7 +544,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { // Define fee sequence with high fees to ensure that without the behind condition, it wouldn't send let fee_sequence = vec![ ( - 1, + 0, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -552,7 +552,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 2, + 1, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -560,7 +560,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 3, + 2, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -568,7 +568,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 4, + 3, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -576,7 +576,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 5, + 4, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -584,7 +584,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { }, ), ( - 6, + 5, Fees { base_fee_per_gas: 7000, reward: 7000, @@ -594,7 +594,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ]; let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), start_discount_percentage: 0.0, @@ -700,7 +700,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ]; let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20, @@ -727,6 +727,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); + eprintln!("about to contrust the committer"); let mut state_committer = StateCommitter::new( ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, @@ -740,9 +741,11 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran }, setup.test_clock(), ); + eprintln!("constructed the committer"); // when state_committer.run().await?; + eprintln!("ran the committer"); // then // Mocks validate that the fragments have been sent due to increased tolerance from nearing max blocks behind From 94c89797b74ef8072ffe547e2f88bdc581201325 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 08:48:18 +0100 Subject: [PATCH 31/47] fixed tests --- packages/services/src/state_committer.rs | 10 +++++----- .../services/src/state_committer/fee_analytics.rs | 15 +++++++++------ packages/services/tests/state_committer.rs | 4 ++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 7cd45e14..eef5b386 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -389,9 +389,9 @@ pub mod port { use super::{Api, Fees, MockApi, SequentialBlockFees}; #[derive(Clone)] - pub struct ApiMockWFees { + pub struct ApiMockWFees

{ pub api: Arc, - fee_provider: Fees, + fee_provider: P, } impl ApiMockWFees { @@ -403,7 +403,7 @@ pub mod port { } } - impl ApiMockWFees { + impl

ApiMockWFees

{ pub fn w_preconfigured_fees( self, fees: impl IntoIterator, @@ -415,9 +415,9 @@ pub mod port { } } - impl Api for ApiMockWFees + impl

Api for ApiMockWFees

where - T: FeesProvider + Send + Sync, + P: FeesProvider + Send + Sync, { async fn fees( &self, diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 36b0f14a..6f6efdfd 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -207,14 +207,17 @@ pub mod testing { impl FeesProvider for ConstantFeesProvider { async fn fees( &self, - _height_range: RangeInclusive, + height_range: RangeInclusive, ) -> crate::Result { - let fees = BlockFees { - height: self.current_block_height().await?, - fees: self.fees, - }; + let fees = height_range + .into_iter() + .map(|height| BlockFees { + height, + fees: self.fees, + }) + .collect_vec(); - Ok(vec![fees].try_into().unwrap()) + Ok(fees.try_into().unwrap()) } async fn current_block_height(&self) -> crate::Result { diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index e407783e..63d307dc 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -594,7 +594,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ]; let fee_algo_config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 5 }, + sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), start_discount_percentage: 0.0, @@ -618,7 +618,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { )]); l1_mock_submit .expect_current_height() - .returning(|| Box::pin(async { Ok(6) })); + .returning(|| Box::pin(async { Ok(5) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 let mut state_committer = StateCommitter::new( From 004be103d8adae60b3a12bd2b3c303f96632e3de Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 09:39:04 +0100 Subject: [PATCH 32/47] validate percentages --- committer/src/main.rs | 2 +- committer/src/setup.rs | 18 ++- packages/services/src/state_committer.rs | 53 ++++++-- .../services/src/state_committer/fee_algo.rs | 113 ++++++++---------- .../src/state_committer/fee_analytics.rs | 1 - packages/services/tests/state_committer.rs | 13 +- packages/test-helpers/src/lib.rs | 1 - 7 files changed, 114 insertions(+), 87 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 0473d0c7..5810df75 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -78,7 +78,7 @@ async fn main() -> Result<()> { storage.clone(), cancel_token.clone(), &config, - ); + )?; let state_importer_handle = setup::block_importer(fuel_adapter, storage.clone(), cancel_token.clone(), &config); diff --git a/committer/src/setup.rs b/committer/src/setup.rs index b09449c6..8f9178dc 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -120,7 +120,7 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, -) -> tokio::task::JoinHandle<()> { +) -> Result> { let state_committer = services::StateCommitter::new( l1, fuel, @@ -137,8 +137,16 @@ pub fn state_committer( }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config.app.fee_algo.start_discount_percentage, - end_premium_percentage: config.app.fee_algo.end_premium_percentage, + start_discount_percentage: config + .app + .fee_algo + .start_discount_percentage + .try_into()?, + end_premium_percentage: config + .app + .fee_algo + .end_premium_percentage + .try_into()?, always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, }, }, @@ -146,12 +154,12 @@ pub fn state_committer( SystemClock, ); - schedule_polling( + Ok(schedule_polling( config.app.tx_finalization_check_interval, state_committer, "State Committer", cancel_token, - ) + )) } pub fn block_importer( diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index eef5b386..f4565d3e 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -10,7 +10,7 @@ pub mod service { use crate::{ state_committer::{fee_algo::Context, fee_analytics::CachingFeesProvider}, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Result, Runner, + Error, Result, Runner, }; use itertools::Itertools; use tracing::info; @@ -23,15 +23,56 @@ pub mod service { pub long: u64, } - // TODO: segfault validate start discount is less than end premium and both are positive + #[derive(Default, Copy, Clone, Debug, PartialEq)] + pub struct Percentage(f64); + + impl TryFrom for Percentage { + type Error = crate::Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } + } + + impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } + } + + impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } + } + #[derive(Debug, Clone, Copy)] pub struct FeeThresholds { pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: f64, - pub end_premium_percentage: f64, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, pub always_acceptable_fee: u128, } + #[cfg(feature = "test-helpers")] + impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } + } + #[derive(Debug, Clone, Copy)] pub struct FeeAlgoConfig { pub sma_periods: SmaPeriods, @@ -61,9 +102,7 @@ pub mod service { sma_periods: SmaPeriods { short: 1, long: 2 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0., - end_premium_percentage: 0., - always_acceptable_fee: u128::MAX, + ..FeeThresholds::default() }, }, } diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 56c92806..c099a51d 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -1,5 +1,7 @@ use std::cmp::min; +use crate::state_committer::service::Percentage; + use super::{ fee_analytics::{FeeAnalytics, FeesProvider}, port::l1::Fees, @@ -105,8 +107,6 @@ impl SendOrWaitDecider

{ fee: u128, context: Context, ) -> u128 { - const PPM: u128 = 1_000_000; // 100% in PPM - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); let blocks_behind = u128::from(context.num_l2_blocks_behind); @@ -117,11 +117,11 @@ impl SendOrWaitDecider

{ max_blocks_behind ); - let start_discount_ppm = percentage_to_ppm(fee_thresholds.start_discount_percentage); - let end_premium_ppm = percentage_to_ppm(fee_thresholds.end_premium_percentage); + let start_discount_ppm = fee_thresholds.start_discount_percentage.ppm(); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = PPM.saturating_sub(start_discount_ppm); + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); // 2. How late are we: eg. late enough to add 25% to our base multiplier let premium_increment = Self::calculate_premium_increment( @@ -134,11 +134,12 @@ impl SendOrWaitDecider

{ // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% let multiplier_ppm = min( base_multiplier.saturating_add(premium_increment), - PPM + end_premium_ppm, + Percentage::PPM + end_premium_ppm, ); // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm).saturating_div(PPM) + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) } fn calculate_premium_increment( @@ -147,19 +148,19 @@ impl SendOrWaitDecider

{ blocks_behind: u128, max_blocks_behind: u128, ) -> u128 { - const PPM: u128 = 1_000_000; // 100% in PPM - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); let proportion = if max_blocks_behind == 0 { 0 } else { blocks_behind - .saturating_mul(PPM) + .saturating_mul(Percentage::PPM) .saturating_div(max_blocks_behind) }; - total_ppm.saturating_mul(proportion).saturating_div(PPM) + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) } // TODO: Segfault maybe dont leak so much eth abstractions @@ -174,16 +175,15 @@ impl SendOrWaitDecider

{ } } -fn percentage_to_ppm(percentage: f64) -> u128 { - (percentage * 1_000_000.0) as u128 -} - #[cfg(test)] mod tests { use super::*; - use crate::state_committer::{ - fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - service::{FeeThresholds, SmaPeriods}, + use crate::{ + state_committer::{ + fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, + service::{FeeThresholds, Percentage, SmaPeriods}, + }, + types::NonNegative, }; use test_case::test_case; @@ -211,9 +211,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }, 0, // not behind at all @@ -228,9 +227,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }, 0, @@ -246,8 +244,7 @@ mod tests { fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, + ..Default::default() } }, 0, @@ -262,9 +259,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -279,9 +275,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -296,9 +291,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -313,9 +307,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -330,9 +323,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -347,9 +339,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -365,9 +356,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -382,9 +372,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -400,9 +389,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -418,9 +406,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -436,9 +423,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() } }, 0, @@ -455,8 +441,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), always_acceptable_fee: 0, }, }, @@ -473,8 +459,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, } }, @@ -490,8 +476,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, }, }, @@ -508,8 +494,8 @@ mod tests { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 2_700_000_000_000 }, }, @@ -553,9 +539,8 @@ mod tests { // Test Case 1: No blocks behind, no discount or premium FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, 1000, Context { @@ -568,8 +553,8 @@ mod tests { #[test_case( FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.25, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), always_acceptable_fee: 0, }, 2000, @@ -583,9 +568,9 @@ mod tests { #[test_case( FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25, - end_premium_percentage: 0.0, + start_discount_percentage: 0.25.try_into().unwrap(), always_acceptable_fee: 0, + ..Default::default() }, 800, Context { @@ -598,9 +583,9 @@ mod tests { #[test_case( FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.30, + end_premium_percentage: 0.30.try_into().unwrap(), always_acceptable_fee: 0, + ..Default::default() }, 1000, Context { @@ -614,8 +599,8 @@ mod tests { // Test Case 8: High fee with premium FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10, // 100,000 PPM - end_premium_percentage: 0.20, // 200,000 PPM + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, }, 10_000, diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 6f6efdfd..2c611226 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -293,7 +293,6 @@ impl

FeeAnalytics

{ } impl FeeAnalytics

{ - // TODO: segfault cache fees pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { eprintln!("asking for fees"); let fees = self.fees_provider.fees(block_range.clone()).await?; diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 63d307dc..9b38c639 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -392,9 +392,8 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }; @@ -499,9 +498,8 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }; @@ -597,9 +595,8 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), - start_discount_percentage: 0.0, - end_premium_percentage: 0.0, always_acceptable_fee: 0, + ..Default::default() }, }; @@ -703,8 +700,8 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20, - end_premium_percentage: 0.20, + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), always_acceptable_fee: 0, }, }; diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index c248ae64..3f8c7cdc 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -1,6 +1,5 @@ #![deny(unused_crate_dependencies)] -use std::sync::Arc; use std::{ops::RangeInclusive, time::Duration}; use clock::TestClock; From 963422e2a1b0a943737f1348c83bcf44e081b0c1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 09:47:39 +0100 Subject: [PATCH 33/47] add capping for discount percentage at 100% --- .../services/src/state_committer/fee_algo.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index c099a51d..11565c98 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -117,7 +117,10 @@ impl SendOrWaitDecider

{ max_blocks_behind ); - let start_discount_ppm = fee_thresholds.start_discount_percentage.ppm(); + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% @@ -611,6 +614,21 @@ mod tests { 11970; "High fee with premium" )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 1.50.try_into().unwrap(), // 150% + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 1, + at_l1_height: 0, + }, + 12; + "Discount exceeds 100%, should be capped to 100%" +)] fn test_calculate_max_upper_fee( fee_thresholds: FeeThresholds, fee: u128, From 80b9cef3afad8f2d2fc849b84ef6543d9e4c6b9f Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 09:57:01 +0100 Subject: [PATCH 34/47] if too far back, makes decision even with a faulty fee provider --- .../services/src/state_committer/fee_algo.rs | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 11565c98..58c5e267 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -29,13 +29,16 @@ pub struct Context { } impl SendOrWaitDecider

{ - // TODO: segfault test that too far behind should work even if we cannot fetch prices due to holes - // (once that is implemented) + // TODO: segfault logging pub async fn should_send_blob_tx( &self, num_blobs: u32, context: Context, ) -> crate::Result { + if self.too_far_behind(context) { + return Ok(true); + } + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller // wants to send more than 6 blobs let last_n_blocks = |n: u64| { @@ -56,16 +59,7 @@ impl SendOrWaitDecider

{ let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); - let fee_always_acceptable = - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee; - eprintln!("fee always acceptable: {}", fee_always_acceptable); - - let too_far_behind = - context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get(); - - eprintln!("too far behind: {}", too_far_behind); - - if fee_always_acceptable || too_far_behind { + if self.fee_always_acceptable(short_term_tx_fee) { return Ok(true); } @@ -102,6 +96,14 @@ impl SendOrWaitDecider

{ Ok(short_term_tx_fee < max_upper_tx_fee) } + fn too_far_behind(&self, context: Context) -> bool { + context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } + fn calculate_max_upper_fee( fee_thresholds: &FeeThresholds, fee: u128, @@ -647,4 +649,37 @@ mod tests { expected_max_upper_fee, max_upper_fee ); } + #[tokio::test] + async fn test_send_when_too_far_behind_and_fee_provider_fails() { + // given + let config = FeeAlgoConfig { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 10.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }; + + // having no fees will make the validation in fee analytics fail + let fee_provider = PreconfiguredFeesProvider::new(vec![]); + let sut = SendOrWaitDecider::new(fee_provider, config); + + let context = Context { + num_l2_blocks_behind: 20, + at_l1_height: 100, + }; + + // when + let should_send = sut + .should_send_blob_tx(1, context) + .await + .expect("Should send despite fee provider failure"); + + // then + assert!( + should_send, + "Should send because too far behind, regardless of fee provider status" + ); + } } From 7fa623dae5fa430fa33dc572c44e6b4a9b08ba26 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 10:04:17 +0100 Subject: [PATCH 35/47] add logging, remove print statements --- packages/services/src/state_committer.rs | 9 +--- .../services/src/state_committer/fee_algo.rs | 41 +++++++------------ .../src/state_committer/fee_analytics.rs | 6 --- packages/services/tests/state_committer.rs | 10 ++--- 4 files changed, 19 insertions(+), 47 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index f4565d3e..22c6b436 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -186,10 +186,6 @@ pub mod service { .oldest_block_in_bundle; let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); - eprintln!( - "deciding whether to send tx with {} fragments", - fragments.len() - ); self.decider .should_send_blob_tx( @@ -208,12 +204,9 @@ pub mod service { previous_tx: Option, ) -> Result<()> { if !self.should_send_tx(&fragments).await? { - eprintln!("decided against sending fragments"); - info!("decided against sending fragments"); + info!("decided against sending fragments due to high fees"); return Ok(()); } - eprintln!("decided to send fragments"); - info!("about to send at most {} fragments", fragments.len()); let data = fragments.clone().map(|f| f.fragment); diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs index 58c5e267..5e6cc4fa 100644 --- a/packages/services/src/state_committer/fee_algo.rs +++ b/packages/services/src/state_committer/fee_algo.rs @@ -1,5 +1,7 @@ use std::cmp::min; +use tracing::info; + use crate::state_committer::service::Percentage; use super::{ @@ -29,13 +31,13 @@ pub struct Context { } impl SendOrWaitDecider

{ - // TODO: segfault logging pub async fn should_send_blob_tx( &self, num_blobs: u32, context: Context, ) -> crate::Result { if self.too_far_behind(context) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", context.num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); return Ok(true); } @@ -49,17 +51,16 @@ impl SendOrWaitDecider

{ .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.short)) .await?; - eprintln!("short term sma: {:?}", short_term_sma); let long_term_sma = self .fee_analytics .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; - eprintln!("long term sma: {:?}", long_term_sma); let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); return Ok(true); } @@ -67,33 +68,21 @@ impl SendOrWaitDecider

{ let max_upper_tx_fee = Self::calculate_max_upper_fee(&self.config.fee_thresholds, long_term_tx_fee, context); - let long_vs_max_delta_perc = - ((max_upper_tx_fee as f64 - long_term_tx_fee as f64) / long_term_tx_fee as f64 * 100.) - .abs(); - - let short_vs_max_delta_perc = ((max_upper_tx_fee as f64 - short_term_tx_fee as f64) - / short_term_tx_fee as f64 - * 100.) - .abs(); + let should_send = short_term_tx_fee < max_upper_tx_fee; - if long_term_tx_fee <= max_upper_tx_fee { - eprintln!("The max upper fee({max_upper_tx_fee}) is above the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); } else { - eprintln!("The max upper fee({max_upper_tx_fee}) is below the long-term fee({long_term_tx_fee}) by {long_vs_max_delta_perc}%",); + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); } - if short_term_tx_fee <= max_upper_tx_fee { - eprintln!("The short term fee({short_term_tx_fee}) is below the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); - } else { - eprintln!("The short term fee({short_term_tx_fee}) is above the max upper fee({max_upper_tx_fee}) by {short_vs_max_delta_perc}%",); - } - - eprintln!( - "Short-term fee: {}, Long-term fee: {}, Max upper fee: {}", - short_term_tx_fee, long_term_tx_fee, max_upper_tx_fee - ); - - Ok(short_term_tx_fee < max_upper_tx_fee) + Ok(should_send) } fn too_far_behind(&self, context: Context) -> bool { diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index 2c611226..e7db5702 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -294,21 +294,15 @@ impl

FeeAnalytics

{ impl FeeAnalytics

{ pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - eprintln!("asking for fees"); let fees = self.fees_provider.fees(block_range.clone()).await?; - eprintln!("fees received"); - eprintln!("checking if fees are complete"); let received_height_range = fees.height_range(); - eprintln!("received height range: {:?}", received_height_range); if received_height_range != block_range { - eprintln!("not equeal {received_height_range:?} != {block_range:?}",); return Err(crate::Error::from(format!( "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" ))); } - eprintln!("calculating mean"); Ok(Self::mean(fees)) } diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 9b38c639..4ada71e7 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -297,10 +297,9 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ), ]); - l1_mock_submit.expect_current_height().returning(|| { - eprintln!("I was called"); - Box::pin(async { Ok(0) }) - }); + l1_mock_submit + .expect_current_height() + .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( @@ -724,7 +723,6 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); - eprintln!("about to contrust the committer"); let mut state_committer = StateCommitter::new( ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), fuel_mock, @@ -738,11 +736,9 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran }, setup.test_clock(), ); - eprintln!("constructed the committer"); // when state_committer.run().await?; - eprintln!("ran the committer"); // then // Mocks validate that the fragments have been sent due to increased tolerance from nearing max blocks behind From 3ab2e934213354b85af909946d26d22eba97c238 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 10:10:58 +0100 Subject: [PATCH 36/47] added fees at height --- .../src/state_committer/fee_analytics.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs index e7db5702..548223cb 100644 --- a/packages/services/src/state_committer/fee_analytics.rs +++ b/packages/services/src/state_committer/fee_analytics.rs @@ -306,6 +306,18 @@ impl FeeAnalytics

{ Ok(Self::mean(fees)) } + pub async fn fees_at_height(&self, height: u64) -> crate::Result { + let fee = self + .fees_provider + .fees(height..=height) + .await? + .into_iter() + .next() + .expect("sequential fees guaranteed not empty"); + + Ok(fee.fees) + } + fn mean(fees: SequentialBlockFees) -> Fees { let count = fees.len() as u128; @@ -331,6 +343,7 @@ impl FeeAnalytics

{ mod tests { use itertools::Itertools; use mockall::{predicate::eq, Sequence}; + use testing::{incrementing_fees, PreconfiguredFeesProvider}; use super::*; @@ -683,6 +696,29 @@ mod tests { assert_eq!(result, generate_sequential_fees(0..=9)); } + #[tokio::test] + async fn price_at_height_returns_correct_fee() { + // given + let fees_map = incrementing_fees(5); + let fees_provider = PreconfiguredFeesProvider::new(fees_map.clone()); + let fee_analytics = FeeAnalytics::new(fees_provider); + let height = 2; + + // when + let fee = fee_analytics.fees_at_height(height).await.unwrap(); + + // then + let expected_fee = Fees { + base_fee_per_gas: 3, + reward: 3, + base_fee_per_blob_gas: 3, + }; + assert_eq!( + fee, expected_fee, + "Fee at height {height} should be {expected_fee:?}" + ); + } + // fn calculate_tx_fee(fees: &Fees) -> u128 { // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 // } From 62f6f3844829f2cece33c7f608644694df7653a0 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:33:47 +0100 Subject: [PATCH 37/47] moving fee tracking into its own service due to a need for regular prices for metrics --- committer/src/setup.rs | 47 +- packages/adapters/eth/src/fee_conversion.rs | 4 +- packages/adapters/eth/src/lib.rs | 29 +- packages/services/src/fee_tracker.rs | 4 + .../services/src/fee_tracker/fee_analytics.rs | 386 ++++++++ packages/services/src/fee_tracker/port.rs | 463 ++++++++++ packages/services/src/fee_tracker/service.rs | 370 ++++++++ packages/services/src/fee_tracker/testing.rs | 2 + packages/services/src/lib.rs | 1 + packages/services/src/state_committer.rs | 189 +--- .../services/src/state_committer/fee_algo.rs | 674 -------------- .../src/state_committer/fee_analytics.rs | 829 ------------------ packages/services/tests/fee_tracker.rs | 379 ++++++++ packages/services/tests/state_committer.rs | 52 +- packages/services/tests/state_listener.rs | 12 +- packages/test-helpers/src/lib.rs | 15 +- 16 files changed, 1719 insertions(+), 1737 deletions(-) create mode 100644 packages/services/src/fee_tracker.rs create mode 100644 packages/services/src/fee_tracker/fee_analytics.rs create mode 100644 packages/services/src/fee_tracker/port.rs create mode 100644 packages/services/src/fee_tracker/service.rs create mode 100644 packages/services/src/fee_tracker/testing.rs delete mode 100644 packages/services/src/state_committer/fee_algo.rs delete mode 100644 packages/services/src/state_committer/fee_analytics.rs create mode 100644 packages/services/tests/fee_tracker.rs diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 8f9178dc..fac00a21 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,10 +9,8 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - state_committer::{ - port::Storage, - service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, - }, + fee_tracker::service::{FeeThresholds, FeeTracker, SmaPeriods}, + state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, wallet_balance_tracker::service::WalletBalanceTracker, @@ -121,6 +119,26 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, ) -> Result> { + let fee_tracker = FeeTracker::new( + l1.clone(), + services::fee_tracker::service::Config { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config + .app + .fee_algo + .start_discount_percentage + .try_into()?, + end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }, + ); + let state_committer = services::StateCommitter::new( l1, fuel, @@ -130,28 +148,9 @@ pub fn state_committer( fragment_accumulation_timeout: config.app.bundle.fragment_accumulation_timeout, fragments_to_accumulate: config.app.bundle.fragments_to_accumulate, gas_bump_timeout: config.app.gas_bump_timeout, - fee_algo: FeeAlgoConfig { - sma_periods: SmaPeriods { - short: config.app.fee_algo.short_sma_blocks, - long: config.app.fee_algo.long_sma_blocks, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config - .app - .fee_algo - .start_discount_percentage - .try_into()?, - end_premium_percentage: config - .app - .fee_algo - .end_premium_percentage - .try_into()?, - always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, - }, - }, }, SystemClock, + fee_tracker, ); Ok(schedule_polling( diff --git a/packages/adapters/eth/src/fee_conversion.rs b/packages/adapters/eth/src/fee_conversion.rs index 3063b321..0d34073c 100644 --- a/packages/adapters/eth/src/fee_conversion.rs +++ b/packages/adapters/eth/src/fee_conversion.rs @@ -3,7 +3,7 @@ use std::ops::RangeInclusive; use alloy::rpc::types::FeeHistory; use itertools::{izip, Itertools}; use services::{ - state_committer::port::l1::{BlockFees, Fees}, + fee_tracker::port::l1::{BlockFees, Fees}, Result, }; @@ -98,7 +98,7 @@ pub fn chunk_range_inclusive( #[cfg(test)] mod test { use alloy::rpc::types::FeeHistory; - use services::state_committer::port::l1::{BlockFees, Fees}; + use services::fee_tracker::port::l1::{BlockFees, Fees}; use std::ops::RangeInclusive; diff --git a/packages/adapters/eth/src/lib.rs b/packages/adapters/eth/src/lib.rs index 2cb7fc31..c3a27c29 100644 --- a/packages/adapters/eth/src/lib.rs +++ b/packages/adapters/eth/src/lib.rs @@ -14,7 +14,7 @@ use delegate::delegate; use futures::{stream, StreamExt, TryStreamExt}; use itertools::{izip, Itertools}; use services::{ - state_committer::port::l1::SequentialBlockFees, + fee_tracker::port::l1::SequentialBlockFees, types::{ BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Height, L1Tx, NonEmpty, NonNegative, TransactionResponse, @@ -206,20 +206,11 @@ impl services::block_committer::port::l1::Api for WebsocketClient { } } -impl services::state_committer::port::l1::Api for WebsocketClient { +impl services::fee_tracker::port::l1::Api for WebsocketClient { async fn current_height(&self) -> Result { self._get_block_number().await } - delegate! { - to (*self) { - async fn submit_state_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> Result<(L1Tx, FragmentsSubmitted)>; - } - } async fn fees(&self, height_range: RangeInclusive) -> Result { const REWARD_PERCENTILE: f64 = alloy::providers::utils::EIP1559_FEE_ESTIMATION_REWARD_PERCENTILE; @@ -248,6 +239,22 @@ impl services::state_committer::port::l1::Api for WebsocketClient { } } +impl services::state_committer::port::l1::Api for WebsocketClient { + async fn current_height(&self) -> Result { + self._get_block_number().await + } + + delegate! { + to (*self) { + async fn submit_state_fragments( + &self, + fragments: NonEmpty, + previous_tx: Option, + ) -> Result<(L1Tx, FragmentsSubmitted)>; + } + } +} + #[cfg(test)] mod test { use alloy::eips::eip4844::DATA_GAS_PER_BLOB; diff --git a/packages/services/src/fee_tracker.rs b/packages/services/src/fee_tracker.rs new file mode 100644 index 00000000..a977c59c --- /dev/null +++ b/packages/services/src/fee_tracker.rs @@ -0,0 +1,4 @@ +pub mod port; +pub mod service; + +mod fee_analytics; diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs new file mode 100644 index 00000000..ce7a8648 --- /dev/null +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -0,0 +1,386 @@ +use std::ops::RangeInclusive; + +use crate::Error; + +use super::port::l1::{Api, Fees, SequentialBlockFees}; + +#[derive(Debug, Clone)] +pub struct FeeAnalytics

{ + fees_provider: P, +} + +impl

FeeAnalytics

{ + pub fn new(fees_provider: P) -> Self { + Self { fees_provider } + } +} + +impl FeeAnalytics

{ + pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { + let fees = self.fees_provider.fees(block_range.clone()).await?; + + let received_height_range = fees.height_range(); + if received_height_range != block_range { + return Err(Error::from(format!( + "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" + ))); + } + + Ok(Self::mean(fees)) + } + + pub async fn fees_at_height(&self, height: u64) -> crate::Result { + let fee = self + .fees_provider + .fees(height..=height) + .await? + .into_iter() + .next() + .expect("sequential fees guaranteed not empty"); + + Ok(fee.fees) + } + + fn mean(fees: SequentialBlockFees) -> Fees { + let count = fees.len() as u128; + + let total = fees + .into_iter() + .map(|bf| bf.fees) + .fold(Fees::default(), |acc, f| Fees { + base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, + reward: acc.reward + f.reward, + base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, + }); + + // TODO: segfault should we round to nearest here? + Fees { + base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), + reward: total.reward.saturating_div(count), + base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), + } + } +} + +#[cfg(test)] +mod tests { + use itertools::Itertools; + use mockall::{predicate::eq, Sequence}; + + use crate::fee_tracker::port::l1::{testing, BlockFees}; + + use super::*; + + #[test] + fn can_create_valid_sequential_fees() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees.clone()); + + // Then + assert!( + result.is_ok(), + "Expected SequentialBlockFees creation to succeed" + ); + let sequential_fees = result.unwrap(); + assert_eq!(sequential_fees.len(), block_fees.len()); + } + + #[test] + fn sequential_fees_cannot_be_empty() { + // Given + let block_fees: Vec = vec![]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for empty input" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"Input cannot be empty\")" + ); + } + + #[test] + fn fees_must_be_sequential() { + // Given + let block_fees = vec![ + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + BlockFees { + height: 3, // Non-sequential height + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + ]; + + // When + let result = SequentialBlockFees::try_from(block_fees); + + // Then + assert!( + result.is_err(), + "Expected SequentialBlockFees creation to fail for non-sequential heights" + ); + assert_eq!( + result.unwrap_err().to_string(), + "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" + ); + } + + // TODO: segfault add more tests so that the in-order iteration invariant is properly tested + #[test] + fn produced_iterator_gives_correct_values() { + // Given + // notice the heights are out of order so that we validate that the returned sequence is in + // order + let block_fees = vec![ + BlockFees { + height: 2, + fees: Fees { + base_fee_per_gas: 110, + reward: 55, + base_fee_per_blob_gas: 15, + }, + }, + BlockFees { + height: 1, + fees: Fees { + base_fee_per_gas: 100, + reward: 50, + base_fee_per_blob_gas: 10, + }, + }, + ]; + let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); + + // When + let iterated_fees: Vec = sequential_fees.into_iter().collect(); + + // Then + let expectation = block_fees + .into_iter() + .sorted_by_key(|b| b.height) + .collect_vec(); + assert_eq!( + iterated_fees, expectation, + "Expected iterator to yield the same block fees" + ); + } + use std::path::PathBuf; + + #[tokio::test] + async fn calculates_sma_correctly_for_last_1_block() { + // given + let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); + + // then + assert_eq!(sma.base_fee_per_gas, 5); + assert_eq!(sma.reward, 5); + assert_eq!(sma.base_fee_per_blob_gas, 5); + } + + #[tokio::test] + async fn calculates_sma_correctly_for_last_5_blocks() { + // given + let fees_provider = testing::PreconfiguredFeeApi::new(testing::incrementing_fees(5)); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); + + // then + let mean = (5 + 4 + 3 + 2 + 1) / 5; + assert_eq!(sma.base_fee_per_gas, mean); + assert_eq!(sma.reward, mean); + assert_eq!(sma.base_fee_per_blob_gas, mean); + } + + #[tokio::test] + async fn errors_out_if_returned_fees_are_not_complete() { + // given + let mut fees = testing::incrementing_fees(5); + fees.remove(&4); + let fees_provider = testing::PreconfiguredFeeApi::new(fees); + let fee_analytics = FeeAnalytics::new(fees_provider); + + // when + let err = fee_analytics + .calculate_sma(0..=4) + .await + .expect_err("should have failed because returned fees are not complete"); + + // then + assert_eq!( + err.to_string(), + "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" + ); + } + + #[tokio::test] + async fn price_at_height_returns_correct_fee() { + // given + let fees_map = testing::incrementing_fees(5); + let fees_provider = testing::PreconfiguredFeeApi::new(fees_map.clone()); + let fee_analytics = FeeAnalytics::new(fees_provider); + let height = 2; + + // when + let fee = fee_analytics.fees_at_height(height).await.unwrap(); + + // then + let expected_fee = Fees { + base_fee_per_gas: 3, + reward: 3, + base_fee_per_blob_gas: 3, + }; + assert_eq!( + fee, expected_fee, + "Fee at height {height} should be {expected_fee:?}" + ); + } + + // fn calculate_tx_fee(fees: &Fees) -> u128 { + // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 + // } + // + // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { + // let mut csv_writer = + // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) + // .unwrap(); + // csv_writer + // .write_record(["height", "tx_fee"].iter()) + // .unwrap(); + // for (height, fee) in tx_fees { + // csv_writer + // .write_record([height.to_string(), fee.to_string()]) + // .unwrap(); + // } + // csv_writer.flush().unwrap(); + // } + + // #[tokio::test] + // async fn something() { + // let client = make_pub_eth_client().await; + // use services::fee_analytics::port::l1::FeesProvider; + // + // let current_block_height = 21408300; + // let starting_block_height = current_block_height - 48 * 3600 / 12; + // let data = client + // .fees(starting_block_height..=current_block_height) + // .await + // .into_iter() + // .collect::>(); + // + // let fee_lookup = data + // .iter() + // .map(|b| (b.height, b.fees)) + // .collect::>(); + // + // let short_sma = 25u64; + // let long_sma = 900; + // + // let current_tx_fees = data + // .iter() + // .map(|b| (b.height, calculate_tx_fee(&b.fees))) + // .collect::>(); + // + // save_tx_fees(¤t_tx_fees, "current_fees.csv"); + // + // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); + // let fee_analytics = FeeAnalytics::new(local_client.clone()); + // + // let mut short_sma_tx_fees = vec![]; + // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - short_sma..=height) + // .await; + // + // let tx_fee = calculate_tx_fee(&fees); + // + // short_sma_tx_fees.push((height, tx_fee)); + // } + // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); + // + // let decider = SendOrWaitDecider::new( + // FeeAnalytics::new(local_client.clone()), + // services::state_committer::fee_optimization::Config { + // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { + // short: short_sma, + // long: long_sma, + // }, + // fee_thresholds: Feethresholds { + // max_l2_blocks_behind: 43200 * 3, + // start_discount_percentage: 0.2, + // end_premium_percentage: 0.2, + // always_acceptable_fee: 1000000000000000u128, + // }, + // }, + // ); + // + // let mut decisions = vec![]; + // let mut long_sma_tx_fees = vec![]; + // + // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { + // let fees = fee_analytics + // .calculate_sma(height - long_sma..=height) + // .await; + // let tx_fee = calculate_tx_fee(&fees); + // long_sma_tx_fees.push((height, tx_fee)); + // + // if decider + // .should_send_blob_tx( + // 6, + // Context { + // at_l1_height: height, + // num_l2_blocks_behind: (height - starting_block_height) * 12, + // }, + // ) + // .await + // { + // let current_fees = fee_lookup.get(&height).unwrap(); + // let current_tx_fee = calculate_tx_fee(current_fees); + // decisions.push((height, current_tx_fee)); + // } + // } + // + // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); + // save_tx_fees(&decisions, "decisions.csv"); + // } +} diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs new file mode 100644 index 00000000..e9112e94 --- /dev/null +++ b/packages/services/src/fee_tracker/port.rs @@ -0,0 +1,463 @@ +pub mod l1 { + #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] + pub struct Fees { + pub base_fee_per_gas: u128, + pub reward: u128, + pub base_fee_per_blob_gas: u128, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct BlockFees { + pub height: u64, + pub fees: Fees, + } + use std::ops::RangeInclusive; + + use itertools::Itertools; + + #[derive(Debug, PartialEq, Eq)] + pub struct SequentialBlockFees { + fees: Vec, + } + + #[derive(Debug)] + pub struct InvalidSequence(String); + + impl std::error::Error for InvalidSequence {} + + impl std::fmt::Display for InvalidSequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } + } + + impl IntoIterator for SequentialBlockFees { + type Item = BlockFees; + type IntoIter = std::vec::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.fees.into_iter() + } + } + + // Cannot be empty + #[allow(clippy::len_without_is_empty)] + impl SequentialBlockFees { + pub fn len(&self) -> usize { + self.fees.len() + } + + pub fn height_range(&self) -> RangeInclusive { + let start = self.fees.first().expect("not empty").height; + let end = self.fees.last().expect("not empty").height; + start..=end + } + } + impl TryFrom> for SequentialBlockFees { + type Error = InvalidSequence; + fn try_from(mut fees: Vec) -> Result { + if fees.is_empty() { + return Err(InvalidSequence("Input cannot be empty".to_string())); + } + + fees.sort_by_key(|f| f.height); + + let is_sequential = fees + .iter() + .tuple_windows() + .all(|(l, r)| l.height + 1 == r.height); + + let heights = fees.iter().map(|f| f.height).collect::>(); + if !is_sequential { + return Err(InvalidSequence(format!( + "blocks are not sequential by height: {heights:?}" + ))); + } + + Ok(Self { fees }) + } + } + + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + #[cfg_attr(feature = "test-helpers", mockall::automock)] + pub trait Api { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result; + async fn current_height(&self) -> crate::Result; + } + + #[cfg(feature = "test-helpers")] + pub mod testing { + + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use itertools::Itertools; + + use super::{Api, BlockFees, Fees, SequentialBlockFees}; + + #[derive(Debug, Clone, Copy)] + pub struct ConstantFeeApi { + fees: Fees, + } + + impl ConstantFeeApi { + pub fn new(fees: Fees) -> Self { + Self { fees } + } + } + + impl Api for ConstantFeeApi { + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let fees = height_range + .into_iter() + .map(|height| BlockFees { + height, + fees: self.fees, + }) + .collect_vec(); + + Ok(fees.try_into().unwrap()) + } + + async fn current_height(&self) -> crate::Result { + Ok(0) + } + } + + #[derive(Debug, Clone)] + pub struct PreconfiguredFeeApi { + fees: BTreeMap, + } + + impl Api for PreconfiguredFeeApi { + async fn current_height(&self) -> crate::Result { + Ok(*self + .fees + .keys() + .last() + .expect("no fees registered with PreconfiguredFeesProvider")) + } + + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let fees = self + .fees + .iter() + .skip_while(|(height, _)| !height_range.contains(height)) + .take_while(|(height, _)| height_range.contains(height)) + .map(|(height, fees)| BlockFees { + height: *height, + fees: *fees, + }) + .collect_vec(); + + Ok(fees.try_into().expect("block fees not sequential")) + } + } + + impl PreconfiguredFeeApi { + pub fn new(blocks: impl IntoIterator) -> Self { + Self { + fees: blocks.into_iter().collect(), + } + } + } + + pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { + (0..num_blocks) + .map(|i| { + ( + i, + Fees { + base_fee_per_gas: i as u128 + 1, + reward: i as u128 + 1, + base_fee_per_blob_gas: i as u128 + 1, + }, + ) + }) + .collect() + } + } +} + +pub mod cache { + use std::{collections::BTreeMap, ops::RangeInclusive}; + + use tokio::sync::RwLock; + + use crate::Error; + + use super::l1::{Api, BlockFees, Fees, SequentialBlockFees}; + + #[derive(Debug)] + pub struct CachingApi

{ + fees_provider: P, + cache: RwLock>, + cache_limit: usize, + } + + impl

CachingApi

{ + pub fn new(fees_provider: P, cache_limit: usize) -> Self { + Self { + fees_provider, + cache: RwLock::new(BTreeMap::new()), + cache_limit, + } + } + } + + impl Api for CachingApi

{ + async fn fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + self.get_fees(height_range).await + } + + async fn current_height(&self) -> crate::Result { + self.fees_provider.current_height().await + } + } + + impl CachingApi

{ + pub async fn get_fees( + &self, + height_range: RangeInclusive, + ) -> crate::Result { + let mut missing_heights = vec![]; + + // Mind the scope to release the read lock + { + let cache = self.cache.read().await; + for height in height_range.clone() { + if !cache.contains_key(&height) { + missing_heights.push(height); + } + } + } + + if !missing_heights.is_empty() { + let fetched_fees = self + .fees_provider + .fees( + *missing_heights.first().expect("not empty") + ..=*missing_heights.last().expect("not empty"), + ) + .await?; + + let mut cache = self.cache.write().await; + for block_fee in fetched_fees { + cache.insert(block_fee.height, block_fee.fees); + } + } + + let fees: Vec<_> = { + let cache = self.cache.read().await; + height_range + .filter_map(|h| { + cache.get(&h).map(|f| BlockFees { + height: h, + fees: *f, + }) + }) + .collect() + }; + + self.shrink_cache().await; + + SequentialBlockFees::try_from(fees).map_err(|e| Error::Other(e.to_string())) + } + + async fn shrink_cache(&self) { + let mut cache = self.cache.write().await; + while cache.len() > self.cache_limit { + cache.pop_first(); + } + } + } + + #[cfg(test)] + mod tests { + use std::ops::RangeInclusive; + + use mockall::{predicate::eq, Sequence}; + + use crate::fee_tracker::port::{ + cache::CachingApi, + l1::{BlockFees, Fees, MockApi, SequentialBlockFees}, + }; + + #[tokio::test] + async fn caching_provider_avoids_duplicate_requests() { + // given + let mut mock_provider = MockApi::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }); + + let provider = CachingApi::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // mock validates no extra calls made + } + + #[tokio::test] + async fn caching_provider_fetches_only_missing_blocks() { + // given + let mut mock_provider = MockApi::new(); + + let mut sequence = Sequence::new(); + mock_provider + .expect_fees() + .with(eq(0..=2)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + mock_provider + .expect_fees() + .with(eq(3..=5)) + .once() + .return_once(|range| { + Box::pin(async move { + Ok(SequentialBlockFees::try_from( + range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap()) + }) + }) + .in_sequence(&mut sequence); + + let provider = CachingApi::new(mock_provider, 5); + let _ = provider.get_fees(0..=2).await.unwrap(); + + // when + let _ = provider.get_fees(2..=5).await.unwrap(); + + // then + // not called for the overlapping area + } + + #[tokio::test] + async fn caching_provider_evicts_oldest_blocks() { + // given + let mut mock_provider = MockApi::new(); + + mock_provider + .expect_fees() + .with(eq(0..=4)) + .times(2) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + mock_provider + .expect_fees() + .with(eq(5..=9)) + .times(1) + .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); + + let provider = CachingApi::new(mock_provider, 5); + let _ = provider.get_fees(0..=4).await.unwrap(); + let _ = provider.get_fees(5..=9).await.unwrap(); + + // when + let _ = provider.get_fees(0..=4).await.unwrap(); + + // then + // will refetch 0..=4 due to eviction + } + + #[tokio::test] + async fn caching_provider_handles_request_larger_than_cache() { + use mockall::predicate::*; + + // given + let mut mock_provider = MockApi::new(); + + let cache_limit = 5; + + mock_provider + .expect_fees() + .with(eq(0..=9)) + .times(1) + .returning(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); + + let provider = CachingApi::new(mock_provider, cache_limit); + + // when + let result = provider.get_fees(0..=9).await.unwrap(); + + assert_eq!(result, generate_sequential_fees(0..=9)); + } + + fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { + SequentialBlockFees::try_from( + height_range + .map(|h| BlockFees { + height: h, + fees: Fees { + base_fee_per_gas: h as u128, + reward: h as u128, + base_fee_per_blob_gas: h as u128, + }, + }) + .collect::>(), + ) + .unwrap() + } + } +} diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs new file mode 100644 index 00000000..8224c003 --- /dev/null +++ b/packages/services/src/fee_tracker/service.rs @@ -0,0 +1,370 @@ +use std::{cmp::min, num::NonZeroU32}; + +use tracing::info; + +use crate::{state_committer::service::SendOrWaitDecider, Error}; + +use super::{ + fee_analytics::FeeAnalytics, + port::l1::{Api, Fees}, +}; + +#[derive(Debug, Clone, Copy)] +pub struct Config { + pub sma_periods: SmaPeriods, + pub fee_thresholds: FeeThresholds, +} + +#[cfg(feature = "test-helpers")] +impl Default for Config { + fn default() -> Self { + Config { + sma_periods: SmaPeriods { short: 1, long: 2 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + ..FeeThresholds::default() + }, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SmaPeriods { + pub short: u64, + pub long: u64, +} + +#[derive(Debug, Clone, Copy)] +pub struct FeeThresholds { + pub max_l2_blocks_behind: NonZeroU32, + pub start_discount_percentage: Percentage, + pub end_premium_percentage: Percentage, + pub always_acceptable_fee: u128, +} + +#[cfg(feature = "test-helpers")] +impl Default for FeeThresholds { + fn default() -> Self { + Self { + max_l2_blocks_behind: NonZeroU32::MAX, + start_discount_percentage: Percentage::ZERO, + end_premium_percentage: Percentage::ZERO, + always_acceptable_fee: u128::MAX, + } + } +} + +impl SendOrWaitDecider for FeeTracker

{ + async fn should_send_blob_tx( + &self, + num_blobs: u32, + num_l2_blocks_behind: u32, + at_l1_height: u64, + ) -> crate::Result { + if self.too_far_behind(num_l2_blocks_behind) { + info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); + return Ok(true); + } + + // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller + // wants to send more than 6 blobs + let last_n_blocks = + |n: u64| at_l1_height.saturating_sub(n.saturating_sub(1))..=at_l1_height; + + let short_term_sma = self + .fee_analytics + .calculate_sma(last_n_blocks(self.config.sma_periods.short)) + .await?; + + let long_term_sma = self + .fee_analytics + .calculate_sma(last_n_blocks(self.config.sma_periods.long)) + .await?; + + let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + + if self.fee_always_acceptable(short_term_tx_fee) { + info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); + return Ok(true); + } + + let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let max_upper_tx_fee = Self::calculate_max_upper_fee( + &self.config.fee_thresholds, + long_term_tx_fee, + num_l2_blocks_behind, + ); + + let should_send = short_term_tx_fee < max_upper_tx_fee; + + if should_send { + info!( + "Sending because short term price {} is lower than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } else { + info!( + "Not sending because short term price {} is higher than the max upper fee {}", + short_term_tx_fee, max_upper_tx_fee + ); + } + + Ok(should_send) + } +} + +#[derive(Default, Copy, Clone, Debug, PartialEq)] +pub struct Percentage(f64); + +impl TryFrom for Percentage { + type Error = Error; + + fn try_from(value: f64) -> std::result::Result { + if value < 0. { + return Err(Error::Other(format!("Invalid percentage value {value}"))); + } + + Ok(Self(value)) + } +} + +impl From for f64 { + fn from(value: Percentage) -> Self { + value.0 + } +} + +impl Percentage { + pub const ZERO: Self = Percentage(0.); + pub const PPM: u128 = 1_000_000; + + pub fn ppm(&self) -> u128 { + (self.0 * 1_000_000.) as u128 + } +} + +pub struct FeeTracker

{ + fee_analytics: FeeAnalytics

, + config: Config, +} + +impl FeeTracker

{ + fn too_far_behind(&self, num_l2_blocks_behind: u32) -> bool { + num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() + } + + fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { + short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee + } + + fn calculate_max_upper_fee( + fee_thresholds: &FeeThresholds, + fee: u128, + num_l2_blocks_behind: u32, + ) -> u128 { + let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); + let blocks_behind = u128::from(num_l2_blocks_behind); + + debug_assert!( + blocks_behind <= max_blocks_behind, + "blocks_behind ({}) should not exceed max_blocks_behind ({})", + blocks_behind, + max_blocks_behind + ); + + let start_discount_ppm = min( + fee_thresholds.start_discount_percentage.ppm(), + Percentage::PPM, + ); + let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); + + // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% + let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); + + // 2. How late are we: eg. late enough to add 25% to our base multiplier + let premium_increment = Self::calculate_premium_increment( + start_discount_ppm, + end_premium_ppm, + blocks_behind, + max_blocks_behind, + ); + + // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% + let multiplier_ppm = min( + base_multiplier.saturating_add(premium_increment), + Percentage::PPM + end_premium_ppm, + ); + + // 3. Final fee: eg. 105% of the base fee + fee.saturating_mul(multiplier_ppm) + .saturating_div(Percentage::PPM) + } + + fn calculate_premium_increment( + start_discount_ppm: u128, + end_premium_ppm: u128, + blocks_behind: u128, + max_blocks_behind: u128, + ) -> u128 { + let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); + + let proportion = if max_blocks_behind == 0 { + 0 + } else { + blocks_behind + .saturating_mul(Percentage::PPM) + .saturating_div(max_blocks_behind) + }; + + total_ppm + .saturating_mul(proportion) + .saturating_div(Percentage::PPM) + } + + // TODO: Segfault maybe dont leak so much eth abstractions + fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { + const DATA_GAS_PER_BLOB: u128 = 131_072u128; + const INTRINSIC_GAS: u128 = 21_000u128; + + let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; + let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; + + base_fee + blob_fee + fees.reward + } +} + +impl

FeeTracker

{ + pub fn new(fee_provider: P, config: Config) -> Self { + Self { + fee_analytics: FeeAnalytics::new(fee_provider), + config, + } + } +} + +#[cfg(test)] +mod tests { + use crate::fee_tracker::port::l1::testing::ConstantFeeApi; + + use super::*; + use test_case::test_case; + + struct Context { + num_l2_blocks_behind: u32, + at_l1_height: u64, + } + + #[test_case( + // Test Case 1: No blocks behind, no discount or premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + Context { + num_l2_blocks_behind: 0, + at_l1_height: 0, + }, + 1000; + "No blocks behind, multiplier should be 100%" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 2000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 2050; + "Half blocks behind with discount and premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.25.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 800, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 700; + "Start discount only, no premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + end_premium_percentage: 0.30.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + 1000, + Context { + num_l2_blocks_behind: 50, + at_l1_height: 0, + }, + 1150; + "End premium only, no discount" + )] + #[test_case( + // Test Case 8: High fee with premium + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.10.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 10_000, + Context { + num_l2_blocks_behind: 99, + at_l1_height: 0, + }, + 11970; + "High fee with premium" + )] + #[test_case( + FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 1.50.try_into().unwrap(), // 150% + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + 1000, + Context { + num_l2_blocks_behind: 1, + at_l1_height: 0, + }, + 12; + "Discount exceeds 100%, should be capped to 100%" +)] + fn test_calculate_max_upper_fee( + fee_thresholds: FeeThresholds, + fee: u128, + context: Context, + expected_max_upper_fee: u128, + ) { + let Context { + num_l2_blocks_behind, + at_l1_height, + } = context; + let max_upper_fee = FeeTracker::::calculate_max_upper_fee( + &fee_thresholds, + fee, + num_l2_blocks_behind, + ); + + assert_eq!( + max_upper_fee, expected_max_upper_fee, + "Expected max_upper_fee to be {}, but got {}", + expected_max_upper_fee, max_upper_fee + ); + } +} diff --git a/packages/services/src/fee_tracker/testing.rs b/packages/services/src/fee_tracker/testing.rs new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/packages/services/src/fee_tracker/testing.rs @@ -0,0 +1,2 @@ + + diff --git a/packages/services/src/lib.rs b/packages/services/src/lib.rs index c6ceb616..48123b0a 100644 --- a/packages/services/src/lib.rs +++ b/packages/services/src/lib.rs @@ -2,6 +2,7 @@ pub mod block_bundler; pub mod block_committer; pub mod block_importer; pub mod cost_reporter; +pub mod fee_tracker; pub mod health_reporter; pub mod state_committer; pub mod state_listener; diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 22c6b436..35b4e5e6 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,6 +1,3 @@ -mod fee_algo; -mod fee_analytics; - pub mod service { use std::{ num::{NonZeroU32, NonZeroUsize}, @@ -8,77 +5,13 @@ pub mod service { }; use crate::{ - state_committer::{fee_algo::Context, fee_analytics::CachingFeesProvider}, + fee_tracker::service::FeeTracker, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, Error, Result, Runner, }; use itertools::Itertools; use tracing::info; - use super::fee_algo::SendOrWaitDecider; - - #[derive(Debug, Clone, Copy)] - pub struct SmaPeriods { - pub short: u64, - pub long: u64, - } - - #[derive(Default, Copy, Clone, Debug, PartialEq)] - pub struct Percentage(f64); - - impl TryFrom for Percentage { - type Error = crate::Error; - - fn try_from(value: f64) -> std::result::Result { - if value < 0. { - return Err(Error::Other(format!("Invalid percentage value {value}"))); - } - - Ok(Self(value)) - } - } - - impl From for f64 { - fn from(value: Percentage) -> Self { - value.0 - } - } - - impl Percentage { - pub const ZERO: Self = Percentage(0.); - pub const PPM: u128 = 1_000_000; - - pub fn ppm(&self) -> u128 { - (self.0 * 1_000_000.) as u128 - } - } - - #[derive(Debug, Clone, Copy)] - pub struct FeeThresholds { - pub max_l2_blocks_behind: NonZeroU32, - pub start_discount_percentage: Percentage, - pub end_premium_percentage: Percentage, - pub always_acceptable_fee: u128, - } - - #[cfg(feature = "test-helpers")] - impl Default for FeeThresholds { - fn default() -> Self { - Self { - max_l2_blocks_behind: NonZeroU32::MAX, - start_discount_percentage: Percentage::ZERO, - end_premium_percentage: Percentage::ZERO, - always_acceptable_fee: u128::MAX, - } - } - } - - #[derive(Debug, Clone, Copy)] - pub struct FeeAlgoConfig { - pub sma_periods: SmaPeriods, - pub fee_thresholds: FeeThresholds, - } - // src/config.rs #[derive(Debug, Clone)] pub struct Config { @@ -87,7 +20,6 @@ pub mod service { pub fragment_accumulation_timeout: Duration, pub fragments_to_accumulate: NonZeroUsize, pub gas_bump_timeout: Duration, - pub fee_algo: FeeAlgoConfig, } #[cfg(feature = "test-helpers")] @@ -98,31 +30,34 @@ pub mod service { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - fee_algo: FeeAlgoConfig { - sma_periods: SmaPeriods { short: 1, long: 2 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - ..FeeThresholds::default() - }, - }, } } } + #[allow(async_fn_in_trait)] + #[trait_variant::make(Send)] + pub trait SendOrWaitDecider { + async fn should_send_blob_tx( + &self, + num_blobs: u32, + num_l2_blocks_behind: u32, + at_l1_height: u64, + ) -> Result; + } + /// The `StateCommitter` is responsible for committing state fragments to L1. - pub struct StateCommitter { + pub struct StateCommitter { l1_adapter: L1, fuel_api: FuelApi, storage: Db, config: Config, clock: Clock, startup_time: DateTime, - decider: SendOrWaitDecider>, + decider: D, } - impl StateCommitter + impl StateCommitter where - L1: Clone + Send + Sync, Clock: crate::state_committer::port::Clock, { /// Creates a new `StateCommitter`. @@ -132,15 +67,10 @@ pub mod service { storage: Db, config: Config, clock: Clock, + decider: Decider, ) -> Self { let startup_time = clock.now(); - let price_algo = config.fee_algo; - // TODO: segfault, configure this cache - let decider = SendOrWaitDecider::new( - CachingFeesProvider::new(l1_adapter.clone(), 24 * 3600 / 12), - price_algo, - ); Self { l1_adapter, fuel_api, @@ -153,12 +83,13 @@ pub mod service { } } - impl StateCommitter + impl StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api, Db: crate::state_committer::port::Storage, Clock: crate::state_committer::port::Clock, + Decider: SendOrWaitDecider, { async fn get_reference_time(&self) -> Result> { Ok(self @@ -190,10 +121,8 @@ pub mod service { self.decider .should_send_blob_tx( u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), - Context { - num_l2_blocks_behind, - at_l1_height: l1_height, - }, + num_l2_blocks_behind, + l1_height, ) .await } @@ -344,12 +273,13 @@ pub mod service { } } - impl Runner for StateCommitter + impl Runner for StateCommitter where L1: crate::state_committer::port::l1::Api + Send + Sync, FuelApi: crate::state_committer::port::fuel::Api + Send + Sync, Db: crate::state_committer::port::Storage + Clone + Send + Sync, Clock: crate::state_committer::port::Clock + Send + Sync, + Decider: SendOrWaitDecider + Send + Sync, { async fn run(&mut self) -> Result<()> { if self.storage.has_nonfinalized_txs().await? { @@ -376,7 +306,6 @@ pub mod port { use nonempty::NonEmpty; - pub use crate::state_committer::fee_analytics::{BlockFees, Fees, SequentialBlockFees}; use crate::{ types::{BlockSubmissionTx, Fragment, FragmentsSubmitted, L1Tx}, Result, @@ -398,80 +327,6 @@ pub mod port { fragments: NonEmpty, previous_tx: Option, ) -> Result<(L1Tx, FragmentsSubmitted)>; - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result; - } - - #[cfg(feature = "test-helpers")] - pub mod testing { - use std::{ops::RangeInclusive, sync::Arc}; - - use nonempty::NonEmpty; - - use crate::{ - state_committer::fee_analytics::{ - testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - FeesProvider, - }, - types::{FragmentsSubmitted, L1Tx}, - }; - - use super::{Api, Fees, MockApi, SequentialBlockFees}; - - #[derive(Clone)] - pub struct ApiMockWFees

{ - pub api: Arc, - fee_provider: P, - } - - impl ApiMockWFees { - pub fn new(api: MockApi) -> Self { - Self { - api: Arc::new(api), - fee_provider: ConstantFeesProvider::new(Fees::default()), - } - } - } - - impl

ApiMockWFees

{ - pub fn w_preconfigured_fees( - self, - fees: impl IntoIterator, - ) -> ApiMockWFees { - ApiMockWFees { - api: self.api, - fee_provider: PreconfiguredFeesProvider::new(fees), - } - } - } - - impl

Api for ApiMockWFees

- where - P: FeesProvider + Send + Sync, - { - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - FeesProvider::fees(&self.fee_provider, height_range).await - } - - async fn current_height(&self) -> crate::Result { - self.api.current_height().await - } - - async fn submit_state_fragments( - &self, - fragments: NonEmpty, - previous_tx: Option, - ) -> crate::Result<(L1Tx, FragmentsSubmitted)> { - self.api - .submit_state_fragments(fragments, previous_tx) - .await - } - } } } diff --git a/packages/services/src/state_committer/fee_algo.rs b/packages/services/src/state_committer/fee_algo.rs deleted file mode 100644 index 5e6cc4fa..00000000 --- a/packages/services/src/state_committer/fee_algo.rs +++ /dev/null @@ -1,674 +0,0 @@ -use std::cmp::min; - -use tracing::info; - -use crate::state_committer::service::Percentage; - -use super::{ - fee_analytics::{FeeAnalytics, FeesProvider}, - port::l1::Fees, - service::{FeeAlgoConfig, FeeThresholds}, -}; - -pub struct SendOrWaitDecider

{ - fee_analytics: FeeAnalytics

, - config: FeeAlgoConfig, -} - -impl

SendOrWaitDecider

{ - pub fn new(fee_provider: P, config: FeeAlgoConfig) -> Self { - Self { - fee_analytics: FeeAnalytics::new(fee_provider), - config, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct Context { - pub num_l2_blocks_behind: u32, - pub at_l1_height: u64, -} - -impl SendOrWaitDecider

{ - pub async fn should_send_blob_tx( - &self, - num_blobs: u32, - context: Context, - ) -> crate::Result { - if self.too_far_behind(context) { - info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", context.num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); - return Ok(true); - } - - // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller - // wants to send more than 6 blobs - let last_n_blocks = |n: u64| { - context.at_l1_height.saturating_sub(n.saturating_sub(1))..=context.at_l1_height - }; - - let short_term_sma = self - .fee_analytics - .calculate_sma(last_n_blocks(self.config.sma_periods.short)) - .await?; - - let long_term_sma = self - .fee_analytics - .calculate_sma(last_n_blocks(self.config.sma_periods.long)) - .await?; - - let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); - - if self.fee_always_acceptable(short_term_tx_fee) { - info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); - return Ok(true); - } - - let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); - let max_upper_tx_fee = - Self::calculate_max_upper_fee(&self.config.fee_thresholds, long_term_tx_fee, context); - - let should_send = short_term_tx_fee < max_upper_tx_fee; - - if should_send { - info!( - "Sending because short term price {} is lower than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } else { - info!( - "Not sending because short term price {} is higher than the max upper fee {}", - short_term_tx_fee, max_upper_tx_fee - ); - } - - Ok(should_send) - } - - fn too_far_behind(&self, context: Context) -> bool { - context.num_l2_blocks_behind >= self.config.fee_thresholds.max_l2_blocks_behind.get() - } - - fn fee_always_acceptable(&self, short_term_tx_fee: u128) -> bool { - short_term_tx_fee <= self.config.fee_thresholds.always_acceptable_fee - } - - fn calculate_max_upper_fee( - fee_thresholds: &FeeThresholds, - fee: u128, - context: Context, - ) -> u128 { - let max_blocks_behind = u128::from(fee_thresholds.max_l2_blocks_behind.get()); - let blocks_behind = u128::from(context.num_l2_blocks_behind); - - debug_assert!( - blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({})", - blocks_behind, - max_blocks_behind - ); - - let start_discount_ppm = min( - fee_thresholds.start_discount_percentage.ppm(), - Percentage::PPM, - ); - let end_premium_ppm = fee_thresholds.end_premium_percentage.ppm(); - - // 1. The highest we're initially willing to go: eg. 100% - 20% = 80% - let base_multiplier = Percentage::PPM.saturating_sub(start_discount_ppm); - - // 2. How late are we: eg. late enough to add 25% to our base multiplier - let premium_increment = Self::calculate_premium_increment( - start_discount_ppm, - end_premium_ppm, - blocks_behind, - max_blocks_behind, - ); - - // 3. Total multiplier consist of the base and the premium increment: eg. 80% + 25% = 105% - let multiplier_ppm = min( - base_multiplier.saturating_add(premium_increment), - Percentage::PPM + end_premium_ppm, - ); - - // 3. Final fee: eg. 105% of the base fee - fee.saturating_mul(multiplier_ppm) - .saturating_div(Percentage::PPM) - } - - fn calculate_premium_increment( - start_discount_ppm: u128, - end_premium_ppm: u128, - blocks_behind: u128, - max_blocks_behind: u128, - ) -> u128 { - let total_ppm = start_discount_ppm.saturating_add(end_premium_ppm); - - let proportion = if max_blocks_behind == 0 { - 0 - } else { - blocks_behind - .saturating_mul(Percentage::PPM) - .saturating_div(max_blocks_behind) - }; - - total_ppm - .saturating_mul(proportion) - .saturating_div(Percentage::PPM) - } - - // TODO: Segfault maybe dont leak so much eth abstractions - fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { - const DATA_GAS_PER_BLOB: u128 = 131_072u128; - const INTRINSIC_GAS: u128 = 21_000u128; - - let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; - let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; - - base_fee + blob_fee + fees.reward - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - state_committer::{ - fee_analytics::testing::{ConstantFeesProvider, PreconfiguredFeesProvider}, - service::{FeeThresholds, Percentage, SmaPeriods}, - }, - types::NonNegative, - }; - - use test_case::test_case; - use tokio; - - fn generate_fees(config: FeeAlgoConfig, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { - let older_fees = std::iter::repeat_n( - old_fees, - (config.sma_periods.long - config.sma_periods.short) as usize, - ); - let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); - - older_fees - .chain(newer_fees) - .enumerate() - .map(|(i, f)| (i as u64, f)) - .collect() - } - - #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, // not behind at all - true; - "Should send because all short-term fees are lower than long-term" - )] - #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }, - 0, - false; - "Should not send because all short-term fees are higher than long-term" - )] - #[test_case( - Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, - max_l2_blocks_behind: 100.try_into().unwrap(), - ..Default::default() - } - }, - 0, - true; - "Should send since short-term fee < always_acceptable_fee" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_gas is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_gas is higher" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term base_fee_per_blob_gas is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term base_fee_per_blob_gas is higher" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because short-term reward is lower" - )] - #[test_case( - Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, - Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, - 5, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because short-term reward is higher" - )] - #[test_case( - // Multiple short-term fees are lower - Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Should send because multiple short-term fees are lower" - )] - #[test_case( - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, - 6, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Should not send because all fees are identical and no tolerance" - )] - #[test_case( - // Zero blobs scenario: blob fee differences don't matter - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, - 0, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: short-term base_fee_per_gas and reward are lower, send" - )] - #[test_case( - // Zero blobs but short-term reward is higher - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, - 0, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - false; - "Zero blobs: short-term reward is higher, don't send" - )] - #[test_case( - // Zero blobs don't care about higher short-term base_fee_per_blob_gas - Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, - 0, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - } - }, - 0, - true; - "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" - )] - // Initially not send, but as num_l2_blocks_behind increases, acceptance grows. - #[test_case( - // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: Percentage::try_from(0.20).unwrap(), - end_premium_percentage: Percentage::try_from(0.20).unwrap(), - always_acceptable_fee: 0, - }, - }, - 0, - false; - "Early: short-term expensive, not send" - )] - #[test_case( - // At max_l2_blocks_behind, send regardless - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6}, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - } - }, - 100, - true; - "Later: after max wait, send regardless" - )] - #[test_case( - Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, - Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - }, - 80, - true; - "Mid-wait: increased tolerance allows acceptance" - )] - #[test_case( - // Short-term fee is huge, but always_acceptable_fee is large, so send immediately - Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, - Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, - 1, - FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 2_700_000_000_000 - }, - }, - 0, - true; - "Always acceptable fee triggers immediate send" - )] - #[tokio::test] - async fn parameterized_send_or_wait_tests( - old_fees: Fees, - new_fees: Fees, - num_blobs: u32, - config: FeeAlgoConfig, - num_l2_blocks_behind: u32, - expected_decision: bool, - ) { - let fees = generate_fees(config, old_fees, new_fees); - let fees_provider = PreconfiguredFeesProvider::new(fees); - let current_block_height = fees_provider.current_block_height().await.unwrap(); - - let sut = SendOrWaitDecider::new(fees_provider, config); - - let should_send = sut - .should_send_blob_tx( - num_blobs, - Context { - at_l1_height: current_block_height, - num_l2_blocks_behind, - }, - ) - .await - .unwrap(); - - assert_eq!( - should_send, expected_decision, - "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", - ); - } - - #[test_case( - // Test Case 1: No blocks behind, no discount or premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - Context { - num_l2_blocks_behind: 0, - at_l1_height: 0, - }, - 1000; - "No blocks behind, multiplier should be 100%" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.20.try_into().unwrap(), - end_premium_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 2000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, - 2050; - "Half blocks behind with discount and premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.25.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 800, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, - 700; - "Start discount only, no premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - end_premium_percentage: 0.30.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - 1000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, - 1150; - "End premium only, no discount" - )] - #[test_case( - // Test Case 8: High fee with premium - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 0.10.try_into().unwrap(), - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 10_000, - Context { - num_l2_blocks_behind: 99, - at_l1_height: 0, - }, - 11970; - "High fee with premium" - )] - #[test_case( - FeeThresholds { - max_l2_blocks_behind: 100.try_into().unwrap(), - start_discount_percentage: 1.50.try_into().unwrap(), // 150% - end_premium_percentage: 0.20.try_into().unwrap(), - always_acceptable_fee: 0, - }, - 1000, - Context { - num_l2_blocks_behind: 1, - at_l1_height: 0, - }, - 12; - "Discount exceeds 100%, should be capped to 100%" -)] - fn test_calculate_max_upper_fee( - fee_thresholds: FeeThresholds, - fee: u128, - context: Context, - expected_max_upper_fee: u128, - ) { - let max_upper_fee = SendOrWaitDecider::::calculate_max_upper_fee( - &fee_thresholds, - fee, - context, - ); - - assert_eq!( - max_upper_fee, expected_max_upper_fee, - "Expected max_upper_fee to be {}, but got {}", - expected_max_upper_fee, max_upper_fee - ); - } - #[tokio::test] - async fn test_send_when_too_far_behind_and_fee_provider_fails() { - // given - let config = FeeAlgoConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: 10.try_into().unwrap(), - always_acceptable_fee: 0, - ..Default::default() - }, - }; - - // having no fees will make the validation in fee analytics fail - let fee_provider = PreconfiguredFeesProvider::new(vec![]); - let sut = SendOrWaitDecider::new(fee_provider, config); - - let context = Context { - num_l2_blocks_behind: 20, - at_l1_height: 100, - }; - - // when - let should_send = sut - .should_send_blob_tx(1, context) - .await - .expect("Should send despite fee provider failure"); - - // then - assert!( - should_send, - "Should send because too far behind, regardless of fee provider status" - ); - } -} diff --git a/packages/services/src/state_committer/fee_analytics.rs b/packages/services/src/state_committer/fee_analytics.rs deleted file mode 100644 index 548223cb..00000000 --- a/packages/services/src/state_committer/fee_analytics.rs +++ /dev/null @@ -1,829 +0,0 @@ -use std::{collections::BTreeMap, ops::RangeInclusive}; - -use itertools::Itertools; -use tokio::sync::RwLock; - -use crate::state_committer::port::l1::Api; - -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] -pub struct Fees { - pub base_fee_per_gas: u128, - pub reward: u128, - pub base_fee_per_blob_gas: u128, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct BlockFees { - pub height: u64, - pub fees: Fees, -} - -#[derive(Debug, PartialEq, Eq)] -pub struct SequentialBlockFees { - fees: Vec, -} - -#[allow(async_fn_in_trait)] -#[trait_variant::make(Send)] -#[cfg_attr(feature = "test-helpers", mockall::automock)] -pub trait FeesProvider { - async fn fees(&self, height_range: RangeInclusive) -> crate::Result; - async fn current_block_height(&self) -> crate::Result; -} - -#[derive(Debug)] -pub struct CachingFeesProvider

{ - fees_provider: P, - cache: RwLock>, - cache_limit: usize, -} - -impl

CachingFeesProvider

{ - pub fn new(fees_provider: P, cache_limit: usize) -> Self { - Self { - fees_provider, - cache: RwLock::new(BTreeMap::new()), - cache_limit, - } - } -} - -impl FeesProvider for CachingFeesProvider

{ - async fn fees(&self, height_range: RangeInclusive) -> crate::Result { - self.get_fees(height_range).await - } - - async fn current_block_height(&self) -> crate::Result { - self.fees_provider.current_block_height().await - } -} - -impl CachingFeesProvider

{ - pub async fn get_fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let mut missing_heights = vec![]; - - // Mind the scope to release the read lock - { - let cache = self.cache.read().await; - for height in height_range.clone() { - if !cache.contains_key(&height) { - missing_heights.push(height); - } - } - } - - if !missing_heights.is_empty() { - let fetched_fees = self - .fees_provider - .fees( - *missing_heights.first().expect("not empty") - ..=*missing_heights.last().expect("not empty"), - ) - .await?; - - let mut cache = self.cache.write().await; - for block_fee in fetched_fees { - cache.insert(block_fee.height, block_fee.fees); - } - } - - let fees: Vec<_> = { - let cache = self.cache.read().await; - height_range - .filter_map(|h| { - cache.get(&h).map(|f| BlockFees { - height: h, - fees: *f, - }) - }) - .collect() - }; - - self.shrink_cache().await; - - SequentialBlockFees::try_from(fees).map_err(|e| crate::Error::Other(e.to_string())) - } - - async fn shrink_cache(&self) { - let mut cache = self.cache.write().await; - while cache.len() > self.cache_limit { - cache.pop_first(); - } - } -} - -impl FeesProvider for T { - async fn fees(&self, height_range: RangeInclusive) -> crate::Result { - Api::fees(self, height_range).await - } - - async fn current_block_height(&self) -> crate::Result { - Api::current_height(self).await - } -} - -impl IntoIterator for SequentialBlockFees { - type Item = BlockFees; - type IntoIter = std::vec::IntoIter; - fn into_iter(self) -> Self::IntoIter { - self.fees.into_iter() - } -} - -// Cannot be empty -#[allow(clippy::len_without_is_empty)] -impl SequentialBlockFees { - pub fn len(&self) -> usize { - self.fees.len() - } - - pub fn height_range(&self) -> RangeInclusive { - let start = self.fees.first().expect("not empty").height; - let end = self.fees.last().expect("not empty").height; - start..=end - } -} - -#[derive(Debug)] -pub struct InvalidSequence(String); - -impl std::error::Error for InvalidSequence {} - -impl std::fmt::Display for InvalidSequence { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") - } -} - -impl TryFrom> for SequentialBlockFees { - type Error = InvalidSequence; - fn try_from(mut fees: Vec) -> Result { - if fees.is_empty() { - return Err(InvalidSequence("Input cannot be empty".to_string())); - } - - fees.sort_by_key(|f| f.height); - - let is_sequential = fees - .iter() - .tuple_windows() - .all(|(l, r)| l.height + 1 == r.height); - - let heights = fees.iter().map(|f| f.height).collect::>(); - if !is_sequential { - return Err(InvalidSequence(format!( - "blocks are not sequential by height: {heights:?}" - ))); - } - - Ok(Self { fees }) - } -} - -#[cfg(feature = "test-helpers")] -pub mod testing { - use std::{collections::BTreeMap, ops::RangeInclusive}; - - use itertools::Itertools; - - use crate::state_committer::port::l1::{BlockFees, Fees}; - - use super::{FeesProvider, SequentialBlockFees}; - - #[derive(Debug, Clone, Copy)] - pub struct ConstantFeesProvider { - fees: Fees, - } - - impl ConstantFeesProvider { - pub fn new(fees: Fees) -> Self { - Self { fees } - } - } - - impl FeesProvider for ConstantFeesProvider { - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let fees = height_range - .into_iter() - .map(|height| BlockFees { - height, - fees: self.fees, - }) - .collect_vec(); - - Ok(fees.try_into().unwrap()) - } - - async fn current_block_height(&self) -> crate::Result { - Ok(0) - } - } - - #[derive(Debug, Clone)] - pub struct PreconfiguredFeesProvider { - fees: BTreeMap, - } - - impl FeesProvider for PreconfiguredFeesProvider { - async fn current_block_height(&self) -> crate::Result { - Ok(*self - .fees - .keys() - .last() - .expect("no fees registered with PreconfiguredFeesProvider")) - } - - async fn fees( - &self, - height_range: RangeInclusive, - ) -> crate::Result { - let fees = self - .fees - .iter() - .skip_while(|(height, _)| !height_range.contains(height)) - .take_while(|(height, _)| height_range.contains(height)) - .map(|(height, fees)| BlockFees { - height: *height, - fees: *fees, - }) - .collect_vec(); - - Ok(fees.try_into().expect("block fees not sequential")) - } - } - - impl PreconfiguredFeesProvider { - pub fn new(blocks: impl IntoIterator) -> Self { - Self { - fees: blocks.into_iter().collect(), - } - } - } - - pub fn incrementing_fees(num_blocks: u64) -> BTreeMap { - (0..num_blocks) - .map(|i| { - ( - i, - Fees { - base_fee_per_gas: i as u128 + 1, - reward: i as u128 + 1, - base_fee_per_blob_gas: i as u128 + 1, - }, - ) - }) - .collect() - } -} - -#[derive(Debug, Clone)] -pub struct FeeAnalytics

{ - fees_provider: P, -} -impl

FeeAnalytics

{ - pub fn new(fees_provider: P) -> Self { - Self { fees_provider } - } -} - -impl FeeAnalytics

{ - pub async fn calculate_sma(&self, block_range: RangeInclusive) -> crate::Result { - let fees = self.fees_provider.fees(block_range.clone()).await?; - - let received_height_range = fees.height_range(); - if received_height_range != block_range { - return Err(crate::Error::from(format!( - "fees received from the adapter({received_height_range:?}) don't cover the requested range ({block_range:?})" - ))); - } - - Ok(Self::mean(fees)) - } - - pub async fn fees_at_height(&self, height: u64) -> crate::Result { - let fee = self - .fees_provider - .fees(height..=height) - .await? - .into_iter() - .next() - .expect("sequential fees guaranteed not empty"); - - Ok(fee.fees) - } - - fn mean(fees: SequentialBlockFees) -> Fees { - let count = fees.len() as u128; - - let total = fees - .into_iter() - .map(|bf| bf.fees) - .fold(Fees::default(), |acc, f| Fees { - base_fee_per_gas: acc.base_fee_per_gas + f.base_fee_per_gas, - reward: acc.reward + f.reward, - base_fee_per_blob_gas: acc.base_fee_per_blob_gas + f.base_fee_per_blob_gas, - }); - - // TODO: segfault should we round to nearest here? - Fees { - base_fee_per_gas: total.base_fee_per_gas.saturating_div(count), - reward: total.reward.saturating_div(count), - base_fee_per_blob_gas: total.base_fee_per_blob_gas.saturating_div(count), - } - } -} - -#[cfg(test)] -mod tests { - use itertools::Itertools; - use mockall::{predicate::eq, Sequence}; - use testing::{incrementing_fees, PreconfiguredFeesProvider}; - - use super::*; - - #[test] - fn can_create_valid_sequential_fees() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees.clone()); - - // Then - assert!( - result.is_ok(), - "Expected SequentialBlockFees creation to succeed" - ); - let sequential_fees = result.unwrap(); - assert_eq!(sequential_fees.len(), block_fees.len()); - } - - #[test] - fn sequential_fees_cannot_be_empty() { - // Given - let block_fees: Vec = vec![]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for empty input" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"Input cannot be empty\")" - ); - } - - #[test] - fn fees_must_be_sequential() { - // Given - let block_fees = vec![ - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - BlockFees { - height: 3, // Non-sequential height - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - ]; - - // When - let result = SequentialBlockFees::try_from(block_fees); - - // Then - assert!( - result.is_err(), - "Expected SequentialBlockFees creation to fail for non-sequential heights" - ); - assert_eq!( - result.unwrap_err().to_string(), - "InvalidSequence(\"blocks are not sequential by height: [1, 3]\")" - ); - } - - // TODO: segfault add more tests so that the in-order iteration invariant is properly tested - #[test] - fn produced_iterator_gives_correct_values() { - // Given - // notice the heights are out of order so that we validate that the returned sequence is in - // order - let block_fees = vec![ - BlockFees { - height: 2, - fees: Fees { - base_fee_per_gas: 110, - reward: 55, - base_fee_per_blob_gas: 15, - }, - }, - BlockFees { - height: 1, - fees: Fees { - base_fee_per_gas: 100, - reward: 50, - base_fee_per_blob_gas: 10, - }, - }, - ]; - let sequential_fees = SequentialBlockFees::try_from(block_fees.clone()).unwrap(); - - // When - let iterated_fees: Vec = sequential_fees.into_iter().collect(); - - // Then - let expectation = block_fees - .into_iter() - .sorted_by_key(|b| b.height) - .collect_vec(); - assert_eq!( - iterated_fees, expectation, - "Expected iterator to yield the same block fees" - ); - } - use std::path::PathBuf; - - #[tokio::test] - async fn calculates_sma_correctly_for_last_1_block() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(4..=4).await.unwrap(); - - // then - assert_eq!(sma.base_fee_per_gas, 5); - assert_eq!(sma.reward, 5); - assert_eq!(sma.base_fee_per_blob_gas, 5); - } - - #[tokio::test] - async fn calculates_sma_correctly_for_last_5_blocks() { - // given - let fees_provider = testing::PreconfiguredFeesProvider::new(testing::incrementing_fees(5)); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let sma = fee_analytics.calculate_sma(0..=4).await.unwrap(); - - // then - let mean = (5 + 4 + 3 + 2 + 1) / 5; - assert_eq!(sma.base_fee_per_gas, mean); - assert_eq!(sma.reward, mean); - assert_eq!(sma.base_fee_per_blob_gas, mean); - } - - #[tokio::test] - async fn errors_out_if_returned_fees_are_not_complete() { - // given - let mut fees = testing::incrementing_fees(5); - fees.remove(&4); - let fees_provider = testing::PreconfiguredFeesProvider::new(fees); - let fee_analytics = FeeAnalytics::new(fees_provider); - - // when - let err = fee_analytics - .calculate_sma(0..=4) - .await - .expect_err("should have failed because returned fees are not complete"); - - // then - assert_eq!( - err.to_string(), - "fees received from the adapter(0..=3) don't cover the requested range (0..=4)" - ); - } - - #[tokio::test] - async fn caching_provider_avoids_duplicate_requests() { - // given - let mut mock_provider = MockFeesProvider::new(); - - mock_provider - .expect_fees() - .with(eq(0..=4)) - .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) - }) - }); - - let provider = CachingFeesProvider::new(mock_provider, 5); - let _ = provider.get_fees(0..=4).await.unwrap(); - - // when - let _ = provider.get_fees(0..=4).await.unwrap(); - - // then - // mock validates no extra calls made - } - - #[tokio::test] - async fn caching_provider_fetches_only_missing_blocks() { - // Given: A mock FeesProvider - let mut mock_provider = MockFeesProvider::new(); - - // Expectation: The provider will fetch blocks 3..=5, since 0..=2 are cached - let mut sequence = Sequence::new(); - mock_provider - .expect_fees() - .with(eq(0..=2)) - .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) - }) - }) - .in_sequence(&mut sequence); - - mock_provider - .expect_fees() - .with(eq(3..=5)) - .once() - .return_once(|range| { - Box::pin(async move { - Ok(SequentialBlockFees::try_from( - range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap()) - }) - }) - .in_sequence(&mut sequence); - - let provider = CachingFeesProvider::new(mock_provider, 5); - let _ = provider.get_fees(0..=2).await.unwrap(); - - // when - let _ = provider.get_fees(2..=5).await.unwrap(); - - // then - // not called for the overlapping area - } - - fn generate_sequential_fees(height_range: RangeInclusive) -> SequentialBlockFees { - SequentialBlockFees::try_from( - height_range - .map(|h| BlockFees { - height: h, - fees: Fees { - base_fee_per_gas: h as u128, - reward: h as u128, - base_fee_per_blob_gas: h as u128, - }, - }) - .collect::>(), - ) - .unwrap() - } - - #[tokio::test] - async fn caching_provider_evicts_oldest_blocks() { - // given - let mut mock_provider = MockFeesProvider::new(); - - mock_provider - .expect_fees() - .with(eq(0..=4)) - .times(2) - .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); - - mock_provider - .expect_fees() - .with(eq(5..=9)) - .times(1) - .returning(|range| Box::pin(async { Ok(generate_sequential_fees(range)) })); - - let provider = CachingFeesProvider::new(mock_provider, 5); - let _ = provider.get_fees(0..=4).await.unwrap(); - let _ = provider.get_fees(5..=9).await.unwrap(); - - // when - let _ = provider.get_fees(0..=4).await.unwrap(); - - // then - // will refetch 0..=4 due to eviction - } - - #[tokio::test] - async fn caching_provider_handles_request_larger_than_cache() { - use mockall::predicate::*; - - // given - let mut mock_provider = MockFeesProvider::new(); - - let cache_limit = 5; - - mock_provider - .expect_fees() - .with(eq(0..=9)) - .times(1) - .returning(|range| Box::pin(async move { Ok(generate_sequential_fees(range)) })); - - let provider = CachingFeesProvider::new(mock_provider, cache_limit); - - // when - let result = provider.get_fees(0..=9).await.unwrap(); - - assert_eq!(result, generate_sequential_fees(0..=9)); - } - - #[tokio::test] - async fn price_at_height_returns_correct_fee() { - // given - let fees_map = incrementing_fees(5); - let fees_provider = PreconfiguredFeesProvider::new(fees_map.clone()); - let fee_analytics = FeeAnalytics::new(fees_provider); - let height = 2; - - // when - let fee = fee_analytics.fees_at_height(height).await.unwrap(); - - // then - let expected_fee = Fees { - base_fee_per_gas: 3, - reward: 3, - base_fee_per_blob_gas: 3, - }; - assert_eq!( - fee, expected_fee, - "Fee at height {height} should be {expected_fee:?}" - ); - } - - // fn calculate_tx_fee(fees: &Fees) -> u128 { - // 21_000 * fees.base_fee_per_gas + fees.reward + 6 * fees.base_fee_per_blob_gas * 131_072 - // } - // - // fn save_tx_fees(tx_fees: &[(u64, u128)], path: &str) { - // let mut csv_writer = - // csv::Writer::from_path(PathBuf::from("/home/segfault_magnet/grafovi/").join(path)) - // .unwrap(); - // csv_writer - // .write_record(["height", "tx_fee"].iter()) - // .unwrap(); - // for (height, fee) in tx_fees { - // csv_writer - // .write_record([height.to_string(), fee.to_string()]) - // .unwrap(); - // } - // csv_writer.flush().unwrap(); - // } - - // #[tokio::test] - // async fn something() { - // let client = make_pub_eth_client().await; - // use services::fee_analytics::port::l1::FeesProvider; - // - // let current_block_height = 21408300; - // let starting_block_height = current_block_height - 48 * 3600 / 12; - // let data = client - // .fees(starting_block_height..=current_block_height) - // .await - // .into_iter() - // .collect::>(); - // - // let fee_lookup = data - // .iter() - // .map(|b| (b.height, b.fees)) - // .collect::>(); - // - // let short_sma = 25u64; - // let long_sma = 900; - // - // let current_tx_fees = data - // .iter() - // .map(|b| (b.height, calculate_tx_fee(&b.fees))) - // .collect::>(); - // - // save_tx_fees(¤t_tx_fees, "current_fees.csv"); - // - // let local_client = TestFeesProvider::new(data.clone().into_iter().map(|e| (e.height, e.fees))); - // let fee_analytics = FeeAnalytics::new(local_client.clone()); - // - // let mut short_sma_tx_fees = vec![]; - // for height in (starting_block_height..=current_block_height).skip(short_sma as usize) { - // let fees = fee_analytics - // .calculate_sma(height - short_sma..=height) - // .await; - // - // let tx_fee = calculate_tx_fee(&fees); - // - // short_sma_tx_fees.push((height, tx_fee)); - // } - // save_tx_fees(&short_sma_tx_fees, "short_sma_fees.csv"); - // - // let decider = SendOrWaitDecider::new( - // FeeAnalytics::new(local_client.clone()), - // services::state_committer::fee_optimization::Config { - // sma_periods: services::state_committer::fee_optimization::SmaBlockNumPeriods { - // short: short_sma, - // long: long_sma, - // }, - // fee_thresholds: Feethresholds { - // max_l2_blocks_behind: 43200 * 3, - // start_discount_percentage: 0.2, - // end_premium_percentage: 0.2, - // always_acceptable_fee: 1000000000000000u128, - // }, - // }, - // ); - // - // let mut decisions = vec![]; - // let mut long_sma_tx_fees = vec![]; - // - // for height in (starting_block_height..=current_block_height).skip(long_sma as usize) { - // let fees = fee_analytics - // .calculate_sma(height - long_sma..=height) - // .await; - // let tx_fee = calculate_tx_fee(&fees); - // long_sma_tx_fees.push((height, tx_fee)); - // - // if decider - // .should_send_blob_tx( - // 6, - // Context { - // at_l1_height: height, - // num_l2_blocks_behind: (height - starting_block_height) * 12, - // }, - // ) - // .await - // { - // let current_fees = fee_lookup.get(&height).unwrap(); - // let current_tx_fee = calculate_tx_fee(current_fees); - // decisions.push((height, current_tx_fee)); - // } - // } - // - // save_tx_fees(&long_sma_tx_fees, "long_sma_fees.csv"); - // save_tx_fees(&decisions, "decisions.csv"); - // } -} diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs new file mode 100644 index 00000000..c8f5f0ed --- /dev/null +++ b/packages/services/tests/fee_tracker.rs @@ -0,0 +1,379 @@ +use services::fee_tracker::port::l1::testing::ConstantFeeApi; +use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; +use services::fee_tracker::port::l1::Api; +use services::fee_tracker::service::FeeThresholds; +use services::fee_tracker::service::FeeTracker; +use services::fee_tracker::service::Percentage; +use services::fee_tracker::service::SmaPeriods; +use services::fee_tracker::{port::l1::Fees, service::Config}; +use services::state_committer::service::SendOrWaitDecider; +use test_case::test_case; + +fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { + let older_fees = std::iter::repeat_n( + old_fees, + (config.sma_periods.long - config.sma_periods.short) as usize, + ); + let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); + + older_fees + .chain(newer_fees) + .enumerate() + .map(|(i, f)| (i as u64, f)) + .collect() +} + +#[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + 6, + Config { + sma_periods: services::fee_tracker::service::SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, // not behind at all + true; + "Should send because all short-term fees are lower than long-term" + )] +#[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }, + 0, + false; + "Should not send because all short-term fees are higher than long-term" + )] +#[test_case( + Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, + max_l2_blocks_behind: 100.try_into().unwrap(), + ..Default::default() + } + }, + 0, + true; + "Should send since short-term fee < always_acceptable_fee" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_gas is lower" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_gas is higher" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term base_fee_per_blob_gas is lower" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term base_fee_per_blob_gas is higher" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because short-term reward is lower" + )] +#[test_case( + Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, + Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, + 5, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because short-term reward is higher" + )] +#[test_case( + // Multiple short-term fees are lower + Fees { base_fee_per_gas: 4000, reward: 8000, base_fee_per_blob_gas: 4000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Should send because multiple short-term fees are lower" + )] +#[test_case( + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, + 6, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Should not send because all fees are identical and no tolerance" + )] +#[test_case( + // Zero blobs scenario: blob fee differences don't matter + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, + 0, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: short-term base_fee_per_gas and reward are lower, send" + )] +#[test_case( + // Zero blobs but short-term reward is higher + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, + 0, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + false; + "Zero blobs: short-term reward is higher, don't send" + )] +#[test_case( + // Zero blobs don't care about higher short-term base_fee_per_blob_gas + Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, + Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, + 0, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + } + }, + 0, + true; + "Zero blobs: ignore blob fee, short-term base_fee_per_gas is lower, send" + )] +// Initially not send, but as num_l2_blocks_behind increases, acceptance grows. +#[test_case( + // Initially short-term fee too high compared to long-term (strict scenario), no send at t=0 + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: Percentage::try_from(0.20).unwrap(), + end_premium_percentage: Percentage::try_from(0.20).unwrap(), + always_acceptable_fee: 0, + }, + }, + 0, + false; + "Early: short-term expensive, not send" + )] +#[test_case( + // At max_l2_blocks_behind, send regardless + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6}, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + } + }, + 100, + true; + "Later: after max wait, send regardless" + )] +#[test_case( + Fees { base_fee_per_gas: 6000, reward: 0, base_fee_per_blob_gas: 6000 }, + Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 0, + }, + }, + 80, + true; + "Mid-wait: increased tolerance allows acceptance" + )] +#[test_case( + // Short-term fee is huge, but always_acceptable_fee is large, so send immediately + Fees { base_fee_per_gas: 100_000, reward: 0, base_fee_per_blob_gas: 100_000 }, + Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, + 1, + Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 100.try_into().unwrap(), + start_discount_percentage: 0.20.try_into().unwrap(), + end_premium_percentage: 0.20.try_into().unwrap(), + always_acceptable_fee: 2_700_000_000_000 + }, + }, + 0, + true; + "Always acceptable fee triggers immediate send" + )] +#[tokio::test] +async fn parameterized_send_or_wait_tests( + old_fees: Fees, + new_fees: Fees, + num_blobs: u32, + config: Config, + num_l2_blocks_behind: u32, + expected_decision: bool, +) { + let fees = generate_fees(config, old_fees, new_fees); + let fees_provider = PreconfiguredFeeApi::new(fees); + let current_block_height = fees_provider.current_height().await.unwrap(); + + let sut = FeeTracker::new(fees_provider, config); + + let should_send = sut + .should_send_blob_tx(num_blobs, num_l2_blocks_behind, current_block_height) + .await + .unwrap(); + + assert_eq!( + should_send, expected_decision, + "For num_blobs={num_blobs}, num_l2_blocks_behind={num_l2_blocks_behind}, config={config:?}: Expected decision: {expected_decision}, got: {should_send}", + ); +} + +#[tokio::test] +async fn test_send_when_too_far_behind_and_fee_provider_fails() { + // given + let config = Config { + sma_periods: SmaPeriods { short: 2, long: 6 }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: 10.try_into().unwrap(), + always_acceptable_fee: 0, + ..Default::default() + }, + }; + + // having no fees will make the validation in fee analytics fail + let fee_provider = PreconfiguredFeeApi::new(vec![]); + let sut = FeeTracker::new(fee_provider, config); + + // when + let should_send = sut + .should_send_blob_tx(1, 20, 100) + .await + .expect("Should send despite fee provider failure"); + + // then + assert!( + should_send, + "Should send because too far behind, regardless of fee provider status" + ); +} diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index 4ada71e7..b02946a5 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,12 +1,13 @@ use services::{ - state_committer::{ - port::l1::{testing::ApiMockWFees, Fees}, - service::{FeeAlgoConfig, FeeThresholds, SmaPeriods}, + fee_tracker::{ + port::l1::{testing::PreconfiguredFeeApi, Fees}, + service::{Config as FeeTrackerConfig, FeeThresholds, FeeTracker, SmaPeriods}, }, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; use std::time::Duration; +use test_helpers::noop_fee_tracker; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -30,7 +31,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -40,6 +41,7 @@ async fn submits_fragments_when_required_count_accumulated() -> Result<()> { ..Default::default() }, setup.test_clock(), + noop_fee_tracker(), ); // when @@ -73,7 +75,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -83,6 +85,7 @@ async fn submits_fragments_on_timeout_before_accumulation() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time beyond the timeout @@ -108,7 +111,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -118,6 +121,7 @@ async fn does_not_submit_fragments_before_required_count_or_timeout() -> Result< ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time less than the timeout @@ -153,7 +157,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -163,6 +167,7 @@ async fn submits_fragments_when_required_count_before_timeout() -> Result<()> { ..Default::default() }, setup.test_clock(), + noop_fee_tracker(), ); // when @@ -199,7 +204,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(1); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -209,6 +214,7 @@ async fn timeout_measured_from_last_finalized_fragment() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time to exceed the timeout since last finalized fragment @@ -245,7 +251,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> .expect_current_height() .returning(|| Box::pin(async { Ok(1) })); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -255,6 +261,7 @@ async fn timeout_measured_from_startup_if_no_finalized_fragment() -> Result<()> ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Advance time beyond the timeout from startup @@ -303,7 +310,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(0); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { @@ -314,6 +321,7 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Submit the initial fragments @@ -387,7 +395,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_algo_config = services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -415,17 +423,17 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), ); // When @@ -493,7 +501,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_algo_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -512,17 +520,17 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( let fuel_mock = test_helpers::mocks::fuel::latest_height_is(6); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock).w_preconfigured_fees(fee_sequence), + l1_mock, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), ); // when @@ -590,7 +598,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_algo_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), @@ -618,17 +626,17 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { let fuel_mock = test_helpers::mocks::fuel::latest_height_is(50); // L2 height is 50, behind by 50 let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), ); // when @@ -695,7 +703,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ), ]; - let fee_algo_config = FeeAlgoConfig { + let fee_tracker_config = services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: 2, long: 5 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -724,17 +732,17 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran let fuel_mock = test_helpers::mocks::fuel::latest_height_is(80); let mut state_committer = StateCommitter::new( - ApiMockWFees::new(l1_mock_submit).w_preconfigured_fees(fee_sequence), + l1_mock_submit, fuel_mock, setup.db(), StateCommitterConfig { lookback_window: 1000, fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 6.try_into().unwrap(), - fee_algo: fee_algo_config, ..Default::default() }, setup.test_clock(), + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_tracker_config), ); // when diff --git a/packages/services/tests/state_listener.rs b/packages/services/tests/state_listener.rs index 20ac4b38..5777b8da 100644 --- a/packages/services/tests/state_listener.rs +++ b/packages/services/tests/state_listener.rs @@ -3,13 +3,15 @@ use std::time::Duration; use metrics::prometheus::IntGauge; use mockall::predicate::eq; use services::{ - state_committer::port::l1::testing::ApiMockWFees, state_listener::{port::Storage, service::StateListener}, types::{L1Height, L1Tx, TransactionResponse}, Result, Runner, StateCommitter, StateCommitterConfig, }; use test_case::test_case; -use test_helpers::mocks::{self, l1::TxStatus}; +use test_helpers::{ + mocks::{self, l1::TxStatus}, + noop_fee_tracker, +}; #[tokio::test] async fn successful_finalized_tx() -> Result<()> { @@ -446,7 +448,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), StateCommitterConfig { @@ -454,6 +456,7 @@ async fn block_inclusion_of_replacement_leaves_no_pending_txs() -> Result<()> { ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Orig tx @@ -549,7 +552,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( .returning(|| Box::pin(async { Ok(0) })); let mut committer = StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, mocks::fuel::latest_height_is(0), setup.db(), crate::StateCommitterConfig { @@ -557,6 +560,7 @@ async fn finalized_replacement_tx_will_leave_no_pending_tx( ..Default::default() }, test_clock.clone(), + noop_fee_tracker(), ); // Orig tx diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index 3f8c7cdc..f7f35683 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,9 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::state_committer::port::l1::testing::ApiMockWFees; +use services::fee_tracker::port::l1::testing::ConstantFeeApi; +use services::fee_tracker::port::l1::Fees; +use services::fee_tracker::service::FeeTracker; use services::types::{ BlockSubmission, CollectNonEmpty, CompressedFuelBlock, Fragment, L1Tx, NonEmpty, }; @@ -485,6 +487,10 @@ pub mod mocks { } } +pub fn noop_fee_tracker() -> FeeTracker { + FeeTracker::new(ConstantFeeApi::new(Fees::default()), Default::default()) +} + pub struct Setup { db: DbWithProcess, test_clock: TestClock, @@ -545,7 +551,7 @@ impl Setup { .return_once(move || Box::pin(async { Ok(0) })); StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, mocks::fuel::latest_height_is(0), self.db(), services::StateCommitterConfig { @@ -556,6 +562,7 @@ impl Setup { ..Default::default() }, self.test_clock.clone(), + noop_fee_tracker(), ) .run() .await @@ -583,7 +590,7 @@ impl Setup { let fuel_mock = mocks::fuel::latest_height_is(height); let mut committer = StateCommitter::new( - ApiMockWFees::new(l1_mock), + l1_mock, fuel_mock, self.db(), services::StateCommitterConfig { @@ -591,9 +598,9 @@ impl Setup { fragment_accumulation_timeout: Duration::from_secs(0), fragments_to_accumulate: 1.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(300), - ..Default::default() }, self.test_clock.clone(), + noop_fee_tracker(), ); committer.run().await.unwrap(); From bfc5eb01571c25d1c07176a0d895152ee15bbb22 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:43:47 +0100 Subject: [PATCH 38/47] add helper to lessen test boilerplate --- packages/services/tests/state_committer.rs | 21 ++++++++++----------- packages/test-helpers/src/lib.rs | 9 ++++++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index b02946a5..ab903e1c 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -1,13 +1,13 @@ use services::{ fee_tracker::{ - port::l1::{testing::PreconfiguredFeeApi, Fees}, - service::{Config as FeeTrackerConfig, FeeThresholds, FeeTracker, SmaPeriods}, + port::l1::Fees, + service::{Config as FeeTrackerConfig, FeeThresholds, SmaPeriods}, }, types::{L1Tx, NonEmpty}, Result, Runner, StateCommitter, StateCommitterConfig, }; use std::time::Duration; -use test_helpers::noop_fee_tracker; +use test_helpers::{noop_fee_tracker, preconfigured_fee_tracker}; #[tokio::test] async fn submits_fragments_when_required_count_accumulated() -> Result<()> { @@ -318,7 +318,6 @@ async fn resubmits_fragments_when_gas_bump_timeout_exceeded() -> Result<()> { fragment_accumulation_timeout: Duration::from_secs(60), fragments_to_accumulate: 5.try_into().unwrap(), gas_bump_timeout: Duration::from_secs(60), - ..Default::default() }, test_clock.clone(), noop_fee_tracker(), @@ -395,7 +394,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ), ]; - let fee_algo_config = services::fee_tracker::service::Config { + let fee_tracker_config = services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -433,7 +432,7 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // When @@ -501,7 +500,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ), ]; - let fee_algo_config = FeeTrackerConfig { + let fee_tracker_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), @@ -530,7 +529,7 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // when @@ -598,7 +597,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ), ]; - let fee_algo_config = FeeTrackerConfig { + let fee_tracker_config = FeeTrackerConfig { sma_periods: SmaPeriods { short: 2, long: 6 }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), @@ -636,7 +635,7 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_algo_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // when @@ -742,7 +741,7 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ..Default::default() }, setup.test_clock(), - FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), fee_tracker_config), + preconfigured_fee_tracker(fee_sequence, fee_tracker_config), ); // when diff --git a/packages/test-helpers/src/lib.rs b/packages/test-helpers/src/lib.rs index f7f35683..2e0f6a72 100644 --- a/packages/test-helpers/src/lib.rs +++ b/packages/test-helpers/src/lib.rs @@ -8,7 +8,7 @@ use fuel_block_committer_encoding::bundle::{self, CompressionLevel}; use metrics::prometheus::IntGauge; use mocks::l1::TxStatus; use rand::{Rng, RngCore}; -use services::fee_tracker::port::l1::testing::ConstantFeeApi; +use services::fee_tracker::port::l1::testing::{ConstantFeeApi, PreconfiguredFeeApi}; use services::fee_tracker::port::l1::Fees; use services::fee_tracker::service::FeeTracker; use services::types::{ @@ -491,6 +491,13 @@ pub fn noop_fee_tracker() -> FeeTracker { FeeTracker::new(ConstantFeeApi::new(Fees::default()), Default::default()) } +pub fn preconfigured_fee_tracker( + fee_sequence: impl IntoIterator, + config: services::fee_tracker::service::Config, +) -> FeeTracker { + FeeTracker::new(PreconfiguredFeeApi::new(fee_sequence), config) +} + pub struct Setup { db: DbWithProcess, test_clock: TestClock, From d39838e9dfbdbfe84c06d61bb1c417545d6079d9 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:45:36 +0100 Subject: [PATCH 39/47] remove context --- packages/services/src/fee_tracker/service.rs | 41 ++++---------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 8224c003..b99b7a4b 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -249,11 +249,6 @@ mod tests { use super::*; use test_case::test_case; - struct Context { - num_l2_blocks_behind: u32, - at_l1_height: u64, - } - #[test_case( // Test Case 1: No blocks behind, no discount or premium FeeThresholds { @@ -262,10 +257,7 @@ mod tests { ..Default::default() }, 1000, - Context { - num_l2_blocks_behind: 0, - at_l1_height: 0, - }, + 0, 1000; "No blocks behind, multiplier should be 100%" )] @@ -277,10 +269,7 @@ mod tests { always_acceptable_fee: 0, }, 2000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, + 50, 2050; "Half blocks behind with discount and premium" )] @@ -292,10 +281,7 @@ mod tests { ..Default::default() }, 800, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, + 50, 700; "Start discount only, no premium" )] @@ -307,10 +293,7 @@ mod tests { ..Default::default() }, 1000, - Context { - num_l2_blocks_behind: 50, - at_l1_height: 0, - }, + 50, 1150; "End premium only, no discount" )] @@ -323,10 +306,7 @@ mod tests { always_acceptable_fee: 0, }, 10_000, - Context { - num_l2_blocks_behind: 99, - at_l1_height: 0, - }, + 99, 11970; "High fee with premium" )] @@ -338,23 +318,16 @@ mod tests { always_acceptable_fee: 0, }, 1000, - Context { - num_l2_blocks_behind: 1, - at_l1_height: 0, - }, + 1, 12; "Discount exceeds 100%, should be capped to 100%" )] fn test_calculate_max_upper_fee( fee_thresholds: FeeThresholds, fee: u128, - context: Context, + num_l2_blocks_behind: u32, expected_max_upper_fee: u128, ) { - let Context { - num_l2_blocks_behind, - at_l1_height, - } = context; let max_upper_fee = FeeTracker::::calculate_max_upper_fee( &fee_thresholds, fee, From f908f3237bdf1218dbbabf936d80d34c8dbf2859 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 14:47:54 +0100 Subject: [PATCH 40/47] cargo fix --- packages/services/src/fee_tracker/fee_analytics.rs | 2 -- packages/services/src/state_committer.rs | 9 ++------- packages/services/tests/fee_tracker.rs | 1 - 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index ce7a8648..9052a578 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -65,7 +65,6 @@ impl FeeAnalytics

{ #[cfg(test)] mod tests { use itertools::Itertools; - use mockall::{predicate::eq, Sequence}; use crate::fee_tracker::port::l1::{testing, BlockFees}; @@ -199,7 +198,6 @@ mod tests { "Expected iterator to yield the same block fees" ); } - use std::path::PathBuf; #[tokio::test] async fn calculates_sma_correctly_for_last_1_block() { diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index 35b4e5e6..f66e4d5a 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -1,13 +1,9 @@ pub mod service { - use std::{ - num::{NonZeroU32, NonZeroUsize}, - time::Duration, - }; + use std::{num::NonZeroUsize, time::Duration}; use crate::{ - fee_tracker::service::FeeTracker, types::{storage::BundleFragment, CollectNonEmpty, DateTime, L1Tx, NonEmpty, Utc}, - Error, Result, Runner, + Result, Runner, }; use itertools::Itertools; use tracing::info; @@ -302,7 +298,6 @@ pub mod port { }; pub mod l1 { - use std::ops::RangeInclusive; use nonempty::NonEmpty; diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index c8f5f0ed..1ee2c820 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -1,4 +1,3 @@ -use services::fee_tracker::port::l1::testing::ConstantFeeApi; use services::fee_tracker::port::l1::testing::PreconfiguredFeeApi; use services::fee_tracker::port::l1::Api; use services::fee_tracker::service::FeeThresholds; From d87d5cb4f0a34f03b2b4c46495bef392ac02d6a1 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 15:59:42 +0100 Subject: [PATCH 41/47] add fee tracker to be run periodically --- committer/src/config.rs | 9 +- committer/src/main.rs | 10 ++ committer/src/setup.rs | 62 +++++--- e2e/src/committer.rs | 1 + .../services/src/fee_tracker/fee_analytics.rs | 25 ++-- packages/services/src/fee_tracker/service.rs | 132 ++++++++++++++++-- packages/services/src/fee_tracker/testing.rs | 2 - packages/services/src/state_committer.rs | 33 +++++ packages/services/tests/fee_tracker.rs | 45 +++--- packages/services/tests/state_committer.rs | 20 ++- 10 files changed, 266 insertions(+), 73 deletions(-) delete mode 100644 packages/services/src/fee_tracker/testing.rs diff --git a/committer/src/config.rs b/committer/src/config.rs index 5b3b6b03..eca2d76e 100644 --- a/committer/src/config.rs +++ b/committer/src/config.rs @@ -1,6 +1,6 @@ use std::{ net::Ipv4Addr, - num::{NonZeroU32, NonZeroUsize}, + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, str::FromStr, time::Duration, }; @@ -93,6 +93,9 @@ pub struct App { /// How often to check for finalized l1 txs #[serde(deserialize_with = "human_readable_duration")] pub tx_finalization_check_interval: Duration, + /// How often to check for l1 prices + #[serde(deserialize_with = "human_readable_duration")] + pub l1_prices_check_interval: Duration, /// Number of L1 blocks that need to pass to accept the tx as finalized pub num_blocks_to_finalize_tx: u64, /// Interval after which to bump a pending tx @@ -119,10 +122,10 @@ pub struct App { #[derive(Debug, Clone, Deserialize)] pub struct FeeAlgoConfig { /// Short-term period for Simple Moving Average (SMA) in block numbers - pub short_sma_blocks: u64, + pub short_sma_blocks: NonZeroU64, /// Long-term period for Simple Moving Average (SMA) in block numbers - pub long_sma_blocks: u64, + pub long_sma_blocks: NonZeroU64, /// Maximum number of unposted L2 blocks before sending a transaction regardless of fees pub max_l2_blocks_behind: NonZeroU32, diff --git a/committer/src/main.rs b/committer/src/main.rs index 5810df75..c2662738 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -72,12 +72,21 @@ async fn main() -> Result<()> { &metrics_registry, ); + let (fee_tracker, fee_tracker_handle) = setup::fee_tracker( + ethereum_rpc.clone(), + cancel_token.clone(), + &config, + &metrics_registry, + )?; + let state_committer_handle = setup::state_committer( fuel_adapter.clone(), ethereum_rpc.clone(), storage.clone(), cancel_token.clone(), &config, + &metrics_registry, + fee_tracker, )?; let state_importer_handle = @@ -104,6 +113,7 @@ async fn main() -> Result<()> { handles.push(block_bundler); handles.push(state_listener_handle); handles.push(state_pruner_handle); + handles.push(fee_tracker_handle); } launch_api_server( diff --git a/committer/src/setup.rs b/committer/src/setup.rs index fac00a21..0cfe9c43 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -118,27 +118,9 @@ pub fn state_committer( storage: Database, cancel_token: CancellationToken, config: &config::Config, + registry: &Registry, + fee_tracker: FeeTracker, ) -> Result> { - let fee_tracker = FeeTracker::new( - l1.clone(), - services::fee_tracker::service::Config { - sma_periods: SmaPeriods { - short: config.app.fee_algo.short_sma_blocks, - long: config.app.fee_algo.long_sma_blocks, - }, - fee_thresholds: FeeThresholds { - max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, - start_discount_percentage: config - .app - .fee_algo - .start_discount_percentage - .try_into()?, - end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, - always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, - }, - }, - ); - let state_committer = services::StateCommitter::new( l1, fuel, @@ -153,6 +135,8 @@ pub fn state_committer( fee_tracker, ); + state_committer.register_metrics(registry); + Ok(schedule_polling( config.app.tx_finalization_check_interval, state_committer, @@ -337,3 +321,41 @@ pub async fn shut_down( storage.close().await; Ok(()) } + +pub fn fee_tracker( + l1: L1, + cancel_token: CancellationToken, + config: &config::Config, + registry: &Registry, +) -> Result<(FeeTracker, tokio::task::JoinHandle<()>)> { + let fee_tracker = FeeTracker::new( + l1, + services::fee_tracker::service::Config { + sma_periods: SmaPeriods { + short: config.app.fee_algo.short_sma_blocks, + long: config.app.fee_algo.long_sma_blocks, + }, + fee_thresholds: FeeThresholds { + max_l2_blocks_behind: config.app.fee_algo.max_l2_blocks_behind, + start_discount_percentage: config + .app + .fee_algo + .start_discount_percentage + .try_into()?, + end_premium_percentage: config.app.fee_algo.end_premium_percentage.try_into()?, + always_acceptable_fee: config.app.fee_algo.always_acceptable_fee as u128, + }, + }, + ); + + fee_tracker.register_metrics(registry); + + let handle = schedule_polling( + config.app.tx_finalization_check_interval, + fee_tracker.clone(), + "Fee Tracker", + cancel_token, + ); + + Ok((fee_tracker, handle)) +} diff --git a/e2e/src/committer.rs b/e2e/src/committer.rs index 4cdc21e2..e6a6f171 100644 --- a/e2e/src/committer.rs +++ b/e2e/src/committer.rs @@ -73,6 +73,7 @@ impl Committer { .env("COMMITTER__APP__HOST", "127.0.0.1") .env("COMMITTER__APP__BLOCK_CHECK_INTERVAL", "5s") .env("COMMITTER__APP__TX_FINALIZATION_CHECK_INTERVAL", "5s") + .env("COMMITTER__APP__L1_PRICES_CHECK_INTERVAL", "5s") .env("COMMITTER__APP__NUM_BLOCKS_TO_FINALIZE_TX", "3") .env("COMMITTER__APP__GAS_BUMP_TIMEOUT", "300s") .env("COMMITTER__APP__TX_MAX_FEE", "4000000000000000") diff --git a/packages/services/src/fee_tracker/fee_analytics.rs b/packages/services/src/fee_tracker/fee_analytics.rs index 9052a578..1f0d8144 100644 --- a/packages/services/src/fee_tracker/fee_analytics.rs +++ b/packages/services/src/fee_tracker/fee_analytics.rs @@ -2,7 +2,7 @@ use std::ops::RangeInclusive; use crate::Error; -use super::port::l1::{Api, Fees, SequentialBlockFees}; +use super::port::l1::{Api, BlockFees, Fees, SequentialBlockFees}; #[derive(Debug, Clone)] pub struct FeeAnalytics

{ @@ -29,7 +29,9 @@ impl FeeAnalytics

{ Ok(Self::mean(fees)) } - pub async fn fees_at_height(&self, height: u64) -> crate::Result { + pub async fn latest_fees(&self) -> crate::Result { + let height = self.fees_provider.current_height().await?; + let fee = self .fees_provider .fees(height..=height) @@ -38,7 +40,7 @@ impl FeeAnalytics

{ .next() .expect("sequential fees guaranteed not empty"); - Ok(fee.fees) + Ok(fee) } fn mean(fees: SequentialBlockFees) -> Fees { @@ -252,21 +254,24 @@ mod tests { } #[tokio::test] - async fn price_at_height_returns_correct_fee() { + async fn latest_fees_on_fee_analytics() { // given let fees_map = testing::incrementing_fees(5); let fees_provider = testing::PreconfiguredFeeApi::new(fees_map.clone()); let fee_analytics = FeeAnalytics::new(fees_provider); - let height = 2; + let height = 4; // when - let fee = fee_analytics.fees_at_height(height).await.unwrap(); + let fee = fee_analytics.latest_fees().await.unwrap(); // then - let expected_fee = Fees { - base_fee_per_gas: 3, - reward: 3, - base_fee_per_blob_gas: 3, + let expected_fee = BlockFees { + height, + fees: Fees { + base_fee_per_gas: 5, + reward: 5, + base_fee_per_blob_gas: 5, + }, }; assert_eq!( fee, expected_fee, diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index b99b7a4b..8cd176a0 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -1,12 +1,20 @@ -use std::{cmp::min, num::NonZeroU32}; +use std::{ + cmp::min, + num::{NonZeroU32, NonZeroU64}, + ops::RangeInclusive, +}; +use metrics::{ + prometheus::{core::Collector, IntGauge, Opts}, + RegistersMetrics, +}; use tracing::info; -use crate::{state_committer::service::SendOrWaitDecider, Error}; +use crate::{state_committer::service::SendOrWaitDecider, Error, Result, Runner}; use super::{ fee_analytics::FeeAnalytics, - port::l1::{Api, Fees}, + port::l1::{Api, BlockFees, Fees}, }; #[derive(Debug, Clone, Copy)] @@ -19,7 +27,10 @@ pub struct Config { impl Default for Config { fn default() -> Self { Config { - sma_periods: SmaPeriods { short: 1, long: 2 }, + sma_periods: SmaPeriods { + short: 1.try_into().expect("not zero"), + long: 2.try_into().expect("not zero"), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), ..FeeThresholds::default() @@ -30,8 +41,8 @@ impl Default for Config { #[derive(Debug, Clone, Copy)] pub struct SmaPeriods { - pub short: u64, - pub long: u64, + pub short: NonZeroU64, + pub long: NonZeroU64, } #[derive(Debug, Clone, Copy)] @@ -60,7 +71,7 @@ impl SendOrWaitDecider for FeeTracker

{ num_blobs: u32, num_l2_blocks_behind: u32, at_l1_height: u64, - ) -> crate::Result { + ) -> Result { if self.too_far_behind(num_l2_blocks_behind) { info!("Sending because we've fallen behind by {} which is more than the configured maximum of {}", num_l2_blocks_behind, self.config.fee_thresholds.max_l2_blocks_behind); return Ok(true); @@ -68,8 +79,7 @@ impl SendOrWaitDecider for FeeTracker

{ // opted out of validating that num_blobs <= 6, it's not this fn's problem if the caller // wants to send more than 6 blobs - let last_n_blocks = - |n: u64| at_l1_height.saturating_sub(n.saturating_sub(1))..=at_l1_height; + let last_n_blocks = |n| Self::last_n_blocks(at_l1_height, n); let short_term_sma = self .fee_analytics @@ -81,14 +91,14 @@ impl SendOrWaitDecider for FeeTracker

{ .calculate_sma(last_n_blocks(self.config.sma_periods.long)) .await?; - let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, short_term_sma); + let short_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, &short_term_sma); if self.fee_always_acceptable(short_term_tx_fee) { info!("Sending because: short term price {} is deemed always acceptable since it is <= {}", short_term_tx_fee, self.config.fee_thresholds.always_acceptable_fee); return Ok(true); } - let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, long_term_sma); + let long_term_tx_fee = Self::calculate_blob_tx_fee(num_blobs, &long_term_sma); let max_upper_tx_fee = Self::calculate_max_upper_fee( &self.config.fee_thresholds, long_term_tx_fee, @@ -143,9 +153,56 @@ impl Percentage { } } +#[derive(Debug, Clone)] +struct Metrics { + current_blob_tx_fee: IntGauge, + short_term_blob_tx_fee: IntGauge, + long_term_blob_tx_fee: IntGauge, +} + +impl Default for Metrics { + fn default() -> Self { + let current_blob_tx_fee = IntGauge::with_opts(Opts::new( + "current_blob_tx_fee", + "The current fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + let short_term_blob_tx_fee = IntGauge::with_opts(Opts::new( + "short_term_blob_tx_fee", + "The short term fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + let long_term_blob_tx_fee = IntGauge::with_opts(Opts::new( + "long_term_blob_tx_fee", + "The long term fee for a transaction with 6 blobs", + )) + .expect("metric config to be correct"); + + Self { + current_blob_tx_fee, + short_term_blob_tx_fee, + long_term_blob_tx_fee, + } + } +} + +impl

RegistersMetrics for FeeTracker

{ + fn metrics(&self) -> Vec> { + vec![ + Box::new(self.metrics.current_blob_tx_fee.clone()), + Box::new(self.metrics.short_term_blob_tx_fee.clone()), + Box::new(self.metrics.long_term_blob_tx_fee.clone()), + ] + } +} + +#[derive(Clone)] pub struct FeeTracker

{ fee_analytics: FeeAnalytics

, config: Config, + metrics: Metrics, } impl FeeTracker

{ @@ -167,7 +224,7 @@ impl FeeTracker

{ debug_assert!( blocks_behind <= max_blocks_behind, - "blocks_behind ({}) should not exceed max_blocks_behind ({})", + "blocks_behind ({}) should not exceed max_blocks_behind ({}), it should have been handled earlier", blocks_behind, max_blocks_behind ); @@ -222,7 +279,7 @@ impl FeeTracker

{ } // TODO: Segfault maybe dont leak so much eth abstractions - fn calculate_blob_tx_fee(num_blobs: u32, fees: Fees) -> u128 { + fn calculate_blob_tx_fee(num_blobs: u32, fees: &Fees) -> u128 { const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; @@ -231,6 +288,44 @@ impl FeeTracker

{ base_fee + blob_fee + fees.reward } + + fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { + current_block.saturating_sub(n.get().saturating_sub(1))..=current_block + } + + pub async fn update_metrics(&self) -> Result<()> { + let latest_fees = self.fee_analytics.latest_fees().await?; + let short_term_sma = self + .fee_analytics + .calculate_sma(Self::last_n_blocks( + latest_fees.height, + self.config.sma_periods.short, + )) + .await?; + + let long_term_sma = self + .fee_analytics + .calculate_sma(Self::last_n_blocks( + latest_fees.height, + self.config.sma_periods.long, + )) + .await?; + + let calc_fee = + |fees: &Fees| i64::try_from(Self::calculate_blob_tx_fee(6, fees)).unwrap_or(i64::MAX); + + self.metrics + .current_blob_tx_fee + .set(calc_fee(&latest_fees.fees)); + self.metrics + .short_term_blob_tx_fee + .set(calc_fee(&short_term_sma)); + self.metrics + .long_term_blob_tx_fee + .set(calc_fee(&long_term_sma)); + + Ok(()) + } } impl

FeeTracker

{ @@ -238,10 +333,21 @@ impl

FeeTracker

{ Self { fee_analytics: FeeAnalytics::new(fee_provider), config, + metrics: Metrics::default(), } } } +impl

Runner for FeeTracker

+where + P: crate::fee_tracker::port::l1::Api + Send + Sync, +{ + async fn run(&mut self) -> Result<()> { + self.update_metrics().await?; + Ok(()) + } +} + #[cfg(test)] mod tests { use crate::fee_tracker::port::l1::testing::ConstantFeeApi; diff --git a/packages/services/src/fee_tracker/testing.rs b/packages/services/src/fee_tracker/testing.rs deleted file mode 100644 index 139597f9..00000000 --- a/packages/services/src/fee_tracker/testing.rs +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index f66e4d5a..caa47636 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -6,6 +6,10 @@ pub mod service { Result, Runner, }; use itertools::Itertools; + use metrics::{ + prometheus::{core::Collector, IntGauge, Opts}, + RegistersMetrics, + }; use tracing::info; // src/config.rs @@ -41,6 +45,29 @@ pub mod service { ) -> Result; } + struct Metrics { + num_l2_blocks_behind: IntGauge, + } + + impl Default for Metrics { + fn default() -> Self { + let num_l2_blocks_behind = IntGauge::with_opts(Opts::new( + "num_l2_blocks_behind", + "How many L2 blocks have been produced since the starting height of the oldest bundle we're committing", + )).expect("metric config to be correct"); + + Self { + num_l2_blocks_behind, + } + } + } + + impl RegistersMetrics for StateCommitter { + fn metrics(&self) -> Vec> { + vec![Box::new(self.metrics.num_l2_blocks_behind.clone())] + } + } + /// The `StateCommitter` is responsible for committing state fragments to L1. pub struct StateCommitter { l1_adapter: L1, @@ -50,6 +77,7 @@ pub mod service { clock: Clock, startup_time: DateTime, decider: D, + metrics: Metrics, } impl StateCommitter @@ -75,6 +103,7 @@ pub mod service { clock, startup_time, decider, + metrics: Default::default(), } } } @@ -114,6 +143,10 @@ pub mod service { let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); + self.metrics + .num_l2_blocks_behind + .set(num_l2_blocks_behind as i64); + self.decider .should_send_blob_tx( u32::try_from(fragments.len()).expect("not to send more than u32::MAX blobs"), diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 1ee2c820..7b92c0c1 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -11,9 +11,9 @@ use test_case::test_case; fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fees)> { let older_fees = std::iter::repeat_n( old_fees, - (config.sma_periods.long - config.sma_periods.short) as usize, + (config.sma_periods.long.get() - config.sma_periods.short.get()) as usize, ); - let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short as usize); + let newer_fees = std::iter::repeat_n(new_fees, config.sma_periods.short.get() as usize); older_fees .chain(newer_fees) @@ -27,7 +27,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 3000, reward: 3000, base_fee_per_blob_gas: 3000 }, 6, Config { - sma_periods: services::fee_tracker::service::SmaPeriods { short: 2, long: 6 }, + sma_periods: services::fee_tracker::service::SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -43,7 +43,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -59,7 +59,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, max_l2_blocks_behind: 100.try_into().unwrap(), @@ -75,7 +75,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 1500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -91,7 +91,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2500, reward: 10000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -107,7 +107,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 900 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -123,7 +123,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 3000, base_fee_per_blob_gas: 1100 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -139,7 +139,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 9000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -155,7 +155,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 11000, base_fee_per_blob_gas: 1000 }, 5, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -172,7 +172,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 3500 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -188,7 +188,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 5000, reward: 5000, base_fee_per_blob_gas: 5000 }, 6, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -205,7 +205,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2500, reward: 5500, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -222,7 +222,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 3000, reward: 7000, base_fee_per_blob_gas: 5000 }, 0, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -239,7 +239,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -257,7 +257,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: Percentage::try_from(0.20).unwrap(), @@ -275,7 +275,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6}, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), @@ -292,7 +292,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 7000, reward: 0, base_fee_per_blob_gas: 7000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), @@ -310,7 +310,7 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Fees { base_fee_per_gas: 2_000_000, reward: 1_000_000, base_fee_per_blob_gas: 20_000_000 }, 1, Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), @@ -352,7 +352,10 @@ async fn parameterized_send_or_wait_tests( async fn test_send_when_too_far_behind_and_fee_provider_fails() { // given let config = Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 10.try_into().unwrap(), always_acceptable_fee: 0, diff --git a/packages/services/tests/state_committer.rs b/packages/services/tests/state_committer.rs index ab903e1c..345fe363 100644 --- a/packages/services/tests/state_committer.rs +++ b/packages/services/tests/state_committer.rs @@ -395,7 +395,10 @@ async fn sends_transaction_when_short_term_fee_favorable() -> Result<()> { ]; let fee_tracker_config = services::fee_tracker::service::Config { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -501,7 +504,10 @@ async fn does_not_send_transaction_when_short_term_fee_unfavorable() -> Result<( ]; let fee_tracker_config = FeeTrackerConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), always_acceptable_fee: 0, @@ -598,7 +604,10 @@ async fn sends_transaction_when_l2_blocks_behind_exceeds_max() -> Result<()> { ]; let fee_tracker_config = FeeTrackerConfig { - sma_periods: SmaPeriods { short: 2, long: 6 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 6.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 50.try_into().unwrap(), always_acceptable_fee: 0, @@ -703,7 +712,10 @@ async fn sends_transaction_when_nearing_max_blocks_behind_with_increased_toleran ]; let fee_tracker_config = services::fee_tracker::service::Config { - sma_periods: SmaPeriods { short: 2, long: 5 }, + sma_periods: SmaPeriods { + short: 2.try_into().unwrap(), + long: 5.try_into().unwrap(), + }, fee_thresholds: FeeThresholds { max_l2_blocks_behind: 100.try_into().unwrap(), start_discount_percentage: 0.20.try_into().unwrap(), From ebb846467a4f60616dff60b625fde4fc554fa375 Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Wed, 18 Dec 2024 18:35:56 +0100 Subject: [PATCH 42/47] revert debug logs --- e2e/src/whole_stack.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/src/whole_stack.rs b/e2e/src/whole_stack.rs index 1a3e257d..59e8e779 100644 --- a/e2e/src/whole_stack.rs +++ b/e2e/src/whole_stack.rs @@ -60,7 +60,7 @@ impl WholeStack { let db = start_db().await?; let committer = start_committer( - true, + logs, blob_support, db.clone(), ð_node, From 06755d7f7c4ffbadd5026a022dfd2332cb21189f Mon Sep 17 00:00:00 2001 From: rymnc <43716372+rymnc@users.noreply.github.com> Date: Wed, 18 Dec 2024 00:22:11 +0530 Subject: [PATCH 43/47] test(costs): get latest costs from api fix: fmt chore: add test, incl finalized only fix: serde for query string test: more shenanigans with query string test: more shenanigans with query string fix: option error handling --- ...8f69f2298cd5b0f7eb1609ed189269c6f677c.json | 58 +++++++++++++++++++ committer/src/api.rs | 24 +++++++- packages/adapters/storage/src/lib.rs | 38 ++++++++++++ packages/adapters/storage/src/postgres.rs | 30 ++++++++++ .../adapters/storage/src/test_instance.rs | 15 +++-- packages/services/src/cost_reporter.rs | 13 +++++ 6 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 .sqlx/query-39d3fae6fdd67a2324fae4d5e828f69f2298cd5b0f7eb1609ed189269c6f677c.json diff --git a/.sqlx/query-39d3fae6fdd67a2324fae4d5e828f69f2298cd5b0f7eb1609ed189269c6f677c.json b/.sqlx/query-39d3fae6fdd67a2324fae4d5e828f69f2298cd5b0f7eb1609ed189269c6f677c.json new file mode 100644 index 00000000..abd5fc53 --- /dev/null +++ b/.sqlx/query-39d3fae6fdd67a2324fae4d5e828f69f2298cd5b0f7eb1609ed189269c6f677c.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n bc.bundle_id,\n bc.cost,\n bc.size,\n bc.da_block_height,\n bc.is_finalized,\n b.start_height,\n b.end_height\n FROM\n bundle_cost bc\n JOIN bundles b ON bc.bundle_id = b.id\n WHERE\n bc.is_finalized = TRUE\n ORDER BY\n b.start_height DESC\n LIMIT $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bundle_id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "cost", + "type_info": "Numeric" + }, + { + "ordinal": 2, + "name": "size", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "da_block_height", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "is_finalized", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "start_height", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "end_height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "39d3fae6fdd67a2324fae4d5e828f69f2298cd5b0f7eb1609ed189269c6f677c" +} diff --git a/committer/src/api.rs b/committer/src/api.rs index cd97a59c..5393ba0a 100644 --- a/committer/src/api.rs +++ b/committer/src/api.rs @@ -90,9 +90,17 @@ async fn metrics(registry: web::Data>) -> impl Responder { std::result::Result::<_, InternalError<_>>::Ok(text) } +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum HeightVariant { + Latest, + Specific, +} + #[derive(Deserialize)] struct CostQueryParams { - from_height: u32, + variant: HeightVariant, + value: Option, limit: Option, } @@ -103,8 +111,18 @@ async fn costs( ) -> impl Responder { let limit = query.limit.unwrap_or(100); - match data.get_costs(query.from_height, limit).await { - Ok(bundle_costs) => HttpResponse::Ok().json(bundle_costs), + let response = match query.variant { + HeightVariant::Latest => data.get_latest_costs(limit).await, + HeightVariant::Specific => match query.value { + Some(height) => data.get_costs(height, limit).await, + None => Err(services::Error::Other( + "height value is required".to_string(), + )), + }, + }; + + match response { + Ok(costs) => HttpResponse::Ok().json(costs), Err(services::Error::Other(e)) => { HttpResponse::from_error(InternalError::new(e, StatusCode::BAD_REQUEST)) } diff --git a/packages/adapters/storage/src/lib.rs b/packages/adapters/storage/src/lib.rs index 8ea34bbc..6f9360d4 100644 --- a/packages/adapters/storage/src/lib.rs +++ b/packages/adapters/storage/src/lib.rs @@ -50,6 +50,10 @@ impl services::cost_reporter::port::Storage for Postgres { .await .map_err(Into::into) } + + async fn get_latest_costs(&self, limit: usize) -> Result> { + self._get_latest_costs(limit).await.map_err(Into::into) + } } impl services::status_reporter::port::Storage for Postgres { @@ -1163,4 +1167,38 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn get_latest_finalized_costs() -> Result<()> { + use services::cost_reporter::port::Storage; + + // given + let storage = start_db().await; + + for i in 0..5 { + let start_height = i * 10 + 1; + let end_height = start_height + 9; + let block_range = start_height..=end_height; + + ensure_finalized_fragments_exist_in_the_db( + storage.clone(), + block_range, + 1000u128, + 5000u64, + ) + .await; + } + + // when + let finalized_costs = storage.get_latest_costs(1).await?; + + // then + assert_eq!(finalized_costs.len(), 1); + let finalized_cost = &finalized_costs[0]; + + assert_eq!(finalized_cost.start_height, 41); + assert_eq!(finalized_cost.end_height, 50); + + Ok(()) + } } diff --git a/packages/adapters/storage/src/postgres.rs b/packages/adapters/storage/src/postgres.rs index fb19bdc7..5401debe 100644 --- a/packages/adapters/storage/src/postgres.rs +++ b/packages/adapters/storage/src/postgres.rs @@ -865,6 +865,36 @@ impl Postgres { .collect::>>() } + pub(crate) async fn _get_latest_costs(&self, limit: usize) -> Result> { + sqlx::query_as!( + tables::BundleCost, + r#" + SELECT + bc.bundle_id, + bc.cost, + bc.size, + bc.da_block_height, + bc.is_finalized, + b.start_height, + b.end_height + FROM + bundle_cost bc + JOIN bundles b ON bc.bundle_id = b.id + WHERE + bc.is_finalized = TRUE + ORDER BY + b.start_height DESC + LIMIT $1 + "#, + limit as i64 + ) + .fetch_all(&self.connection_pool) + .await? + .into_iter() + .map(BundleCost::try_from) + .collect::>>() + } + pub(crate) async fn _next_bundle_id(&self) -> Result> { let next_id = sqlx::query!("SELECT nextval(pg_get_serial_sequence('bundles', 'id'))") .fetch_one(&self.connection_pool) diff --git a/packages/adapters/storage/src/test_instance.rs b/packages/adapters/storage/src/test_instance.rs index b4baa4ea..824df64c 100644 --- a/packages/adapters/storage/src/test_instance.rs +++ b/packages/adapters/storage/src/test_instance.rs @@ -1,9 +1,3 @@ -use std::{ - borrow::Cow, - ops::RangeInclusive, - sync::{Arc, Weak}, -}; - use delegate::delegate; use services::{ block_bundler, block_committer, block_importer, @@ -14,6 +8,11 @@ use services::{ }, }; use sqlx::Executor; +use std::{ + borrow::Cow, + ops::RangeInclusive, + sync::{Arc, Weak}, +}; use testcontainers::{ core::{ContainerPort, WaitFor}, runners::AsyncRunner, @@ -351,4 +350,8 @@ impl services::cost_reporter::port::Storage for DbWithProcess { .await .map_err(Into::into) } + + async fn get_latest_costs(&self, limit: usize) -> services::Result> { + self.db._get_latest_costs(limit).await.map_err(Into::into) + } } diff --git a/packages/services/src/cost_reporter.rs b/packages/services/src/cost_reporter.rs index b564c879..600b699e 100644 --- a/packages/services/src/cost_reporter.rs +++ b/packages/services/src/cost_reporter.rs @@ -36,6 +36,17 @@ pub mod service { .get_finalized_costs(from_block_height, limit) .await } + + pub async fn get_latest_costs(&self, limit: usize) -> Result> { + if limit > self.request_limit { + return Err(Error::Other(format!( + "requested: {} items, but limit is: {}", + limit, self.request_limit + ))); + } + + self.storage.get_latest_costs(limit).await + } } } @@ -50,5 +61,7 @@ pub mod port { from_block_height: u32, limit: usize, ) -> Result>; + + async fn get_latest_costs(&self, limit: usize) -> Result>; } } From c8a7ca7d8195b38c3296ed9413173aca6928563d Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 19 Dec 2024 01:48:22 +0100 Subject: [PATCH 44/47] enable cache --- committer/src/main.rs | 1 + committer/src/setup.rs | 11 +++++++---- packages/services/src/fee_tracker/port.rs | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/committer/src/main.rs b/committer/src/main.rs index 9a654eea..202ecece 100644 --- a/committer/src/main.rs +++ b/committer/src/main.rs @@ -7,6 +7,7 @@ mod setup; use api::launch_api_server; use errors::{Result, WithContext}; use metrics::prometheus::Registry; +use services::fee_tracker::port::cache::CachingApi; use setup::last_finalization_metric; use tokio_util::sync::CancellationToken; diff --git a/committer/src/setup.rs b/committer/src/setup.rs index 0cfe9c43..8e2fcd18 100644 --- a/committer/src/setup.rs +++ b/committer/src/setup.rs @@ -9,7 +9,10 @@ use metrics::{ }; use services::{ block_committer::{port::l1::Contract, service::BlockCommitter}, - fee_tracker::service::{FeeThresholds, FeeTracker, SmaPeriods}, + fee_tracker::{ + port::cache::CachingApi, + service::{FeeThresholds, FeeTracker, SmaPeriods}, + }, state_committer::port::Storage, state_listener::service::StateListener, state_pruner::service::StatePruner, @@ -119,7 +122,7 @@ pub fn state_committer( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, - fee_tracker: FeeTracker, + fee_tracker: FeeTracker>, ) -> Result> { let state_committer = services::StateCommitter::new( l1, @@ -327,9 +330,9 @@ pub fn fee_tracker( cancel_token: CancellationToken, config: &config::Config, registry: &Registry, -) -> Result<(FeeTracker, tokio::task::JoinHandle<()>)> { +) -> Result<(FeeTracker>, tokio::task::JoinHandle<()>)> { let fee_tracker = FeeTracker::new( - l1, + CachingApi::new(l1, 24 * 3600 / 12), services::fee_tracker::service::Config { sma_periods: SmaPeriods { short: config.app.fee_algo.short_sma_blocks, diff --git a/packages/services/src/fee_tracker/port.rs b/packages/services/src/fee_tracker/port.rs index e9112e94..e7ba4f55 100644 --- a/packages/services/src/fee_tracker/port.rs +++ b/packages/services/src/fee_tracker/port.rs @@ -188,7 +188,7 @@ pub mod l1 { } pub mod cache { - use std::{collections::BTreeMap, ops::RangeInclusive}; + use std::{collections::BTreeMap, ops::RangeInclusive, sync::Arc}; use tokio::sync::RwLock; @@ -196,10 +196,10 @@ pub mod cache { use super::l1::{Api, BlockFees, Fees, SequentialBlockFees}; - #[derive(Debug)] + #[derive(Debug, Clone)] pub struct CachingApi

{ fees_provider: P, - cache: RwLock>, + cache: Arc>>, cache_limit: usize, } @@ -207,7 +207,7 @@ pub mod cache { pub fn new(fees_provider: P, cache_limit: usize) -> Self { Self { fees_provider, - cache: RwLock::new(BTreeMap::new()), + cache: Arc::new(RwLock::new(BTreeMap::new())), cache_limit, } } From c1a9ea33c974206f02d892e8e56a30d2a49d295a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Thu, 19 Dec 2024 14:57:26 +0100 Subject: [PATCH 45/47] add more logging to debug --- packages/services/src/fee_tracker/service.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index 8cd176a0..d5a9a4de 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -105,6 +105,8 @@ impl SendOrWaitDecider for FeeTracker

{ num_l2_blocks_behind, ); + info!("short_term_tx_fee: {short_term_tx_fee}, long_term_tx_fee: {long_term_tx_fee}, max_upper_tx_fee: {max_upper_tx_fee}"); + let should_send = short_term_tx_fee < max_upper_tx_fee; if should_send { @@ -252,6 +254,8 @@ impl FeeTracker

{ Percentage::PPM + end_premium_ppm, ); + info!("start_discount_ppm: {start_discount_ppm}, end_premium_ppm: {end_premium_ppm}, base_multiplier: {base_multiplier}, premium_increment: {premium_increment}, multiplier_ppm: {multiplier_ppm}"); + // 3. Final fee: eg. 105% of the base fee fee.saturating_mul(multiplier_ppm) .saturating_div(Percentage::PPM) From 173a26934c1f698b08371c9bbcf6f24b9ba1ff2a Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 23 Dec 2024 10:34:55 +0100 Subject: [PATCH 46/47] fix bug, minimum not maximum starting height when calculating l2 blocks behind --- packages/services/src/state_committer.rs | 42 ++++++++++++++++++------ 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/services/src/state_committer.rs b/packages/services/src/state_committer.rs index caa47636..de728ad8 100644 --- a/packages/services/src/state_committer.rs +++ b/packages/services/src/state_committer.rs @@ -137,15 +137,8 @@ pub mod service { let l1_height = self.l1_adapter.current_height().await?; let l2_height = self.fuel_api.latest_height().await?; - let oldest_l2_block_in_fragments = fragments - .maximum_by_key(|b| b.oldest_block_in_bundle) - .oldest_block_in_bundle; - - let num_l2_blocks_behind = l2_height.saturating_sub(oldest_l2_block_in_fragments); - - self.metrics - .num_l2_blocks_behind - .set(num_l2_blocks_behind as i64); + let num_l2_blocks_behind = self.num_l2_blocks_behind(fragments, l2_height); + self.update_l2_blocks_behind_metric(num_l2_blocks_behind); self.decider .should_send_blob_tx( @@ -156,6 +149,18 @@ pub mod service { .await } + fn num_l2_blocks_behind( + &self, + fragments: &NonEmpty, + l2_height: u32, + ) -> u32 { + let oldest_l2_block_in_fragments = fragments + .minimum_by_key(|b| b.oldest_block_in_bundle) + .oldest_block_in_bundle; + + l2_height.saturating_sub(oldest_l2_block_in_fragments) + } + async fn submit_fragments( &self, fragments: NonEmpty, @@ -224,7 +229,23 @@ pub mod service { .oldest_nonfinalized_fragments(starting_height, 6) .await?; - Ok(NonEmpty::collect(existing_fragments)) + let fragments = NonEmpty::collect(existing_fragments); + + if let Some(fragments) = fragments.as_ref() { + // Tracking the metric here as well to get updates more often -- because + // submit_fragments might not be called + self.update_l2_blocks_behind_metric( + self.num_l2_blocks_behind(fragments, latest_height), + ); + } + + Ok(fragments) + } + + fn update_l2_blocks_behind_metric(&self, l2_blocks_behind: u32) { + self.metrics + .num_l2_blocks_behind + .set(l2_blocks_behind as i64); } async fn should_submit_fragments(&self, fragment_count: NonZeroUsize) -> Result { @@ -256,6 +277,7 @@ pub mod service { self.submit_fragments(fragments, None).await?; } } + Ok(()) } From 3a554e400e7c35baf31173a19ea242f59e88cb1b Mon Sep 17 00:00:00 2001 From: segfault-magnet Date: Mon, 23 Dec 2024 11:57:13 +0100 Subject: [PATCH 47/47] mult reward by INTRINSIC_GAS --- packages/services/src/fee_tracker/service.rs | 12 ++++++++---- packages/services/tests/fee_tracker.rs | 7 +++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/services/src/fee_tracker/service.rs b/packages/services/src/fee_tracker/service.rs index d5a9a4de..680afa13 100644 --- a/packages/services/src/fee_tracker/service.rs +++ b/packages/services/src/fee_tracker/service.rs @@ -287,10 +287,14 @@ impl FeeTracker

{ const DATA_GAS_PER_BLOB: u128 = 131_072u128; const INTRINSIC_GAS: u128 = 21_000u128; - let base_fee = INTRINSIC_GAS * fees.base_fee_per_gas; - let blob_fee = fees.base_fee_per_blob_gas * num_blobs as u128 * DATA_GAS_PER_BLOB; - - base_fee + blob_fee + fees.reward + let base_fee = INTRINSIC_GAS.saturating_mul(fees.base_fee_per_gas); + let blob_fee = fees + .base_fee_per_blob_gas + .saturating_mul(num_blobs as u128) + .saturating_mul(DATA_GAS_PER_BLOB); + let reward_fee = fees.reward.saturating_mul(INTRINSIC_GAS); + + base_fee.saturating_add(blob_fee).saturating_add(reward_fee) } fn last_n_blocks(current_block: u64, n: NonZeroU64) -> RangeInclusive { diff --git a/packages/services/tests/fee_tracker.rs b/packages/services/tests/fee_tracker.rs index 7b92c0c1..6936791b 100644 --- a/packages/services/tests/fee_tracker.rs +++ b/packages/services/tests/fee_tracker.rs @@ -61,14 +61,14 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6 .try_into().unwrap()}, fee_thresholds: FeeThresholds { - always_acceptable_fee: (21_000 * 5000) + (6 * 131_072 * 5000) + 5000 + 1, + always_acceptable_fee: (21_000 * (5000 + 5000)) + (6 * 131_072 * 5000) + 1, max_l2_blocks_behind: 100.try_into().unwrap(), ..Default::default() } }, 0, true; - "Should send since short-term fee < always_acceptable_fee" + "Should send since short-term fee less than always_acceptable_fee" )] #[test_case( Fees { base_fee_per_gas: 2000, reward: 10000, base_fee_per_blob_gas: 1000 }, @@ -234,9 +234,8 @@ fn generate_fees(config: Config, old_fees: Fees, new_fees: Fees) -> Vec<(u64, Fe "Zero blobs: short-term reward is higher, don't send" )] #[test_case( - // Zero blobs don't care about higher short-term base_fee_per_blob_gas Fees { base_fee_per_gas: 3000, reward: 6000, base_fee_per_blob_gas: 5000 }, - Fees { base_fee_per_gas: 2000, reward: 7000, base_fee_per_blob_gas: 50_000_000 }, + Fees { base_fee_per_gas: 2000, reward: 6000, base_fee_per_blob_gas: 50_000_000 }, 0, Config { sma_periods: SmaPeriods { short: 2.try_into().unwrap(), long: 6.try_into().unwrap()},