From 275ee3c73779ac682fdf165b6882f0dec9f211af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 8 Jan 2025 14:46:18 +0100 Subject: [PATCH 1/5] Add boilerplate for command line and parsing csv file --- .../gas_price_service/simulation/Cargo.toml | 4 + .../gas_price_service/simulation/src/main.rs | 89 ++++++++++++++++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/crates/services/gas_price_service/simulation/Cargo.toml b/crates/services/gas_price_service/simulation/Cargo.toml index 665c0b80cc7..3312001748e 100644 --- a/crates/services/gas_price_service/simulation/Cargo.toml +++ b/crates/services/gas_price_service/simulation/Cargo.toml @@ -12,5 +12,9 @@ fuel-core-types = { version = "0.40.2", path = "../../../types", default-feature fuel-gas-price-algorithm = { path = "../../../fuel-gas-price-algorithm" } async-trait = "0.1.83" tokio = "1.41.1" +serde = { version = "1.0.217", features = ["derive"] } +serde-reflection = "0.5.0" +csv = "1.3.1" +clap = { version = "4.5.24", features = ["derive"] } [workspace] diff --git a/crates/services/gas_price_service/simulation/src/main.rs b/crates/services/gas_price_service/simulation/src/main.rs index 558867d87e6..97b1babb84f 100644 --- a/crates/services/gas_price_service/simulation/src/main.rs +++ b/crates/services/gas_price_service/simulation/src/main.rs @@ -1,15 +1,70 @@ +use std::path::{ + Path, + PathBuf, +}; + use crate::service::{ get_service_controller, ServiceController, }; +use clap::{ + Parser, + Subcommand, +}; +use csv::StringRecord; use fuel_core_gas_price_service::{ common::utils::BlockInfo, v1::da_source_service::DaBlockCosts, }; +use serde_reflection::{ + Samples, + Tracer, + TracerConfig, +}; pub mod data_sources; pub mod service; +#[allow(dead_code)] +#[derive(Debug, serde::Deserialize)] +struct Record { + l1_block_number: u64, + l1_blob_fee_wei: u64, + l2_block_number: u64, + l2_gas_fullness: u64, + l2_gas_capacity: u64, + l2_byte_size: u64, + l2_byte_capacity: u64, +} + +pub(crate) fn fields_of_struct_in_order() -> Vec +where + T: serde::de::DeserializeOwned, +{ + let mut tracer = Tracer::new(TracerConfig::default()); + let samples = Samples::new(); + tracer.trace_type::(&samples).unwrap(); + let type_name = std::any::type_name::().split("::").last().unwrap(); + let registry = tracer.registry().unwrap(); + let Some(serde_reflection::ContainerFormat::Struct(fields)) = registry.get(type_name) + else { + panic!("No fields?") + }; + + fields.iter().map(|f| f.name.clone()).collect() +} + +#[derive(Subcommand, Debug)] +enum DataSource { + /// Load data from a CSV file. + File { + #[arg(short, long)] + path: PathBuf, + }, + /// Generate arbitrary data (not supported yet). + Generated, +} + pub struct Data { inner: Vec<(BlockInfo, Option)>, } @@ -22,8 +77,29 @@ impl Data { struct SimulationResults {} -fn get_data() -> anyhow::Result { - // TODO +fn get_data(source: &DataSource) -> anyhow::Result { + match source { + DataSource::File { path } => { + let headers = csv::StringRecord::from(fields_of_struct_in_order::()); + let mut rdr = csv::ReaderBuilder::new() + .has_headers(true) + .from_path(path) + .unwrap(); + + let records: Result, _> = rdr + .records() + .map(|line_entry| { + let line = line_entry?; + line.deserialize(Some(&headers)) + }) + .collect(); + let records = records?; + + dbg!(&records); + } + DataSource::Generated => unimplemented!(), + } + let data = Data { inner: vec![] }; Ok(data) } @@ -48,9 +124,16 @@ fn display_results(_results: SimulationResults) -> anyhow::Result<()> { Ok(()) } +#[derive(Parser, Debug)] +struct Args { + #[command(subcommand)] + data_source: DataSource, +} + #[tokio::main] async fn main() -> anyhow::Result<()> { - let data = get_data()?; + let args = Args::parse(); + let data = get_data(&args.data_source)?; let mut service_controller = get_service_controller().await; let results = simulation(&data, &mut service_controller).await?; display_results(results)?; From f690b6586f093b25021a04b1f849cc70492012a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 8 Jan 2025 15:56:12 +0100 Subject: [PATCH 2/5] Build inner data from file --- .../gas_price_service/simulation/Cargo.toml | 1 + .../gas_price_service/simulation/src/main.rs | 54 +++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/crates/services/gas_price_service/simulation/Cargo.toml b/crates/services/gas_price_service/simulation/Cargo.toml index 3312001748e..32c3e0d87da 100644 --- a/crates/services/gas_price_service/simulation/Cargo.toml +++ b/crates/services/gas_price_service/simulation/Cargo.toml @@ -16,5 +16,6 @@ serde = { version = "1.0.217", features = ["derive"] } serde-reflection = "0.5.0" csv = "1.3.1" clap = { version = "4.5.24", features = ["derive"] } +itertools = "0.14.0" [workspace] diff --git a/crates/services/gas_price_service/simulation/src/main.rs b/crates/services/gas_price_service/simulation/src/main.rs index 97b1babb84f..27b104dcd27 100644 --- a/crates/services/gas_price_service/simulation/src/main.rs +++ b/crates/services/gas_price_service/simulation/src/main.rs @@ -16,6 +16,7 @@ use fuel_core_gas_price_service::{ common::utils::BlockInfo, v1::da_source_service::DaBlockCosts, }; +use itertools::Itertools; use serde_reflection::{ Samples, Tracer, @@ -75,10 +76,24 @@ impl Data { } } +impl From<&Record> for BlockInfo { + fn from(value: &Record) -> Self { + // TODO: Also support `GenesisBlock` variant? + return BlockInfo::Block { + height: value.l2_block_number.try_into().unwrap(), + gas_used: value.l2_gas_fullness, + block_gas_capacity: value.l2_gas_capacity, + block_bytes: value.l2_byte_size, + block_fees: 0, // TODO + gas_price: 0, // TODO + } + } +} + struct SimulationResults {} fn get_data(source: &DataSource) -> anyhow::Result { - match source { + let records = match source { DataSource::File { path } => { let headers = csv::StringRecord::from(fields_of_struct_in_order::()); let mut rdr = csv::ReaderBuilder::new() @@ -93,16 +108,45 @@ fn get_data(source: &DataSource) -> anyhow::Result { line.deserialize(Some(&headers)) }) .collect(); - let records = records?; - - dbg!(&records); + records? } DataSource::Generated => unimplemented!(), + }; + + let mut data = vec![]; + let groups = records.iter().chunk_by(|record| record.l1_block_number); + for (l1_block_number, block_records) in groups.into_iter() { + let blocks: Vec<_> = block_records.into_iter().collect(); + let mut blocks_iter = blocks.iter().peekable(); + + while let Some(block_record) = blocks_iter.next() { + let l2_block: BlockInfo = (*block_record).into(); + let da_block_costs = if blocks_iter.peek().is_none() { + // TODO: Check if these are generated correctly. + let bundle_id: u32 = l1_block_number.try_into().unwrap(); + let range = blocks.first().unwrap().l2_block_number as u32 + ..=blocks.last().unwrap().l2_block_number as u32; + let bundle_size_bytes: u32 = + blocks.iter().map(|r| r.l2_byte_size as u32).sum(); + let blob_cost_wei = block_record.l1_blob_fee_wei as u128; + + Some(DaBlockCosts { + bundle_id, + l2_blocks: range, + bundle_size_bytes, + blob_cost_wei, + }) + } else { + None + }; + data.push((l2_block, da_block_costs)); + } } - let data = Data { inner: vec![] }; + let data = Data { inner: data }; Ok(data) } + async fn simulation( data: &Data, service_controller: &mut ServiceController, From 48e74eb9d907acbe7be8ccbab4cd34ff2348996e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 8 Jan 2025 16:07:08 +0100 Subject: [PATCH 3/5] Code clean-up --- .../services/gas_price_service/simulation/src/main.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/services/gas_price_service/simulation/src/main.rs b/crates/services/gas_price_service/simulation/src/main.rs index 27b104dcd27..dee09056fb9 100644 --- a/crates/services/gas_price_service/simulation/src/main.rs +++ b/crates/services/gas_price_service/simulation/src/main.rs @@ -66,6 +66,7 @@ enum DataSource { Generated, } +#[derive(Debug)] pub struct Data { inner: Vec<(BlockInfo, Option)>, } @@ -121,7 +122,7 @@ fn get_data(source: &DataSource) -> anyhow::Result { while let Some(block_record) = blocks_iter.next() { let l2_block: BlockInfo = (*block_record).into(); - let da_block_costs = if blocks_iter.peek().is_none() { + let da_block_costs = blocks_iter.peek().is_none().then(|| { // TODO: Check if these are generated correctly. let bundle_id: u32 = l1_block_number.try_into().unwrap(); let range = blocks.first().unwrap().l2_block_number as u32 @@ -130,15 +131,13 @@ fn get_data(source: &DataSource) -> anyhow::Result { blocks.iter().map(|r| r.l2_byte_size as u32).sum(); let blob_cost_wei = block_record.l1_blob_fee_wei as u128; - Some(DaBlockCosts { + DaBlockCosts { bundle_id, l2_blocks: range, bundle_size_bytes, blob_cost_wei, - }) - } else { - None - }; + } + }); data.push((l2_block, da_block_costs)); } } From 6116aab5bc81fed0c30115dcdae5809761025610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Chabowski?= Date: Wed, 8 Jan 2025 17:08:18 +0100 Subject: [PATCH 4/5] Inject simulated gas price and fee into the L2 blocks --- .../simulation/src/data_sources.rs | 62 +++++++++++++++---- .../gas_price_service/simulation/src/main.rs | 15 ++--- .../simulation/src/service.rs | 5 +- 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/crates/services/gas_price_service/simulation/src/data_sources.rs b/crates/services/gas_price_service/simulation/src/data_sources.rs index d6d30b7b00d..5165cf84c82 100644 --- a/crates/services/gas_price_service/simulation/src/data_sources.rs +++ b/crates/services/gas_price_service/simulation/src/data_sources.rs @@ -4,9 +4,12 @@ use fuel_core_gas_price_service::{ l2_block_source::L2BlockSource, utils::BlockInfo, }, - v1::da_source_service::{ - service::DaBlockCostsSource, - DaBlockCosts, + v1::{ + algorithm::SharedV1Algorithm, + da_source_service::{ + service::DaBlockCostsSource, + DaBlockCosts, + }, }, }; @@ -18,30 +21,65 @@ use fuel_core_types::fuel_types::BlockHeight; pub struct SimulatedL2Blocks { recv: tokio::sync::mpsc::Receiver, + shared_algo: SharedV1Algorithm, } impl SimulatedL2Blocks { - pub fn new(recv: tokio::sync::mpsc::Receiver) -> Self { - Self { recv } + pub fn new( + recv: tokio::sync::mpsc::Receiver, + shared_algo: SharedV1Algorithm, + ) -> Self { + Self { recv, shared_algo } } - pub fn new_with_sender() -> (Self, tokio::sync::mpsc::Sender) { + pub fn new_with_sender( + shared_algo: SharedV1Algorithm, + ) -> (Self, tokio::sync::mpsc::Sender) { let (send, recv) = tokio::sync::mpsc::channel(16); - (Self { recv }, send) + (Self { recv, shared_algo }, send) } } #[async_trait] impl L2BlockSource for SimulatedL2Blocks { async fn get_l2_block(&mut self) -> GasPriceResult { - // TODO: do we want to modify these values to somehow reflect the previously chosen gas - // price better? We might be able to do that by having a handle to the shared algo. - - self.recv.recv().await.ok_or({ + let block = self.recv.recv().await.ok_or({ GasPriceError::CouldNotFetchL2Block { source_error: anyhow::anyhow!("no more blocks; channel closed"), } - }) + })?; + + let BlockInfo::Block { + gas_used, + height, + block_gas_capacity, + block_bytes, + .. + } = block + else { + return Err(GasPriceError::CouldNotFetchL2Block { + source_error: anyhow::anyhow!("unexpected genesis block"), + }); + }; + + let gas = gas_used; + let new_gas_price = self.shared_algo.next_gas_price(); + let gas_price_factor = 1_150_000; // TODO: Read from CLI/config + + let mut fee = (gas as u128).checked_mul(new_gas_price as u128).expect( + "Impossible to overflow because multiplication of two `u64` <= `u128`", + ); + fee = fee.div_ceil(gas_price_factor as u128); + + let block = BlockInfo::Block { + height, + gas_used, + block_gas_capacity, + block_bytes, + block_fees: fee.try_into().expect("overflow"), + gas_price: new_gas_price, + }; + Ok(block) } } diff --git a/crates/services/gas_price_service/simulation/src/main.rs b/crates/services/gas_price_service/simulation/src/main.rs index dee09056fb9..398bbf5f0cb 100644 --- a/crates/services/gas_price_service/simulation/src/main.rs +++ b/crates/services/gas_price_service/simulation/src/main.rs @@ -79,14 +79,13 @@ impl Data { impl From<&Record> for BlockInfo { fn from(value: &Record) -> Self { - // TODO: Also support `GenesisBlock` variant? return BlockInfo::Block { height: value.l2_block_number.try_into().unwrap(), gas_used: value.l2_gas_fullness, block_gas_capacity: value.l2_gas_capacity, block_bytes: value.l2_byte_size, - block_fees: 0, // TODO - gas_price: 0, // TODO + block_fees: 0, // Will be overwritten by the simulation code + gas_price: 0, // Will be overwritten by the simulation code } } } @@ -124,11 +123,10 @@ fn get_data(source: &DataSource) -> anyhow::Result { let l2_block: BlockInfo = (*block_record).into(); let da_block_costs = blocks_iter.peek().is_none().then(|| { // TODO: Check if these are generated correctly. - let bundle_id: u32 = l1_block_number.try_into().unwrap(); + let bundle_id: u32 = l1_block_number as u32; // Could be an arbitrary number, but we use L1 block number for convenience. + let bundle_size_bytes: u32 = 0; // Modify scrape tool to provide this let range = blocks.first().unwrap().l2_block_number as u32 ..=blocks.last().unwrap().l2_block_number as u32; - let bundle_size_bytes: u32 = - blocks.iter().map(|r| r.l2_byte_size as u32).sum(); let blob_cost_wei = block_record.l1_blob_fee_wei as u128; DaBlockCosts { @@ -153,11 +151,6 @@ async fn simulation( let res = SimulationResults {}; for (block, maybe_costs) in data.get_iter() { service_controller.advance(block, maybe_costs).await? - // GET GAS PRICE - - // MODIFY WITH GAS PRICE FACTOR - - // RECORD LATEST VALUES } Ok(res) } diff --git a/crates/services/gas_price_service/simulation/src/service.rs b/crates/services/gas_price_service/simulation/src/service.rs index c9856688771..730d1f8e2d3 100644 --- a/crates/services/gas_price_service/simulation/src/service.rs +++ b/crates/services/gas_price_service/simulation/src/service.rs @@ -73,7 +73,7 @@ fn read_metadata_from_file(_metadata_path: &str) -> V1Metadata { // TODO: read from file and/or CLI V1Metadata { new_scaled_exec_price: 0, - l2_block_height: 0, + l2_block_height: 999, // TODO: Use first L2 block height from the CSV new_scaled_da_gas_price: 0, gas_price_factor: NonZero::new(100).unwrap(), total_da_rewards_excess: 0, @@ -139,7 +139,8 @@ pub async fn get_service_controller() -> ServiceController { let algo = algorithm_updater.algorithm(); let shared_algo = SharedV1Algorithm::new_with_algorithm(algo); - let (l2_block_source, l2_block_sender) = SimulatedL2Blocks::new_with_sender(); + let (l2_block_source, l2_block_sender) = + SimulatedL2Blocks::new_with_sender(shared_algo.clone()); let (da_block_source, da_costs_sender) = SimulatedDACosts::new_with_sender(); let poll_interval = poll_interval(); From 1c4bb14f792d15e5477962fccd158afbabf94050 Mon Sep 17 00:00:00 2001 From: rymnc <43716372+rymnc@users.noreply.github.com> Date: Wed, 8 Jan 2025 22:37:46 +0530 Subject: [PATCH 5/5] fix: lint toml --- crates/services/gas_price_service/simulation/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/services/gas_price_service/simulation/Cargo.toml b/crates/services/gas_price_service/simulation/Cargo.toml index 32c3e0d87da..7235ef35e1b 100644 --- a/crates/services/gas_price_service/simulation/Cargo.toml +++ b/crates/services/gas_price_service/simulation/Cargo.toml @@ -7,7 +7,10 @@ edition = "2021" anyhow = "1.0" # this is floating and needs to match what is used by other packages fuel-core-gas-price-service = { path = ".." } fuel-core-services = { path = "../../../services", features = ["test-helpers"] } -fuel-core-storage = { path = "../../../storage", features = ["std", "test-helpers"] } +fuel-core-storage = { path = "../../../storage", features = [ + "std", + "test-helpers", +] } fuel-core-types = { version = "0.40.2", path = "../../../types", default-features = false } fuel-gas-price-algorithm = { path = "../../../fuel-gas-price-algorithm" } async-trait = "0.1.83"