From 8426dbe14e47d84ae6571e23362d7a9b1274f52a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Thu, 16 Jan 2025 11:50:19 +0000 Subject: [PATCH] feat(pallet-market): implement benchmarks --- .github/workflows/ci.yaml | 26 +- Cargo.lock | 4 +- Cargo.toml | 3 +- node/Cargo.toml | 4 +- pallets/market/Cargo.toml | 47 ++- pallets/market/src/benchmarking.rs | 409 ++++++++++++++++++++++ pallets/market/src/lib.rs | 182 +++++----- pallets/market/src/mock.rs | 49 ++- pallets/market/src/test.rs | 4 +- pallets/market/src/weights.rs | 205 +++++++++++ pallets/storage-provider/src/tests/mod.rs | 13 +- primitives/src/lib.rs | 5 + primitives/src/sector/pre_commit.rs | 31 +- runtime/Cargo.toml | 1 + runtime/src/benchmarks.rs | 1 + runtime/src/configs/mod.rs | 82 ++++- storagext/lib/Cargo.toml | 2 +- 17 files changed, 946 insertions(+), 122 deletions(-) create mode 100644 pallets/market/src/benchmarking.rs create mode 100644 pallets/market/src/weights.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c80739ee..4738bc4eb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,6 +5,7 @@ on: types: [labeled] push: branches: [develop, main] + workflow_dispatch: # manual launch concurrency: group: ${{ github.ref }} @@ -107,11 +108,12 @@ jobs: run: cargo test --profile ci --locked -p pallet-proofs --features runtime-benchmarks -- benchmark --nocapture - name: pallet-proofs — Run node benchmarks - run: cargo run \ + run: | + cargo run \ -p polka-storage-node -r -F runtime-benchmarks -- \ benchmark pallet \ --wasm-execution=compiled \ - --pallet "pallet_proofs"\ + --pallet "pallet_proofs" \ --extrinsic "*" \ --steps 5 \ --repeat 1 \ @@ -121,11 +123,27 @@ jobs: run: cargo test --profile ci --locked -p pallet-randomness --features runtime-benchmarks -- benchmark --nocapture - name: pallet-randomness — Run node benchmarks - run: cargo run \ + run: | + cargo run \ -p polka-storage-node -r -F runtime-benchmarks -- \ benchmark pallet \ --wasm-execution=compiled \ - --pallet "pallet_randomness"\ + --pallet "pallet_randomness" \ + --extrinsic "*" \ + --steps 5 \ + --repeat 1 \ + --template node/benchmark_template.hbs + + - name: pallet-market — Run test benchmarks + run: cargo test --profile ci --locked -p pallet-market --features runtime-benchmarks -- benchmark --nocapture + + - name: pallet-market — Run node benchmarks + run: | + cargo run \ + -p polka-storage-node -r -F runtime-benchmarks -F testnet -- \ + benchmark pallet \ + --wasm-execution=compiled \ + --pallet "pallet_market" \ --extrinsic "*" \ --steps 5 \ --repeat 1 \ diff --git a/Cargo.lock b/Cargo.lock index aa4d6d1fa..755be3b72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11504,17 +11504,19 @@ dependencies = [ "frame-support 28.0.0", "frame-system 28.0.0", "hex", + "itertools 0.13.0", "log", - "multihash-codetable", "pallet-balances 28.0.0", "pallet-proofs", "pallet-storage-provider", "parity-scale-codec", "primitives", + "rand", "scale-info", "sp-arithmetic 23.0.0", "sp-core 28.0.0", "sp-io 30.0.0", + "sp-keystore 0.34.0", "sp-runtime 31.0.1", "sp-std 14.0.0 (git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-stable2412)", "thiserror 2.0.8", diff --git a/Cargo.toml b/Cargo.toml index 431134584..d3b18cc73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,8 @@ indexmap = "2.2.6" integer-encoding = "4.0.0" ipld-core = "0.4.1" ipld-dagpb = "0.2.1" -itertools = "0.13.0" +# Use `use_alloc` for crates that compile to WASM +itertools = { version = "0.13.0", default-features = false } jsonrpsee = { version = "0.24.7" } libp2p = { version = "0.54", default-features = false } libp2p-core = "0.42.0" diff --git a/node/Cargo.toml b/node/Cargo.toml index 664cff856..59f24419b 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -14,6 +14,8 @@ version = "0.0.0" name = "polka-storage-node" [dependencies] +polka-storage-runtime.workspace = true + clap = { features = ["derive"], workspace = true } codec = { workspace = true, default-features = true } color-print = { workspace = true } @@ -33,7 +35,6 @@ futures = { workspace = true } jsonrpsee = { features = ["server"], workspace = true } log = { workspace = true, default-features = true } pallet-transaction-payment-rpc = { workspace = true, default-features = true } -polka-storage-runtime.workspace = true polkadot-cli = { features = ["rococo-native"], workspace = true, default-features = true } polkadot-primitives = { workspace = true, default-features = true } prometheus-endpoint = { workspace = true, default-features = true } @@ -82,4 +83,5 @@ runtime-benchmarks = [ "sc-service/runtime-benchmarks", "sp-runtime/runtime-benchmarks", ] +testnet = ["polka-storage-runtime/testnet"] try-runtime = ["polka-storage-runtime/try-runtime", "polkadot-cli/try-runtime", "sp-runtime/try-runtime"] diff --git a/pallets/market/Cargo.toml b/pallets/market/Cargo.toml index 5fa9e6abc..78ae724d8 100644 --- a/pallets/market/Cargo.toml +++ b/pallets/market/Cargo.toml @@ -16,13 +16,16 @@ workspace = true targets = ["x86_64-unknown-linux-gnu"] [dependencies] -cid = { workspace = true, default-features = false, features = ["scale-codec"] } +# Homegrown +primitives = { workspace = true, default-features = false } + +# De/Encoding codec = { workspace = true, default-features = false, features = ["derive"] } +scale-info = { workspace = true, default-features = false, features = ["derive"] } + +cid = { workspace = true, default-features = false, features = ["scale-codec"] } hex = { workspace = true, default-features = false, features = ["alloc"] } log = { workspace = true } -multihash-codetable = { workspace = true, features = ["blake2b"] } -primitives = { workspace = true } -scale-info = { workspace = true, default-features = false, features = ["derive"] } thiserror = { workspace = true, default-features = false } # frame deps @@ -32,16 +35,37 @@ frame-system = { workspace = true, default-features = false } sp-arithmetic = { workspace = true, default-features = false } sp-std = { workspace = true, default-features = false } +# Benchmark specific +## Homegrown +pallet-balances = { workspace = true, default-features = false, optional = true } +pallet-storage-provider = { workspace = true, default-features = false, optional = true } +## Required for .chunks() +itertools = { workspace = true, optional = true, default-features = false, features = ["use_alloc"] } +## Required for random CIDs +rand = { workspace = true, features = ["small_rng"], optional = true } +## Substrate Primitives & Friends +sp-core = { workspace = true, default_features = false, optional = true } +sp-io = { workspace = true, optional = true } +sp-keystore = { workspace = true, default-features = false, optional = true } +sp-runtime = { workspace = true, optional = true } + [dev-dependencies] -blake2b_simd = { workspace = true, default-features = true } -cid = { workspace = true, default-features = false, features = ["alloc", "scale-codec"] } -env_logger = { workspace = true } +# Homegrown pallet-balances = { workspace = true, default-features = false } pallet-proofs = { workspace = true, default-features = false } pallet-storage-provider = { workspace = true, default-features = false } primitives = { workspace = true, default-features = false, features = ["builder", "testing"] } + +blake2b_simd = { workspace = true, default-features = true } +cid = { workspace = true, default-features = false, features = ["alloc", "scale-codec"] } +env_logger = { workspace = true } +# Just used in benchmarks but they're also considered tests +itertools = { workspace = true, default-features = false, features = ["use_alloc"] } +# Just used in benchmarks but they're also considered tests +rand = { workspace = true, features = ["small_rng"] } sp-core = { workspace = true, default-features = false } sp-io = { workspace = true } +sp-keystore = { workspace = true, default-features = false } sp-runtime = { workspace = true, default-features = false } [features] @@ -50,8 +74,17 @@ runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", + "itertools", + "pallet-balances", + "pallet-storage-provider", + "primitives/builder", "primitives/builder", "primitives/testing", + "rand", + "sp-core", + "sp-io", + "sp-keystore", + "sp-runtime", "sp-runtime/runtime-benchmarks", ] std = [ diff --git a/pallets/market/src/benchmarking.rs b/pallets/market/src/benchmarking.rs new file mode 100644 index 000000000..27185a9eb --- /dev/null +++ b/pallets/market/src/benchmarking.rs @@ -0,0 +1,409 @@ +#![cfg(feature = "runtime-benchmarks")] + +use cid::{multihash::Multihash, Cid}; +use codec::Encode; +use frame_benchmarking::v2::*; +use frame_support::{pallet_prelude::ConstU32, sp_runtime::BoundedVec, traits::Currency}; +use frame_system::{ + self, + pallet_prelude::{BlockNumberFor, OriginFor}, + RawOrigin, +}; +use pallet_storage_provider::Pallet as SpPallet; +use primitives::{ + commitment::{FIL_COMMITMENT_UNSEALED, SHA2_256_TRUNC254_PADDED}, + proofs::RegisteredPoStProof, + sector::{builder::SectorPreCommitInfoBuilder, ProveCommitSector, SectorPreCommitInfo}, + MAX_LABEL_SIZE, MAX_SECTORS_PER_CALL, PEER_ID_MAX_BYTES, +}; +use rand::{rngs::SmallRng, RngCore, SeedableRng}; +use scale_info::prelude::format; +use sp_core::{ed25519, hashing::sha2_256}; +use sp_io::crypto::{ed25519_generate, ed25519_sign}; +use sp_runtime::{traits::IdentifyAccount, AccountId32, MultiSignature, MultiSigner}; +use sp_std::{vec, vec::Vec}; + +use super::*; +#[allow(unused)] +use crate::Pallet as MarketPallet; + +type BoundedPeerIdBytes = BoundedVec>; +const COLLATERAL: u32 = 10; + +fn random_cid(rng: &mut R) -> Cid { + let mut buffer = [0u8; 32]; + rng.fill_bytes(&mut buffer); + + let hash = sha2_256(&buffer); + let mh = Multihash::<64>::wrap(SHA2_256_TRUNC254_PADDED, &hash).unwrap(); + Cid::new_v1(FIL_COMMITMENT_UNSEALED, mh) +} + +fn generate_benchmark_account(name: &'static str) -> (AccountId32, MultiSigner) { + // Generate a deterministic seed + let seed = format!("//{}", name); + + let signer: sp_core::ed25519::Public = + ed25519_generate(0.into(), Some(seed.into_bytes())).into(); + let signer: MultiSigner = signer.into(); + let account_id = signer.clone().into_account(); + + (account_id, signer) +} + +fn create_ed25519_signature(payload: &[u8], pubkey: MultiSigner) -> MultiSignature { + let edpubkey = ed25519::Public::try_from(pubkey).unwrap(); + let edsig = ed25519_sign(0.into(), &edpubkey, payload).unwrap(); + edsig.into() +} + +fn prepare_proposals( + n: u32, + caller: T::AccountId, + client: T::AccountId, + client_pair: MultiSigner, +) -> ( + Vec, BlockNumberFor, MultiSignature>>, + u32, +) +where + T: crate::Config, +{ + let mut small_rng = SmallRng::seed_from_u64(0x101097); + + let start_block: u32 = 50; + let end_block: u32 = 100; + let price_per_block: u32 = 10; + let cost = (end_block - start_block) * price_per_block * n; + + let mut proposals = vec![]; + for _ in 1..=n { + let label = vec![0; MAX_LABEL_SIZE as usize]; + + let proposal = DealProposal::, BlockNumberFor> { + piece_cid: random_cid(&mut small_rng).to_bytes().try_into().unwrap(), + piece_size: 2048, + client: client.clone(), + provider: caller.clone(), + label: BoundedVec::try_from(label).unwrap(), + start_block: start_block.into(), + end_block: end_block.into(), + storage_price_per_block: price_per_block.into(), + provider_collateral: COLLATERAL.into(), + state: DealState::<_>::Published, + }; + + let signed_deal_proposal = + create_ed25519_signature(proposal.encode().as_slice(), client_pair.clone()).into(); + + let proposal = ClientDealProposal::, BlockNumberFor, _> { + proposal, + client_signature: signed_deal_proposal, + }; + + proposals.push(proposal); + } + return (proposals, cost); +} + +#[frame_benchmarking::v2::benchmarks( + where + T: crate::Config, + T: pallet_balances::Config, + T: pallet_storage_provider::Config< + PeerId = BoundedPeerIdBytes, + >, + T: frame_system::Config +)] +mod benchmarks { + use itertools::Itertools; + + use super::*; + + const EXISTENTIAL_DEPOSIT: u32 = 1_000_000_000; + + fn setup_account_balance(account: T::AccountId) + where + T: crate::Config, + T: pallet_balances::Config, + { + pallet_balances::Pallet::::make_free_balance_be( + &account, + (EXISTENTIAL_DEPOSIT * 2).into(), + ); + + // Add some balance so we can withdraw it + MarketPallet::::add_balance( + RawOrigin::Signed(account.clone()).into(), + EXISTENTIAL_DEPOSIT.into(), + ) + .unwrap(); + + assert_eq!( + MarketPallet::::free(&account).unwrap(), + EXISTENTIAL_DEPOSIT.into() + ); + assert_eq!(MarketPallet::::locked(&account).unwrap(), 0u32.into()); + } + + /// Run until a particular block. + /// + /// Stolen't from: + pub fn run_to_block(n: u32) + where + T: crate::Config, + T: pallet_storage_provider::Config, + { + use frame_support::traits::Hooks; + + while frame_system::Pallet::::block_number() < n.into() { + if frame_system::Pallet::::block_number() > 1u32.into() { + pallet_storage_provider::Pallet::::on_finalize( + frame_system::Pallet::::block_number(), + ); + crate::Pallet::::on_finalize(frame_system::Pallet::::block_number()); + frame_system::Pallet::::on_finalize(frame_system::Pallet::::block_number()); + } + + frame_system::Pallet::::set_block_number( + frame_system::Pallet::::block_number() + 1u32.into(), + ); + frame_system::Pallet::::on_initialize(frame_system::Pallet::::block_number()); + crate::Pallet::::on_initialize(frame_system::Pallet::::block_number()); + pallet_storage_provider::Pallet::::on_initialize( + frame_system::Pallet::::block_number(), + ); + } + } + + #[benchmark] + fn add_balance() { + let caller = whitelisted_caller(); + + // `make_free_balance_be` returns an imbalance that gets automatically dropped here + // if not consumed by other functions, that imbalance updates the total issuance on drop + // as such, it should be dropped ASAP so that when a transfer occurs the issuance is "valid" + // otherwise, an underflow occurs during the transfer + // https://github.com/paritytech/polkadot-sdk/blob/721f6d97613b0ece9c8414e8ec8ba31d2f67d40c/substrate/frame/balances/src/impl_currency.rs#L328-L345 + // https://github.com/paritytech/polkadot-sdk/blob/721f6d97613b0ece9c8414e8ec8ba31d2f67d40c/substrate/frame/balances/src/impl_currency.rs#L222-L231 + // https://github.com/paritytech/polkadot-sdk/blob/6eca7647dc99dd0e78aacb740ba931e99e6ba71f/substrate/frame/support/src/traits/tokens/fungible/regular.rs#L317-L339 + // https://github.com/paritytech/polkadot-sdk/blob/721f6d97613b0ece9c8414e8ec8ba31d2f67d40c/substrate/frame/balances/src/impl_fungible.rs#L104-L151 + pallet_balances::Pallet::::make_free_balance_be( + &caller, + // Must add more than the amount that will be added to the market balance + // otherwise the account may not have enough to pay fees + (EXISTENTIAL_DEPOSIT * 2).into(), + ); + + // #[extrinsic_call] requires type shenanigans, using #[block] is MUCH simpler + #[block] + { + Pallet::::add_balance( + RawOrigin::Signed(caller.clone()).into(), + EXISTENTIAL_DEPOSIT.into(), + ) + .unwrap(); + } + + let balance_entry = BalanceTable::::get(&caller); + assert_eq!(balance_entry.free, EXISTENTIAL_DEPOSIT.into()); + assert_eq!(balance_entry.locked, 0u32.into()); + } + + #[benchmark] + fn withdraw_balance() { + let caller: T::AccountId = whitelisted_caller(); + pallet_balances::Pallet::::make_free_balance_be( + &caller, + (EXISTENTIAL_DEPOSIT * 2).into(), + ); + // Add some balance so we can withdraw it + Pallet::::add_balance( + RawOrigin::Signed(caller.clone().into()).into(), + EXISTENTIAL_DEPOSIT.into(), + ) + .unwrap(); + + // #[extrinsic_call] requires type shenanigans, using #[block] is MUCH simpler + #[block] + { + Pallet::::withdraw_balance( + RawOrigin::Signed(caller.clone()).into(), + EXISTENTIAL_DEPOSIT.into(), + ) + .unwrap(); + } + + let balance_entry = BalanceTable::::get(&caller); + assert_eq!(balance_entry.free, 0u32.into()); + assert_eq!(balance_entry.locked, 0u32.into()); + } + + // When running `cargo test` the first get's called, + // for the proper benchmarks we get the second + #[cfg(test)] + const MAX_DEALS: u32 = 32; + #[cfg(not(test))] + const MAX_DEALS: u32 = 128; + + /// `n`: number of submitted deals + #[benchmark] + fn publish_storage_deals(n: Linear<1, MAX_DEALS>) { + let caller: T::AccountId = whitelisted_caller(); + setup_account_balance::(caller.clone()); + // Register the caller as a storage provider + pallet_storage_provider::Pallet::::register_storage_provider( + RawOrigin::Signed(caller.clone()).into(), + BoundedVec::try_from("placeholder".as_bytes().to_vec()).unwrap(), + RegisteredPoStProof::StackedDRGWindow2KiBV1P1, + ) + .unwrap(); + + // Setup a client account + let (client, client_pair): (T::AccountId, _) = generate_benchmark_account("client"); + setup_account_balance::(client.clone()); + + let (proposals, cost) = + prepare_proposals::(n, caller.clone(), client.clone(), client_pair); + let proposals = BoundedVec::try_from(proposals).unwrap(); + + // #[extrinsic_call] requires type shenanigans, using #[block] is MUCH simpler + #[block] + { + Pallet::::publish_storage_deals(RawOrigin::Signed(caller.clone()).into(), proposals) + .unwrap(); + } + + let balance_entry = BalanceTable::::get(&caller); + assert_eq!( + balance_entry.free, + (EXISTENTIAL_DEPOSIT - (COLLATERAL * n)).into() + ); + assert_eq!(balance_entry.locked, (COLLATERAL * n).into()); + + let balance_entry = BalanceTable::::get(&client); + assert_eq!(balance_entry.free, (EXISTENTIAL_DEPOSIT - cost).into()); + assert_eq!(balance_entry.locked, cost.into()); + } + + /// `n`: number of submitted deals + #[benchmark] + fn settle_deal_payments(n: Linear<1, MAX_DEALS>) { + let caller: T::AccountId = whitelisted_caller(); + setup_account_balance::(caller.clone()); + // Register the caller as a storage provider + pallet_storage_provider::Pallet::::register_storage_provider( + RawOrigin::Signed(caller.clone()).into(), + BoundedVec::try_from("placeholder".as_bytes().to_vec()).unwrap(), + RegisteredPoStProof::StackedDRGWindow2KiBV1P1, + ) + .unwrap(); + + // Setup a client account + let (client, client_pair): (T::AccountId, _) = generate_benchmark_account("client"); + setup_account_balance::(client.clone()); + + let (proposals, cost) = + prepare_proposals::(n, caller.clone(), client.clone(), client_pair); + let proposals = BoundedVec::try_from(proposals).unwrap(); + + let storage_provider: OriginFor = RawOrigin::Signed(caller.clone()).into(); + Pallet::::publish_storage_deals(storage_provider.clone(), proposals.clone()).unwrap(); + + // Run to 1 to get VRF randomness + run_to_block::(1); + + proposals + .into_iter() + .enumerate() + // Create SectorPreCommitInfo from the proposals + .map(|(idx, p)| { + SectorPreCommitInfoBuilder::>::default() + .deals(vec![idx as u64]) + .sector_number( + (idx as u32) + .try_into() + .expect("n only goes up to 32 so this should be ok"), + ) + .raw_unsealed_cid(p.proposal.piece_cid.clone()) + .build() + }) + // Chunk into `MAX_DEALS` groups, required because of the cfg flag + // meaning that sometimes MAX_DEALS > MAX_SECTORS_PER_CALL + .chunks(MAX_SECTORS_PER_CALL as usize) + // Convert IntoChunks to Iterator + .into_iter() + // Convert the chunks into Vecs + .map(Vec::from_iter) + // Convert each chunk into a BoundedVec + .map(|sectors_for_call| { + BoundedVec::< + SectorPreCommitInfo>, + ConstU32 + >::try_from(sectors_for_call).unwrap() + }) + // Submit each "chunk" + .for_each(|pre_commit_infos| { + SpPallet::::pre_commit_sectors(storage_provider.clone(), pre_commit_infos) + .unwrap(); + }); + + // Run to 11 to enter pre-commit period + run_to_block::(11); + + // Similar pattern to before + (0..n) + // Create ProveCommitSectors from the `n` "index" + .map(|n| ProveCommitSector { + sector_number: n.try_into().unwrap(), + // Empty proof is considered valid under the DummyProofVerifier + proof: BoundedVec::new(), + }) + // Chunk into MAX_SECTORS_PER_CALL due to the cfg + .chunks(MAX_SECTORS_PER_CALL as usize) + // Convert IntoChunks to Iterator + .into_iter() + // Convert the chunks into Vecs + .map(Vec::from_iter) + // Convert the Vecs into BoundedVecs + .map(|prove_commit_sectors| { + BoundedVec::<_, ConstU32>::try_from(prove_commit_sectors) + .unwrap() + }) + // Submit them + .for_each(|prove_commit_sectors| { + SpPallet::::prove_commit_sectors(storage_provider.clone(), prove_commit_sectors) + .unwrap(); + }); + + run_to_block::(101); + + // We know that DealIds are incrementing integers, making this a "valid approach", + // it is kinda cheating but this way we don't need to care for the events + let deal_ids = BoundedVec::try_from(Vec::from_iter(0..(n as u64))).unwrap(); + + // #[extrinsic_call] requires type shenanigans, using #[block] is MUCH simpler + #[block] + { + Pallet::::settle_deal_payments(storage_provider.clone(), deal_ids).unwrap(); + } + + assert_eq!( + MarketPallet::::free(&caller).unwrap(), + (EXISTENTIAL_DEPOSIT + cost).into() + ); + assert_eq!(MarketPallet::::locked(&caller).unwrap(), 0u32.into()); + + assert_eq!( + MarketPallet::::free(&client).unwrap(), + (EXISTENTIAL_DEPOSIT - cost).into() + ); + assert_eq!(MarketPallet::::locked(&client).unwrap(), 0u32.into()); + } + + impl_benchmark_test_suite! { + MarketPallet, + crate::mock::new_test_ext(), + crate::mock::Test, + } +} diff --git a/pallets/market/src/lib.rs b/pallets/market/src/lib.rs index cc94430c6..9398df52f 100644 --- a/pallets/market/src/lib.rs +++ b/pallets/market/src/lib.rs @@ -10,6 +10,7 @@ pub use pallet::*; mod error; +pub mod weights; #[cfg(test)] mod mock; @@ -17,7 +18,9 @@ mod mock; #[cfg(test)] mod test; -// TODO(@th7nder,#77,14/06/2024): take the pallet out of dev mode +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; + #[frame_support::pallet(dev_mode)] pub mod pallet { use cid::Cid; @@ -47,13 +50,13 @@ pub mod pallet { pallets::{ActiveDeal, ActiveSector, Market, SectorDeal, StorageProviderValidation}, proofs::RegisteredSealProof, sector::{SectorNumber, SectorSize}, - DealId, MAX_DEALS_PER_SECTOR, MAX_SECTORS_PER_CALL, + DealId, CID_SIZE_IN_BYTES, MAX_DEALS_PER_SECTOR, MAX_LABEL_SIZE, MAX_SECTORS_PER_CALL, }; use scale_info::TypeInfo; use sp_arithmetic::traits::BaseArithmetic; use sp_std::vec::Vec; - use crate::error::*; + use crate::{error::*, weights::WeightInfo}; pub const LOG_TARGET: &'static str = "runtime::market"; @@ -70,6 +73,9 @@ pub mod pallet { /// The currency mechanism. type Currency: ReservableCurrency; + /// The pallet weights; + type WeightInfo: WeightInfo; + /// PalletId used to derive AccountId which stores funds of the Market Participants. #[pallet::constant] type PalletId: Get; @@ -220,7 +226,7 @@ pub mod pallet { // We use BoundedVec here, as cid::Cid do not implement `TypeInfo`, so it cannot be saved into the Runtime Storage. // It maybe doable using newtype pattern, however not sure how the UI on the frontend side would handle that anyways. // There is Encode/Decode implementation though, through the feature flag: `scale-codec`. - pub piece_cid: BoundedVec>, + pub piece_cid: BoundedVec>, /// The value represents the size of the data piece after padding to the /// nearest power of two. Padding ensures that all pieces can be /// efficiently arranged in a binary tree structure for Merkle proofs. @@ -231,7 +237,7 @@ pub mod pallet { pub provider: Address, /// Arbitrary client chosen label to apply to the deal - pub label: BoundedVec>, + pub label: BoundedVec>, /// Nominal start block. Deal payment is linear between StartBlock and EndBlock, /// with total amount StoragePricePerBlock * (EndBlock - StartBlock). @@ -274,7 +280,7 @@ pub mod pallet { ) } - fn piece_commitment(&self) -> Result, CommitmentError> { + pub fn piece_commitment(&self) -> Result, CommitmentError> { let commitment = Commitment::from_cid_bytes(&self.piece_cid[..])?; Ok(commitment) } @@ -306,7 +312,7 @@ pub mod pallet { /// `account(MarketPallet).balance == all_accounts.map(|balance| balance[account]].locked + balance[account].free).sum()` #[pallet::storage] pub type BalanceTable = - StorageMap<_, _, T::AccountId, BalanceEntry>, ValueQuery>; + StorageMap<_, Blake2_128Concat, T::AccountId, BalanceEntry>, ValueQuery>; /// Simple incremental ID generator for `Deal` Identification purposes. /// Starts as 0, increments once for each published deal. @@ -320,8 +326,12 @@ pub mod pallet { /// Deals are identified by `DealId`. /// Proposals are stored here until terminated and settled or expired (not activated in time). #[pallet::storage] - pub type Proposals = - StorageMap<_, _, DealId, DealProposal, BlockNumberFor>>; + pub type Proposals = StorageMap< + _, + Blake2_128Concat, + DealId, + DealProposal, BlockNumberFor>, + >; /// Stores Proposals which have been Published but not yet Activated. /// Only `T::MaxDeals` Pending Proposals can be held at any time. @@ -339,7 +349,7 @@ pub mod pallet { #[pallet::storage] pub type DealsForBlock = StorageMap< _, - _, + Blake2_128Concat, BlockNumberFor, BoundedBTreeSet, ValueQuery, @@ -349,7 +359,7 @@ pub mod pallet { #[pallet::storage] pub type SectorDeals = StorageMap< _, - _, + Blake2_128Concat, (T::AccountId, SectorNumber), BoundedVec>, >; @@ -520,6 +530,8 @@ pub mod pallet { /// Transfers `amount` of Balance from the `origin` to the Market Pallet account. /// It is marked as _free_ in the Market bookkeeping. /// Free balance can be withdrawn at any moment from the Market. + #[pallet::call_index(0)] + #[pallet::weight((T::WeightInfo::add_balance(), DispatchClass::Normal))] pub fn add_balance(origin: OriginFor, amount: BalanceOf) -> DispatchResult { let caller = ensure_signed(origin)?; @@ -543,6 +555,8 @@ pub mod pallet { /// Transfers `amount` of Balance from the Market Pallet account to the `origin`. /// Only _free_ balance can be withdrawn. + #[pallet::call_index(1)] + #[pallet::weight((T::WeightInfo::withdraw_balance(), DispatchClass::Normal))] pub fn withdraw_balance(origin: OriginFor, amount: BalanceOf) -> DispatchResult { let caller = ensure_signed(origin)?; @@ -566,6 +580,79 @@ pub mod pallet { Ok(()) } + /// Publish a new set of storage deals (not yet included in a sector). + /// It saves valid deals as [`DealState::Published`] and locks up client fees and provider's collaterals. + /// Locked up balances cannot be withdrawn until a deal is terminated. + /// All of the deals must belong to a single Storage Provider. + /// It is permissive, if some of the deals are correct and some are not, it emits events for valid deals. + /// On success emits [`Event::::DealPublished`] for each successful deal. + #[pallet::call_index(2)] + #[pallet::weight((T::WeightInfo::settle_deal_payments(deals.len() as u32), DispatchClass::Normal))] + pub fn publish_storage_deals( + origin: OriginFor, + deals: BoundedVec< + ClientDealProposal< + T::AccountId, + BalanceOf, + BlockNumberFor, + T::OffchainSignature, + >, + T::MaxDeals, + >, + ) -> DispatchResult { + let provider = ensure_signed(origin)?; + ensure!( + T::StorageProviderValidation::is_registered_storage_provider(&provider), + Error::::StorageProviderNotRegistered + ); + let current_block = >::block_number(); + let (valid_deals, total_provider_lockup) = + Self::validate_deals(provider.clone(), deals, current_block)?; + + let mut published_deals = BoundedVec::new(); + + // Lock up funds for the clients and emit events + for deal in valid_deals.into_iter() { + // PRE-COND: always succeeds, validated by `validate_deals` + let client_fee: BalanceOf = deal + .total_storage_fee() + .ok_or(Error::::UnexpectedValidationError)? + .try_into() + .map_err(|_| Error::::UnexpectedValidationError)?; + + // PRE-COND: always succeeds, validated by `validate_deals` + lock_funds::(&deal.client, client_fee)?; + + let deal_id = Self::generate_deal_id(); + + let mut deals_for_block = DealsForBlock::::get(&deal.start_block); + deals_for_block.try_insert(deal_id).map_err(|_| { + log::error!("there is not enough space to activate all of the deals at the given block {:?}", deal.start_block); + Error::::TooManyDealsPerBlock + })?; + DealsForBlock::::insert(deal.start_block, deals_for_block); + Proposals::::insert(deal_id, deal.clone()); + + // Only deposit the event after storing everything + // force_push is ok since the bound is the same as the input one + published_deals.force_push(PublishedDeal { + client: deal.client, + deal_id, + }); + } + + // Lock up funds for the Storage Provider + // PRE-COND: always succeeds, validated by `validate_deals` + lock_funds::(&provider, total_provider_lockup)?; + + Self::deposit_event(Event::::DealsPublished { + deals: published_deals, + provider, + }); + + Ok(()) + } + /// Settle pending deal payments for the given deal IDs. /// /// This function *should* only fully fail when a block was last updated after its `end_block` target. @@ -582,6 +669,8 @@ pub mod pallet { /// * The deal's last update is after the current block, meaning the deal's last update is in the future. /// The returned error is [`DealSettlementError::FutureLastUpdate`]. /// * The deal is not active + #[pallet::call_index(3)] + #[pallet::weight((T::WeightInfo::settle_deal_payments(deal_ids.len() as u32), DispatchClass::Normal))] pub fn settle_deal_payments( origin: OriginFor, // The original `deals` structure is a bitfield from fvm-ipld-bitfield @@ -713,77 +802,6 @@ pub mod pallet { Ok(()) } - - /// Publish a new set of storage deals (not yet included in a sector). - /// It saves valid deals as [`DealState::Published`] and locks up client fees and provider's collaterals. - /// Locked up balances cannot be withdrawn until a deal is terminated. - /// All of the deals must belong to a single Storage Provider. - /// It is permissive, if some of the deals are correct and some are not, it emits events for valid deals. - /// On success emits [`Event::::DealPublished`] for each successful deal. - pub fn publish_storage_deals( - origin: OriginFor, - deals: BoundedVec< - ClientDealProposal< - T::AccountId, - BalanceOf, - BlockNumberFor, - T::OffchainSignature, - >, - T::MaxDeals, - >, - ) -> DispatchResult { - let provider = ensure_signed(origin)?; - ensure!( - T::StorageProviderValidation::is_registered_storage_provider(&provider), - Error::::StorageProviderNotRegistered - ); - let current_block = >::block_number(); - let (valid_deals, total_provider_lockup) = - Self::validate_deals(provider.clone(), deals, current_block)?; - - let mut published_deals = BoundedVec::new(); - - // Lock up funds for the clients and emit events - for deal in valid_deals.into_iter() { - // PRE-COND: always succeeds, validated by `validate_deals` - let client_fee: BalanceOf = deal - .total_storage_fee() - .ok_or(Error::::UnexpectedValidationError)? - .try_into() - .map_err(|_| Error::::UnexpectedValidationError)?; - - // PRE-COND: always succeeds, validated by `validate_deals` - lock_funds::(&deal.client, client_fee)?; - - let deal_id = Self::generate_deal_id(); - - let mut deals_for_block = DealsForBlock::::get(&deal.start_block); - deals_for_block.try_insert(deal_id).map_err(|_| { - log::error!("there is not enough space to activate all of the deals at the given block {:?}", deal.start_block); - Error::::TooManyDealsPerBlock - })?; - DealsForBlock::::insert(deal.start_block, deals_for_block); - Proposals::::insert(deal_id, deal.clone()); - - // Only deposit the event after storing everything - // force_push is ok since the bound is the same as the input one - published_deals.force_push(PublishedDeal { - client: deal.client, - deal_id, - }); - } - - // Lock up funds for the Storage Provider - // PRE-COND: always succeeds, validated by `validate_deals` - lock_funds::(&provider, total_provider_lockup)?; - - Self::deposit_event(Event::::DealsPublished { - deals: published_deals, - provider, - }); - - Ok(()) - } } /// Functions exposed by the pallet diff --git a/pallets/market/src/mock.rs b/pallets/market/src/mock.rs index 60d71b776..ba80ee6af 100644 --- a/pallets/market/src/mock.rs +++ b/pallets/market/src/mock.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use codec::Encode; use frame_support::{ assert_ok, derive_impl, parameter_types, @@ -8,8 +10,9 @@ use frame_support::{ use frame_system::pallet_prelude::BlockNumberFor; use primitives::{proofs::RegisteredPoStProof, PEER_ID_MAX_BYTES}; use sp_core::Pair; +use sp_keystore::{testing::MemoryKeystore, KeystoreExt}; use sp_runtime::{ - traits::{ConstU32, ConstU64, IdentifyAccount, IdentityLookup, Verify, Zero}, + traits::{ConstU32, IdentifyAccount, IdentityLookup, Verify, Zero}, AccountId32, BuildStorage, MultiSignature, MultiSigner, }; @@ -54,32 +57,54 @@ parameter_types! { pub const MarketPalletId: PalletId = PalletId(*b"spMarket"); // Storage Provider Pallet - pub const WPoStPeriodDeadlines: u64 = 10; - pub const WpostProvingPeriod: BlockNumber = 40 * MINUTES; - pub const WpostChallengeWindow: BlockNumber = 4 * MINUTES; + pub const WpostProvingPeriod: BlockNumber = 6 * MINUTES; + pub const WPoStPeriodDeadlines: u64 = 3; + pub const WpostChallengeWindow: BlockNumber = 2 * MINUTES; pub const WpostChallengeLookBack: BlockNumber = MINUTES; pub const MinSectorExpiration: BlockNumber = 5 * MINUTES; - pub const MaxSectorExpiration: BlockNumber = 360 * MINUTES; + pub const MaxSectorExpiration: BlockNumber = 60 * MINUTES; pub const SectorMaximumLifetime: BlockNumber = 120 * MINUTES; pub const MaxProveCommitDuration: BlockNumber = 5 * MINUTES; pub const MaxPartitionsPerDeadline: u64 = 3000; pub const FaultMaxAge: BlockNumber = (5 * MINUTES) * 42; - pub const FaultDeclarationCutoff: BlockNumber = 2 * MINUTES; - pub const PreCommitChallengeDelay: BlockNumber = 1 * MINUTES; + pub const FaultDeclarationCutoff: BlockNumber = 1 * MINUTES; // pub const AddressedSectorsMax: u64 = 25_000; } +// NOTE(@jmg-duarte,20/01/2025): this is not ideal, however, in the name of time this is A solution +// the test parameters SHOULD be in sync with the parameters for the testnet configuration +// otherwise, it's impossible to run benchmarks on the node (to get weights) +// this BREAKS the normal tests when running benchmarks, as such you MUST run them in isolation +// cargo t -p pallet-market -F runtime-benchmarks -- bench +#[cfg(not(feature = "runtime-benchmarks"))] +parameter_types! { + pub const MinDealDuration: u64 = 2; + pub const MaxDealDuration: u64 = 30; + // 0 allows us to publish the prove-commit on the same block as the + // pre-commit. + pub const PreCommitChallengeDelay: BlockNumber = 0; +} + +#[cfg(feature = "runtime-benchmarks")] +parameter_types! { + pub const MinDealDuration: u64 = 5 * MINUTES; + pub const MaxDealDuration: u64 = 180 * MINUTES; + pub const PreCommitChallengeDelay: BlockNumber = 1 * MINUTES; +} + impl crate::Config for Test { type RuntimeEvent = RuntimeEvent; type PalletId = MarketPalletId; + type WeightInfo = (); + type Currency = Balances; type OffchainSignature = Signature; type OffchainPublic = AccountPublic; type StorageProviderValidation = StorageProvider; type MaxDeals = ConstU32<32>; - type MinDealDuration = ConstU64<2>; - type MaxDealDuration = ConstU64<30>; + type MinDealDuration = MinDealDuration; + type MaxDealDuration = MaxDealDuration; type MaxDealsPerBlock = ConstU32<32>; } @@ -204,6 +229,12 @@ pub fn new_test_ext() -> sp_io::TestExternalities { let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); + + // Required to perform signatures. Given that benchmarks run inside the runtime, this is how + // we're able to prepare signed client deal proposals. + let keystore = MemoryKeystore::new(); + ext.register_extension(KeystoreExt(Arc::new(keystore))); + ext } diff --git a/pallets/market/src/test.rs b/pallets/market/src/test.rs index d8be52648..b145bc71f 100644 --- a/pallets/market/src/test.rs +++ b/pallets/market/src/test.rs @@ -13,7 +13,7 @@ use primitives::{ pallets::{ActiveDeal, ActiveSector, Market as MarketTrait, SectorDeal}, proofs::RegisteredSealProof, sector::SectorNumber, - DealId, MAX_DEALS_PER_SECTOR, + DealId, CID_SIZE_IN_BYTES, MAX_DEALS_PER_SECTOR, }; use sp_core::H256; use sp_runtime::AccountId32; @@ -1756,7 +1756,7 @@ impl Default for SectorDealBuilder { /// Builder to simplify writing complex tests of [`DealProposal`]. /// Exclusively uses [`Test`] for simplification purposes. pub struct DealProposalBuilder { - piece_cid: BoundedVec>, + piece_cid: BoundedVec>, piece_size: u64, client: AccountIdOf, provider: AccountIdOf, diff --git a/pallets/market/src/weights.rs b/pallets/market/src/weights.rs new file mode 100644 index 000000000..81d6ae95b --- /dev/null +++ b/pallets/market/src/weights.rs @@ -0,0 +1,205 @@ + +//! Autogenerated weights for `pallet_market` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0 +//! DATE: 2025-01-24, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `parthenon`, CPU: `12th Gen Intel(R) Core(TM) i7-12700H` +//! WASM-EXECUTION: `Compiled`, CHAIN: `None`, DB CACHE: 1024 + +// Executed Command: +// target/release/polka-storage-node +// benchmark +// pallet +// --wasm-execution=compiled +// --pallet +// pallet_market +// --extrinsic +// * +// --steps +// 50 +// --repeat +// 20 +// --output +// pallets/market/src/weights.rs +// --template +// node/benchmark_template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +pub trait WeightInfo { + fn add_balance() -> Weight; + fn withdraw_balance() -> Weight; + fn publish_storage_deals(n: u32, ) -> Weight; + fn settle_deal_payments(n: u32, ) -> Weight; +} + +/// Weight functions for `pallet_market`. +pub struct Weights(PhantomData); +impl WeightInfo for Weights { + /// Storage: `Market::BalanceTable` (r:1 w:1) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + fn add_balance() -> Weight { + // Proof Size summary in bytes: + // Measured: `76` + // Estimated: `3593` + // Minimum execution time: 42_074_000 picoseconds. + Weight::from_parts(42_849_000, 0) + .saturating_add(Weight::from_parts(0, 3593)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `Market::BalanceTable` (r:1 w:1) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + fn withdraw_balance() -> Weight { + // Proof Size summary in bytes: + // Measured: `285` + // Estimated: `3750` + // Minimum execution time: 48_792_000 picoseconds. + Weight::from_parts(49_733_000, 0) + .saturating_add(Weight::from_parts(0, 3750)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `StorageProvider::StorageProviders` (r:1 w:0) + /// Proof: `StorageProvider::StorageProviders` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::BalanceTable` (r:2 w:2) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::PendingProposals` (r:1 w:1) + /// Proof: `Market::PendingProposals` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Market::NextDealId` (r:1 w:1) + /// Proof: `Market::NextDealId` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Market::DealsForBlock` (r:1 w:1) + /// Proof: `Market::DealsForBlock` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::Proposals` (r:0 w:128) + /// Proof: `Market::Proposals` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// The range of component `n` is `[1, 128]`. + fn publish_storage_deals(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `383` + // Estimated: `6323` + // Minimum execution time: 63_315_000 picoseconds. + Weight::from_parts(65_960_000, 0) + .saturating_add(Weight::from_parts(0, 6323)) + // Standard Error: 78_356 + .saturating_add(Weight::from_parts(50_851_959, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(6)) + .saturating_add(T::DbWeight::get().writes(5)) + .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) + } + /// Storage: `Market::Proposals` (r:128 w:128) + /// Proof: `Market::Proposals` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::BalanceTable` (r:2 w:2) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// The range of component `n` is `[1, 128]`. + fn settle_deal_payments(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `372 + n * (336 ±0)` + // Estimated: `6311 + n * (2812 ±0)` + // Minimum execution time: 28_818_000 picoseconds. + Weight::from_parts(28_773_328, 0) + .saturating_add(Weight::from_parts(0, 6311)) + // Standard Error: 21_440 + .saturating_add(Weight::from_parts(8_383_425, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes(2)) + .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2812).saturating_mul(n.into())) + } +} + +impl WeightInfo for () { + /// Storage: `Market::BalanceTable` (r:1 w:1) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + fn add_balance() -> Weight { + use frame_support::weights::constants::RocksDbWeight; + + // Proof Size summary in bytes: + // Measured: `76` + // Estimated: `3593` + // Minimum execution time: 42_074_000 picoseconds. + Weight::from_parts(42_849_000, 0) + .saturating_add(Weight::from_parts(0, 3593)) + .saturating_add(RocksDbWeight::get().reads(2)) + .saturating_add(RocksDbWeight::get().writes(2)) + } + /// Storage: `Market::BalanceTable` (r:1 w:1) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + fn withdraw_balance() -> Weight { + use frame_support::weights::constants::RocksDbWeight; + + // Proof Size summary in bytes: + // Measured: `285` + // Estimated: `3750` + // Minimum execution time: 48_792_000 picoseconds. + Weight::from_parts(49_733_000, 0) + .saturating_add(Weight::from_parts(0, 3750)) + .saturating_add(RocksDbWeight::get().reads(2)) + .saturating_add(RocksDbWeight::get().writes(2)) + } + /// Storage: `StorageProvider::StorageProviders` (r:1 w:0) + /// Proof: `StorageProvider::StorageProviders` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::BalanceTable` (r:2 w:2) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::PendingProposals` (r:1 w:1) + /// Proof: `Market::PendingProposals` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Market::NextDealId` (r:1 w:1) + /// Proof: `Market::NextDealId` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Market::DealsForBlock` (r:1 w:1) + /// Proof: `Market::DealsForBlock` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::Proposals` (r:0 w:128) + /// Proof: `Market::Proposals` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// The range of component `n` is `[1, 128]`. + fn publish_storage_deals(n: u32, ) -> Weight { + use frame_support::weights::constants::RocksDbWeight; + + // Proof Size summary in bytes: + // Measured: `383` + // Estimated: `6323` + // Minimum execution time: 63_315_000 picoseconds. + Weight::from_parts(65_960_000, 0) + .saturating_add(Weight::from_parts(0, 6323)) + // Standard Error: 78_356 + .saturating_add(Weight::from_parts(50_851_959, 0).saturating_mul(n.into())) + .saturating_add(RocksDbWeight::get().reads(6)) + .saturating_add(RocksDbWeight::get().writes(5)) + .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(n.into()))) + } + /// Storage: `Market::Proposals` (r:128 w:128) + /// Proof: `Market::Proposals` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Market::BalanceTable` (r:2 w:2) + /// Proof: `Market::BalanceTable` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// The range of component `n` is `[1, 128]`. + fn settle_deal_payments(n: u32, ) -> Weight { + use frame_support::weights::constants::RocksDbWeight; + + // Proof Size summary in bytes: + // Measured: `372 + n * (336 ±0)` + // Estimated: `6311 + n * (2812 ±0)` + // Minimum execution time: 28_818_000 picoseconds. + Weight::from_parts(28_773_328, 0) + .saturating_add(Weight::from_parts(0, 6311)) + // Standard Error: 21_440 + .saturating_add(Weight::from_parts(8_383_425, 0).saturating_mul(n.into())) + .saturating_add(RocksDbWeight::get().reads(2)) + .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(RocksDbWeight::get().writes(2)) + .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2812).saturating_mul(n.into())) + } +} diff --git a/pallets/storage-provider/src/tests/mod.rs b/pallets/storage-provider/src/tests/mod.rs index d40d3ab58..cdf96e950 100644 --- a/pallets/storage-provider/src/tests/mod.rs +++ b/pallets/storage-provider/src/tests/mod.rs @@ -12,7 +12,8 @@ use primitives::{ commitment::{CommP, Commitment}, proofs::RegisteredPoStProof, sector::SectorNumber, - PartitionNumber, MAX_PARTITIONS_PER_DEADLINE, MAX_TERMINATIONS_PER_CALL, PEER_ID_MAX_BYTES, + PartitionNumber, CID_SIZE_IN_BYTES, MAX_PARTITIONS_PER_DEADLINE, MAX_TERMINATIONS_PER_CALL, + PEER_ID_MAX_BYTES, }; use sp_arithmetic::traits::Zero; use sp_core::{bounded_vec, Pair}; @@ -79,6 +80,8 @@ impl pallet_balances::Config for Test { impl pallet_market::Config for Test { type RuntimeEvent = RuntimeEvent; type PalletId = MarketPalletId; + type WeightInfo = (); + type Currency = Balances; type OffchainSignature = Signature; type OffchainPublic = AccountPublic; @@ -148,12 +151,18 @@ where impl pallet_storage_provider::Config for Test { type RuntimeEvent = RuntimeEvent; + + // Randomness Provider type Randomness = DummyRandomnessGenerator; type AuthorVrfHistory = DummyRandomnessGenerator; + type PeerId = BoundedVec>; // https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type Currency = Balances; type Market = Market; + + // Proof Verification Provider type ProofVerification = primitives::testing::DummyProofsVerification; + type WPoStProvingPeriod = WPoStProvingPeriod; type WPoStChallengeWindow = WPoStChallengeWindow; type WPoStChallengeLookBack = WPoStChallengeLookBack; @@ -329,7 +338,7 @@ fn publish_deals(storage_provider: &str) { /// Builder to simplify writing complex tests of [`DealProposal`]. /// Exclusively uses [`Test`] for simplification purposes. struct DealProposalBuilder { - piece_cid: BoundedVec>, + piece_cid: BoundedVec>, piece_size: u64, client: AccountIdOf, provider: AccountIdOf, diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index 6e35c9e82..7a29b5c47 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -88,3 +88,8 @@ pub const MAX_POST_PROOF_BYTES: u32 = 192; /// References: /// * https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids pub const PEER_ID_MAX_BYTES: u32 = 42; + +/// The maximum size of a deal label. +/// +/// Arbitrary value. +pub const MAX_LABEL_SIZE: u32 = 128; diff --git a/primitives/src/sector/pre_commit.rs b/primitives/src/sector/pre_commit.rs index 987faa0b2..a373fb519 100644 --- a/primitives/src/sector/pre_commit.rs +++ b/primitives/src/sector/pre_commit.rs @@ -39,8 +39,7 @@ pub mod builder { use cid::Cid; use sp_core::ConstU32; - use sp_runtime::{BoundedVec, Vec}; - use sp_std::vec; + use sp_runtime::BoundedVec; use super::SectorPreCommitInfo; use crate::{ @@ -86,12 +85,15 @@ pub mod builder { .try_into() .expect("hash is always 32 bytes"); + let mut deal_ids = BoundedVec::new(); + deal_ids.try_push(0).unwrap(); + deal_ids.try_push(1).unwrap(); + Self { seal_proof: RegisteredSealProof::StackedDRG2KiBV1P1, sector_number: SectorNumber::new(1).unwrap(), sealed_cid, - deal_ids: BoundedVec::try_from(vec![0, 1]) - .expect("default valid should always be within bounds"), + deal_ids, expiration: 120u32.into(), unsealed_cid, seal_randomness_height: BlockNumber::one(), @@ -109,7 +111,7 @@ pub mod builder { } /// Panics if the length of `deal_ids` is larger than [`MAX_DEALS_PER_SECTOR`]. - pub fn deals(mut self, deal_ids: Vec) -> Self { + pub fn deals(mut self, deal_ids: sp_std::vec::Vec) -> Self { self.deal_ids = BoundedVec::try_from(deal_ids).unwrap(); self } @@ -119,9 +121,24 @@ pub mod builder { self } - pub fn unsealed_cid(mut self, unsealed_cid: &str) -> Self { + pub fn unsealed_cid(self, unsealed_cid: &str) -> Self { let cid = Cid::from_str(unsealed_cid).expect("valid unsealed_cid"); - self.unsealed_cid = BoundedVec::try_from(cid.to_bytes()).unwrap(); + self.raw_unsealed_cid(BoundedVec::try_from(cid.to_bytes()).unwrap()) + } + + pub fn raw_unsealed_cid( + mut self, + unsealed_cid: BoundedVec>, + ) -> Self { + self.unsealed_cid = unsealed_cid; + self + } + + pub fn raw_sealed_cid( + mut self, + sealed_cid: BoundedVec>, + ) -> Self { + self.sealed_cid = sealed_cid; self } diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 96a090245..a058e2c87 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -173,6 +173,7 @@ runtime-benchmarks = [ "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", + "primitives/testing", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", diff --git a/runtime/src/benchmarks.rs b/runtime/src/benchmarks.rs index 258786e29..944d86ded 100644 --- a/runtime/src/benchmarks.rs +++ b/runtime/src/benchmarks.rs @@ -36,4 +36,5 @@ frame_benchmarking::define_benchmarks!( // Our crates [pallet_randomness, Randomness] [pallet_proofs, Proofs] + [pallet_market, Market] ); diff --git a/runtime/src/configs/mod.rs b/runtime/src/configs/mod.rs index fdc358615..4290751e7 100644 --- a/runtime/src/configs/mod.rs +++ b/runtime/src/configs/mod.rs @@ -352,27 +352,90 @@ parameter_types! { pub const MaxPartitionsPerDeadline: u64 = 3000; pub const FaultMaxAge: BlockNumber = (5 * MINUTES) * 42; pub const FaultDeclarationCutoff: BlockNumber = 1 * MINUTES; - pub const PreCommitChallengeDelay: BlockNumber = 1 * MINUTES; + // pub const AddressedSectorsMax: u64 = 25_000; // Market Pallet pub const MinDealDuration: u64 = 5 * MINUTES; pub const MaxDealDuration: u64 = 180 * MINUTES; +} +// NOTE(@jmg-duarte,22/01/2025): The following bit of code is confusing BUT +// using #[cfg(...)] inside parameter_types results in errors +// feature set == {testnet} +#[cfg(all(feature = "testnet", not(feature = "runtime-benchmarks")))] +parameter_types! { + pub const PreCommitChallengeDelay: BlockNumber = 1 * MINUTES; +} +// feature set = {testnet, runtime-benchmarks} +// used for benchmarking +#[cfg(all(feature = "testnet", feature = "runtime-benchmarks"))] +parameter_types! { + pub const PreCommitChallengeDelay: BlockNumber = 0; +} - // Faucet pallet - pub const FaucetDripAmount: Balance = 10_000_000_000_000; - pub const FaucetDripDelay: BlockNumber = DAYS; +#[cfg(feature = "runtime-benchmarks")] +mod dummy { + use frame_support::pallet_prelude::Zero; + use frame_system::pallet_prelude::BlockNumberFor; + /// Randomness generator used by tests. + pub struct DummyRandomnessGenerator(core::marker::PhantomData) + where + C: frame_system::Config; + + impl frame_support::traits::Randomness> + for DummyRandomnessGenerator + where + C: frame_system::Config, + { + fn random(_subject: &[u8]) -> (C::Hash, BlockNumberFor) { + ( + Default::default(), + >::block_number(), + ) + } + } + + impl primitives::randomness::AuthorVrfHistory, C::Hash> + for DummyRandomnessGenerator + where + C: frame_system::Config, + { + fn author_vrf_history(block_number: BlockNumberFor) -> Option { + if block_number == as Zero>::zero() { + None + } else { + Some(Default::default()) + } + } + } } impl pallet_storage_provider::Config for Runtime { type RuntimeEvent = RuntimeEvent; + + #[cfg(not(feature = "runtime-benchmarks"))] type Randomness = crate::Randomness; + #[cfg(not(feature = "runtime-benchmarks"))] type AuthorVrfHistory = crate::Randomness; + + #[cfg(feature = "runtime-benchmarks")] + type Randomness = dummy::DummyRandomnessGenerator; + #[cfg(feature = "runtime-benchmarks")] + type AuthorVrfHistory = dummy::DummyRandomnessGenerator; + type PeerId = BoundedVec>; // https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#peer-ids type Currency = Balances; type Market = crate::Market; + + #[cfg(not(feature = "runtime-benchmarks"))] type ProofVerification = crate::Proofs; + // FIX(@jmg-duarte,#695,22/1/25) + // It is true that this stops the weight of the proving process from being calculated + // but (right now) we cannot create a proof inside a benchmark, so this is the best we can do + #[cfg(feature = "runtime-benchmarks")] + type ProofVerification = primitives::testing::DummyProofsVerification; + type WPoStProvingPeriod = WpostProvingPeriod; type WPoStChallengeWindow = WpostChallengeWindow; type WPoStChallengeLookBack = WPoStChallengeLookBack; @@ -398,9 +461,11 @@ parameter_types! { pub type AccountPublic = ::Signer; impl pallet_market::Config for Runtime { + type PalletId = MarketPalletId; type RuntimeEvent = RuntimeEvent; + type WeightInfo = pallet_market::weights::Weights; + type Currency = Balances; - type PalletId = MarketPalletId; type OffchainSignature = MultiSignature; type OffchainPublic = AccountPublic; type StorageProviderValidation = crate::StorageProvider; @@ -421,6 +486,13 @@ impl pallet_randomness::Config for Runtime { type WeightInfo = pallet_randomness::weights::Weights; } +#[cfg(feature = "testnet")] +parameter_types! { + // Faucet pallet + pub const FaucetDripAmount: Balance = 10_000_000_000_000; + pub const FaucetDripDelay: BlockNumber = DAYS; +} + #[cfg(feature = "testnet")] impl pallet_faucet::Config for Runtime { type RuntimeEvent = RuntimeEvent; diff --git a/storagext/lib/Cargo.toml b/storagext/lib/Cargo.toml index 0357c7442..6f1d998bf 100644 --- a/storagext/lib/Cargo.toml +++ b/storagext/lib/Cargo.toml @@ -19,7 +19,7 @@ codec.workspace = true frame-support = { workspace = true, features = ["std"] } futures.workspace = true hex = { workspace = true, features = ["serde"] } -itertools = { workspace = true } +itertools = { workspace = true, default-features = true } primitives = { workspace = true, features = ["serde", "std"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true }