From 3887a877cbaf3be7c7ed7db4c1c83f81ad9c05fb Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sun, 18 Feb 2024 00:23:45 +0800 Subject: [PATCH] update db exporter resource --- Cargo.lock | 1 + cmd/db-exporter/Cargo.toml | 1 + cmd/db-exporter/src/main.rs | 180 ++++++++++++++---------------- cmd/resource-exporter/src/main.rs | 11 -- 4 files changed, 85 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4954b5d70a..c53b658ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2139,6 +2139,7 @@ dependencies = [ "starcoin-genesis", "starcoin-logger", "starcoin-resource-viewer", + "starcoin-rpc-api", "starcoin-state-tree", "starcoin-statedb", "starcoin-storage", diff --git a/cmd/db-exporter/Cargo.toml b/cmd/db-exporter/Cargo.toml index ee4b2dfb82..d77f374cb5 100644 --- a/cmd/db-exporter/Cargo.toml +++ b/cmd/db-exporter/Cargo.toml @@ -35,6 +35,7 @@ starcoin-vm-runtime = { workspace = true } futures = { workspace = true } rayon = { workspace = true } num_cpus = { workspace = true } +starcoin-rpc-api = { workspace = true } [package] authors = { workspace = true } diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index 98b9d02f2c..b2647bda65 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, format_err, Result}; -use bcs_ext::{BCSCodec, Sample}; +use bcs_ext::Sample; use clap::{IntoApp, Parser}; use csv::Writer; use db_exporter::{ @@ -22,6 +22,8 @@ use starcoin_consensus::Consensus; use starcoin_crypto::HashValue; use starcoin_genesis::Genesis; use starcoin_resource_viewer::{AnnotatedMoveStruct, AnnotatedMoveValue, MoveValueAnnotator}; +use starcoin_rpc_api::types::StrView; +use starcoin_state_tree::StateTree; use starcoin_statedb::{ChainStateDB, ChainStateReader, ChainStateWriter}; use starcoin_storage::{ block::FailedBlock, @@ -36,6 +38,7 @@ use starcoin_storage::{ use starcoin_transaction_builder::{ build_signed_empty_txn, create_signed_txn_with_association_account, DEFAULT_MAX_GAS_AMOUNT, }; +use starcoin_types::account_state::AccountState; use starcoin_types::{ account::{peer_to_peer_txn, Account, DEFAULT_EXPIRATION_TIME}, account_address::AccountAddress, @@ -45,12 +48,12 @@ use starcoin_types::{ transaction::Transaction, }; use starcoin_vm_runtime::starcoin_vm::StarcoinVM; +use starcoin_vm_types::access_path::DataType; use starcoin_vm_types::{ account_config::stc_type_tag, genesis_config::ConsensusStrategy, identifier::Identifier, language_storage::{ModuleId, StructTag}, - parser::parse_struct_tag, transaction::{ScriptFunction, SignedUserTransaction, TransactionPayload}, }; use std::{ @@ -58,7 +61,7 @@ use std::{ fmt::{Debug, Formatter}, fs::{File, OpenOptions}, io::{BufRead, BufReader, Write}, - path::{Path, PathBuf}, + path::PathBuf, str::FromStr, sync::{ atomic::{AtomicU64, Ordering}, @@ -392,29 +395,24 @@ pub struct ApplySnapshotOptions { #[derive(Debug, Clone, Parser)] #[clap(name = "export-resource", about = "onchain resource exporter")] pub struct ExportResourceOptions { + #[clap(long, short = 'n')] + /// Chain Network, like main, proxima + pub net: BuiltinNetworkID, #[clap(long, short = 'o', parse(from_os_str))] /// output file, like accounts.csv pub output: PathBuf, #[clap(long, short = 'i', parse(from_os_str))] - /// starcoin node db path. like ~/.starcoin/barnard/starcoindb/db/starcoindb + /// starcoin node db path. like ~/.starcoin/barnard pub db_path: PathBuf, - #[clap(long)] - /// block hash of the snapshot. - pub block_hash: HashValue, + #[clap(long, short = 'b')] + pub block_number: Option, #[clap( - short='r', - default_value = "0x1::Account::Balance<0x1::STC::STC>", - parse(try_from_str=parse_struct_tag) + help = "resource struct tag,", + default_value = "0x1::Account::Balance<0x1::STC::STC>" )] - /// resource struct tag. - resource_type: StructTag, - - #[clap(min_values = 1, required = true)] - /// fields of the struct to output. it use pointer syntax of serde_json. - /// like: /authentication_key /sequence_number /deposit_events/counter /token/value - pub fields: Vec, + resource_type: StrView, } #[derive(Debug, Parser)] @@ -574,16 +572,12 @@ async fn main() -> anyhow::Result<()> { Cmd::ExportResource(option) => { #[cfg(target_os = "linux")] let guard = pprof::ProfilerGuard::new(100).unwrap(); - let output = option.output.as_path(); - let block_hash = option.block_hash; - let resource = option.resource_type.clone(); - // let result = apply_block(option.to_path, option.input_path, option.net, verifier); export_resource( - option.db_path.display().to_string().as_str(), - output, - block_hash, - resource, - option.fields.as_slice(), + option.db_path, + option.output, + option.net, + option.block_number, + option.resource_type.0.clone(), )?; #[cfg(target_os = "linux")] if let Ok(report) = guard.report().build() { @@ -1818,14 +1812,15 @@ pub struct AccountData { } pub fn export_resource( - db: &str, - output: &Path, - block_hash: HashValue, + from_dir: PathBuf, + output: PathBuf, + network: BuiltinNetworkID, + block_number: Option, resource_struct_tag: StructTag, - fields: &[String], ) -> anyhow::Result<()> { + let net = ChainNetwork::new_builtin(network); let db_storage = DBStorage::open_with_cfs( - db, + from_dir.join("starcoindb/db/starcoindb"), StorageVersion::current_version() .get_column_family_names() .to_vec(), @@ -1833,85 +1828,76 @@ pub fn export_resource( Default::default(), None, )?; - let storage = Storage::new(StorageInstance::new_db_instance(db_storage))?; - let storage = Arc::new(storage); - let block = storage - .get_block(block_hash)? - .ok_or_else(|| anyhow::anyhow!("block {} not exist", block_hash))?; + let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance( + CacheStorage::new(None), + db_storage, + ))?); + let (chain_info, _) = + Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?; + let chain = BlockChain::new( + net.time_service(), + chain_info.head().id(), + storage.clone(), + None, + ) + .expect("create block chain should success."); + let cur_num = block_number.unwrap_or(chain_info.head().number()); + let block = chain + .get_block_by_number(cur_num)? + .ok_or_else(|| format_err!("get block by number {} error", cur_num))?; let root = block.header.state_root(); - let statedb = ChainStateDB::new(storage, Some(root)); + let statedb = ChainStateDB::new(storage.clone(), Some(root)); let value_annotator = MoveValueAnnotator::new(&statedb); - let mut csv_writer = csv::WriterBuilder::new().from_path(output)?; + let state_tree = StateTree::::new(storage.clone(), Some(root)); - // write csv header. - { - csv_writer.write_field("address")?; - for f in fields { - csv_writer.write_field(f)?; - } - csv_writer.write_record(None::<&[u8]>)?; - } - - use std::time::Instant; + let mut file = File::create(output)?; - let now = Instant::now(); - - let global_states_iter = statedb.dump_iter()?; - println!("t1: {}", now.elapsed().as_millis()); + let global_states = state_tree.dump()?; + use std::time::Instant; let now = Instant::now(); - let mut t1_sum = 0; - let mut t2_sum = 0; - let mut loop_count = 0; - let mut now3 = Instant::now(); - let mut iter_time = 0; - for (account_address, account_state_set) in global_states_iter { - iter_time += now3.elapsed().as_nanos(); - - loop_count += 1; - - let now1 = Instant::now(); - let resource_set = account_state_set.resource_set().unwrap(); - // t1_sum += now1.elapsed().as_micros(); - t1_sum += now1.elapsed().as_nanos(); - // println!("t1 sum in loop: {}", t1_sum); - - let now2 = Instant::now(); - for (k, v) in resource_set.iter() { - let struct_tag = StructTag::decode(k.as_slice())?; - if struct_tag == resource_struct_tag { - let annotated_struct = - value_annotator.view_struct(resource_struct_tag.clone(), v.as_slice())?; - let resource_struct = annotated_struct; - let resource_json_value = serde_json::to_value(MoveStruct(resource_struct))?; - let resource = Some(resource_json_value); - let record: Option> = resource - .as_ref() - .map(|v| fields.iter().map(|f| v.pointer(f.as_str())).collect()); - if let Some(mut record) = record { - let account_value = serde_json::to_value(account_address).unwrap(); - record.insert(0, Some(&account_value)); - csv_writer.serialize(record)?; + let mut sum: u128 = 0; + for (address_bytes, account_state_bytes) in global_states.iter() { + let account: AccountAddress = bcs_ext::from_bytes(address_bytes)?; + let account_state: AccountState = account_state_bytes.as_slice().try_into()?; + let resource_root = account_state.storage_roots()[DataType::RESOURCE.storage_index()]; + let resource = match resource_root { + None => None, + Some(root) => { + let account_tree = StateTree::::new(storage.clone(), Some(root)); + let data = account_tree.get(&resource_struct_tag)?; + + if let Some(d) = data { + let annotated_struct = + value_annotator.view_struct(resource_struct_tag.clone(), d.as_slice())?; + let resource = annotated_struct; + let resource_json_value = serde_json::to_value(MoveStruct(resource))?; + Some(resource_json_value) + } else { + None } - break; + } + }; + if let Some(res) = resource { + let balance = (res + .get("token") + .unwrap() + .get("value") + .unwrap() + .as_f64() + .unwrap() + / 1000000000.0) as u128; + if balance > 0 { + writeln!(file, "{} {}", account, balance)?; + sum += balance; } } - // let d = now2.elapsed().as_micros(); - let d = now2.elapsed().as_nanos(); - // println!("d is: {}", d); - t2_sum += d; - // println!("t2 sum in loop: {}", t2_sum); - now3 = Instant::now(); } - println!("iter time: {}, {}", iter_time, iter_time / 1_000_000); - println!("loop count: {}", loop_count); - println!("t1_sum: {}, {}", t1_sum, t1_sum / 1000000); - println!("t2_sum: {}, {}", t2_sum, t2_sum / 1000000); - println!("t2: {}", now.elapsed().as_millis()); - csv_writer.flush()?; + writeln!(file, "total {}", sum)?; + file.flush()?; Ok(()) } diff --git a/cmd/resource-exporter/src/main.rs b/cmd/resource-exporter/src/main.rs index 7051eec265..5399f99c5a 100644 --- a/cmd/resource-exporter/src/main.rs +++ b/cmd/resource-exporter/src/main.rs @@ -160,17 +160,6 @@ impl serde::Serialize for MoveValue { } } -#[derive(Parser)] -struct Opt { - #[clap(subcommand)] - cmd: Option, -} - -#[derive(Parser)] -enum Cmd { - Exporter(ExporterOptions), -} - #[derive(Debug, Clone, Parser)] #[clap(name = "resource-exporter", about = "onchain resource exporter")] pub struct ExporterOptions {