diff --git a/Cargo.lock b/Cargo.lock index 9e2e8d7..035ca09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1234,7 +1234,7 @@ checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "client" -version = "0.0.6" +version = "0.0.7" dependencies = [ "anyhow", "bigdecimal", @@ -1243,6 +1243,8 @@ dependencies = [ "common", "hex", "reqwest", + "reqwest-middleware", + "reqwest-retry", "sp-core 33.0.1", "sp-runtime 37.0.0", "ss58-registry", @@ -1256,7 +1258,7 @@ dependencies = [ [[package]] name = "client-interface" -version = "0.0.6" +version = "0.0.7" dependencies = [ "anyhow", "common", @@ -1275,6 +1277,7 @@ dependencies = [ "subxt-signer", "thiserror", "tokio", + "tracing", ] [[package]] @@ -1285,7 +1288,7 @@ checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "common" -version = "0.0.6" +version = "0.0.7" dependencies = [ "aws-nitro-enclaves-cose", "aws-nitro-enclaves-nsm-api", @@ -1822,7 +1825,7 @@ dependencies = [ [[package]] name = "enclave" -version = "0.0.6" +version = "0.0.7" dependencies = [ "anyhow", "aws-nitro-enclaves-nsm-api", @@ -1844,7 +1847,7 @@ dependencies = [ [[package]] name = "enclave-interface" -version = "0.0.6" +version = "0.0.7" dependencies = [ "common", "nix 0.27.1", @@ -2186,8 +2189,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2767,6 +2772,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -3505,6 +3513,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -3512,7 +3531,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -3523,7 +3556,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", "windows-targets 0.52.5", ] @@ -3889,6 +3922,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.1" @@ -4010,6 +4052,52 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39346a33ddfe6be00cbc17a34ce996818b97b230b87229f10114693becca1268" +dependencies = [ + "anyhow", + "async-trait", + "http 1.1.0", + "reqwest", + "serde", + "thiserror", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2a94ba69ceb30c42079a137e2793d6d0f62e581a24c06cd4e9bb32e973c7da" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "futures", + "getrandom", + "http 1.1.0", + "hyper 1.3.1", + "parking_lot 0.11.2", + "reqwest", + "reqwest-middleware", + "retry-policies", + "tokio", + "tracing", + "wasm-timer", +] + +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -4615,11 +4703,12 @@ dependencies = [ [[package]] name = "service" -version = "0.0.6" +version = "0.0.7" dependencies = [ "anyhow", "aws-config", "aws-sdk-dynamodb", + "aws-smithy-runtime-api", "axum", "cfg-if", "clap", @@ -4840,7 +4929,7 @@ dependencies = [ "log", "lru", "no-std-net", - "parking_lot", + "parking_lot 0.12.3", "pin-project", "rand", "rand_chacha", @@ -4961,7 +5050,7 @@ dependencies = [ "merlin", "parity-bip39", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "paste", "primitive-types", "rand", @@ -5008,7 +5097,7 @@ dependencies = [ "merlin", "parity-bip39", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "paste", "primitive-types", "rand", @@ -5140,7 +5229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdbab8b61bd61d5f8625a0c75753b5d5a23be55d3445419acd42caf59cf6236b" dependencies = [ "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "sp-core 31.0.0", "sp-externalities 0.27.0", ] @@ -5152,7 +5241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92a909528663a80829b95d582a20dd4c9acd6e575650dee2bcaf56f4740b305e" dependencies = [ "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "sp-core 33.0.1", "sp-externalities 0.28.0", ] @@ -5282,7 +5371,7 @@ dependencies = [ "hash-db", "log", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "rand", "smallvec", "sp-core 31.0.0", @@ -5304,7 +5393,7 @@ dependencies = [ "hash-db", "log", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "rand", "smallvec", "sp-core 33.0.1", @@ -5386,7 +5475,7 @@ dependencies = [ "memory-db", "nohash-hasher", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "rand", "scale-info", "schnellru", @@ -5411,7 +5500,7 @@ dependencies = [ "memory-db", "nohash-hasher", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.3", "rand", "scale-info", "schnellru", @@ -5522,6 +5611,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stress-tool" +version = "0.0.7" +dependencies = [ + "anyhow", + "clap", + "client", + "client-interface", + "common", + "rand", + "reqwest", + "sp-runtime 37.0.0", + "subxt-signer", + "tokio", +] + [[package]] name = "strsim" version = "0.10.0" @@ -6527,6 +6632,21 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmi" version = "0.31.2" diff --git a/Cargo.toml b/Cargo.toml index 0a14d75..00c4bd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,19 @@ [workspace] -members = [ "common", "service", "enclave", "enclave-interface", "client", "client-interface" ] +members = [ + "common", + "service", + "enclave", + "enclave-interface", + "client", + "client-interface", + "stress-tool" +] resolver = "2" [workspace.package] homepage = "https://projectglove.io/" repository = "https://github.com/projectglove/glove-monorepo/" -version = "0.0.6" +version = "0.0.7" [workspace.dependencies] anyhow = "1.0.86" @@ -26,7 +34,9 @@ parity-scale-codec = "3.6.12" bigdecimal = "0.4.3" clap = { version = "4.5.4", features = ["derive"] } tokio = { version = "1.37.0" } -reqwest = { version = "0.12.4", features = ["json"] } +reqwest = { version = "0.12.4" } +reqwest-middleware = "0.3.2" +reqwest-retry = "0.6.0" strum = { version = "0.26.2", features = ["derive"] } axum = "0.7.5" itertools = "0.13.0" @@ -45,3 +55,4 @@ sha2 = "0.10.8" flate2 = "1.0.30" aws-config = "1.5.4" aws-sdk-dynamodb = "1.38.0" +aws-smithy-runtime-api = "1.7.1" diff --git a/README.md b/README.md index c20213e..c8dac06 100644 --- a/README.md +++ b/README.md @@ -145,12 +145,6 @@ The Glove proxy account address. Users will need to first assign this account as The substrate-based network the Glove service is connected to. -#### `node_endpoint` - -The [node endpoint](https://wiki.polkadot.network/docs/maintain-endpoints) URL the service is using to interact with the -network. This is only provided as a convenience for Glove clients, otherwise they can use any node endpoint as long as -it points to the same network. - #### `attestation_bundle` The attestation bundle of the enclave the service is using. This is a hex-encoded string (without the `0x` prefix), @@ -170,7 +164,6 @@ The version of the Glove service. { "proxy_account": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", "network_name": "rococo", - "node_endpoint": "wss://rococo-rpc.polkadot.io", "attestation_bundle": "6408de7737c59c238890533af25896a2c20608d8b380bb01029acb3927...", "version": "0.0.4" } @@ -211,8 +204,9 @@ If the vote request was successfully received and accepted by the service then a code is returned. This does not mean, however, the vote was mixed and submitted on-chain; just that the Glove service will do so at the appropriate time. -If there was something wrong with the vote request then a `400 Bad Request` is returned with a JSON object containing -the error type (`error`) and description (`description`). +If request body is invalid then a `422 Unprocessable Entity` is returned. If there was something wrong with the +request itself then a `400 Bad Request` is returned with a JSON object containing the error type (`error`) and +description (`description`). ## `POST /remove-vote` @@ -239,8 +233,9 @@ i.e. the `request` field without the hex-encoding. An empty response with `200 OK` status code is returned if the previous vote was successfully removed or if there was no matching vote. -If there was something wrong with the request itself then a `400 Bad Request` is returned with a JSON object containing -the error type (`error`) and description (`description`). +If request body is invalid then a `422 Unprocessable Entity` is returned. If there was something wrong with the +request itself then a `400 Bad Request` is returned with a JSON object containing the error type (`error`) and +description (`description`). # Client CLI diff --git a/client-interface/Cargo.toml b/client-interface/Cargo.toml index b92d959..5c63ffb 100644 --- a/client-interface/Cargo.toml +++ b/client-interface/Cargo.toml @@ -20,7 +20,8 @@ serde.workspace = true parity-scale-codec.workspace = true tokio = { workspace = true, features = ["sync"] } serde_with.workspace = true -reqwest.workspace = true +reqwest = { workspace = true, features = ["json"] } +tracing.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/client-interface/src/lib.rs b/client-interface/src/lib.rs index d74918d..f20fbb6 100644 --- a/client-interface/src/lib.rs +++ b/client-interface/src/lib.rs @@ -98,6 +98,9 @@ impl SubstrateNetwork { } } + // TODO The substrate node endpoints are proving to be very unreliable when it comes to looking + // up old blocks. This is happening to both Rococo and Kusama. Replace uses of this function + // with the equivalent subscan one. pub async fn get_extrinsic( &self, location: ExtrinsicLocation @@ -392,7 +395,6 @@ pub fn account_to_subxt_multi_address(account: AccountId32) -> SubxtMultiAddress pub struct ServiceInfo { pub proxy_account: AccountId32, pub network_name: String, - pub node_endpoint: String, #[serde(with = "common::serde_over_hex_scale")] pub attestation_bundle: AttestationBundle, pub version: String @@ -439,7 +441,6 @@ mod tests { let service_info = ServiceInfo { proxy_account: dev::alice().public_key().0.into(), network_name: "polkadot".to_string(), - node_endpoint: "wss://polkadot.api.onfinality.io/public-ws".to_string(), attestation_bundle: AttestationBundle { attested_data: AttestedData { genesis_hash: random::<[u8; 32]>().into(), @@ -459,7 +460,6 @@ mod tests { json!({ "proxy_account": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY", "network_name": "polkadot", - "node_endpoint": "wss://polkadot.api.onfinality.io/public-ws", "attestation_bundle": hex::encode(&service_info.attestation_bundle.encode()), "version": "1.0.0" }) diff --git a/client-interface/src/subscan.rs b/client-interface/src/subscan.rs index add52a3..8502749 100644 --- a/client-interface/src/subscan.rs +++ b/client-interface/src/subscan.rs @@ -1,44 +1,126 @@ +use std::fmt::Debug; +use std::time::Duration; + +use reqwest::header::{HeaderMap, RETRY_AFTER}; use serde::{Deserialize, Serialize}; +use serde::de::DeserializeOwned; use serde_with::DisplayFromStr; use serde_with::serde_as; use sp_runtime::AccountId32; +use tokio::time::sleep; +use tracing::warn; use common::ExtrinsicLocation; -pub async fn get_votes( - http_client: &reqwest::Client, - network_name: &str, - poll_index: u32, - account: Option -) -> Result, reqwest::Error> { - let mut all_votes = Vec::new(); - - let mut request = Request { - referendum_index: poll_index, - account, - valid: Valid::Valid, - row: 100, - page: 0, - }; - - loop { - let response = http_client - .post(&format!("https://{}.api.subscan.io/api/scan/referenda/votes", network_name)) - .json(&request) - .send().await? - .json::().await?; - let Some(mut votes) = response.data.list else { - break; +#[derive(Clone)] +pub struct Subscan { + http_client: reqwest::Client, + network: String, +} + +impl Subscan { + pub fn new(network: String) -> Self { + Self { + http_client: reqwest::Client::new(), + network, + } + } + + pub async fn get_votes( + &self, + poll_index: u32, + account: Option + ) -> Result, Error> { + let mut all_votes = Vec::new(); + + let mut request = VotesRequest { + referendum_index: poll_index, + account, + valid: Valid::Valid, + row: 100, + page: 0, }; - all_votes.append(&mut votes); - request.page += 1; + + loop { + let (headers, votes_response) = self.api_call_for::( + "referenda/votes", + &request + ).await?; + let Some(data) = votes_response.data else { + handle_error(headers, &votes_response.api_response, &request).await?; + continue; + }; + let Some(mut votes) = data.list else { + break; + }; + all_votes.append(&mut votes); + request.page += 1; + } + + Ok(all_votes) } - Ok(all_votes) + pub async fn get_extrinsic( + &self, + extrinsic_location: ExtrinsicLocation + ) -> Result, Error> { + let request = ExtrinsicRequest { + events_limit: 1, + extrinsic_index: extrinsic_location, + only_extrinsic_event: true, + }; + loop { + let (headers, extrinsic_response) = self.api_call_for::( + "extrinsic", + &request + ).await?; + if extrinsic_response.api_response.code != 0 { + handle_error(headers, &extrinsic_response.api_response, &request).await?; + continue; + } + return Ok(extrinsic_response.data); + } + } + + async fn api_call_for( + &self, + end_point: &str, + request: &(impl Serialize + ?Sized) + ) -> Result<(HeaderMap, Resp), Error> { + let http_response = self.http_client + .post(format!("https://{}.api.subscan.io/api/scan/{}", self.network, end_point)) + .json(&request) + .send().await?; + let headers = http_response.headers().clone(); + let response = http_response.json::().await?; + Ok((headers, response)) + } +} + +async fn handle_error( + headers: HeaderMap, + api_response: &ApiResponse, + request: &impl Debug +) -> Result<(), Error> { + let retry_after = headers + .get(RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + match retry_after { + Some(retry_after) => { + warn!("Rate limited, retrying after {} seconds ({:?})", retry_after, request); + sleep(Duration::from_secs(retry_after)).await; + Ok(()) + } + None => Err(Error::Api { + code: api_response.code, + message: api_response.message.clone(), + }) + } } #[derive(Debug, Clone, Serialize)] -struct Request { +struct VotesRequest { referendum_index: u32, account: Option, valid: Valid, @@ -46,6 +128,15 @@ struct Request { row: u8, } +#[serde_as] +#[derive(Debug, Clone, Serialize)] +struct ExtrinsicRequest { + events_limit: u32, + #[serde_as(as = "DisplayFromStr")] + extrinsic_index: ExtrinsicLocation, + only_extrinsic_event: bool +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "snake_case")] enum Valid { @@ -53,12 +144,20 @@ enum Valid { } #[derive(Debug, Clone, Deserialize)] -struct Response { - data: Data, +struct ApiResponse { + code: i64, + message: String } #[derive(Debug, Clone, Deserialize)] -struct Data { +struct VotesResponse { + #[serde(flatten)] + api_response: ApiResponse, + data: Option, +} + +#[derive(Debug, Clone, Deserialize)] +struct VotesData { list: Option>, } @@ -74,3 +173,26 @@ pub struct ConvictionVote { pub struct Account { pub address: AccountId32, } + +#[derive(Debug, Clone, Deserialize)] +struct ExtrinsicResponse { + #[serde(flatten)] + api_response: ApiResponse, + data: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ExtrinsicDetail { + pub account_display: Option +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Reqwest error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error("API error: ({code}) {message}")] + Api { + code: i64, + message: String, + } +} diff --git a/client/Cargo.toml b/client/Cargo.toml index e93a096..126ae67 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -16,6 +16,8 @@ subxt-core.workspace = true subxt-signer.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } reqwest.workspace = true +reqwest-middleware = { workspace = true, features = ["json"] } +reqwest-retry.workspace = true strum.workspace = true sp-core.workspace = true sp-runtime.workspace = true diff --git a/client/src/lib.rs b/client/src/lib.rs index f94a2a4..3774eab 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,250 +1,489 @@ -use std::collections::HashMap; +use std::ffi::OsString; +use std::process::{ExitCode, Termination}; +use std::time::Duration; +use anyhow::{anyhow, bail, Context, Result}; +use bigdecimal::{BigDecimal, ToPrimitive}; +use clap::{Parser, Subcommand}; +use DispatchError::Module; +use reqwest::{Client, StatusCode, Url}; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use reqwest_retry::{Jitter, RetryTransientMiddleware}; +use reqwest_retry::policies::ExponentialBackoff; use sp_core::crypto::AccountId32; -use subxt::Error as SubxtError; - -use attestation::EnclaveInfo; -use AttestationBundleLocation::SubstrateRemark; -use client_interface::{account, ExtrinsicDetails, SubstrateNetwork}; -use client_interface::metadata::runtime_types; -use client_interface::metadata::runtime_types::frame_system::pallet::Call as SystemCall; -use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Call as ConvictionVotingCall; -use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::AccountVote; -use client_interface::metadata::runtime_types::pallet_proxy::pallet::Call as ProxyCall; -use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall; -use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall::ConvictionVoting; -use client_interface::metadata::system::calls::types::Remark; -use client_interface::metadata::utility::calls::types::Batch; -use common::{AssignedBalance, attestation, BASE_AYE, Conviction, ExtrinsicLocation, GloveResult, VoteDirection}; -use common::attestation::{AttestationBundle, AttestationBundleLocation, AttestedData, GloveProof, GloveProofLite}; -use runtime_types::pallet_conviction_voting::vote::Vote; - -#[derive(Debug, Clone, PartialEq)] -pub struct VerifiedGloveProof { - pub result: GloveResult, - /// If `None` then enclave was running in insecure mode. - pub enclave_info: Option, - pub attested_data: AttestedData -} - -// TODO API for checking EnclaveInfo for expected measurements -impl VerifiedGloveProof { - pub fn get_assigned_balance(&self, account: &AccountId32) -> Option { - self.result - .assigned_balances - .iter() - .find(|assigned_balance| assigned_balance.account == *account) - .cloned() +use sp_core::Encode; +use sp_runtime::MultiSignature; +use strum::Display; +use subxt::error::DispatchError; +use subxt::Error::Runtime; +use subxt_signer::sr25519::Keypair; + +use client_interface::{account_to_subxt_multi_address, CallableSubstrateNetwork, is_glove_member, SignedRemoveVoteRequest, SubstrateNetwork}; +use client_interface::metadata::referenda::storage::types::referendum_info_for::ReferendumInfoFor; +use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::Duplicate; +use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::NotFound; +use client_interface::metadata::runtime_types::polkadot_runtime::{ProxyType, RuntimeError}; +use client_interface::RemoveVoteRequest; +use client_interface::ServiceInfo; +use client_interface::subscan::Subscan; +use Command::{Info, JoinGlove, LeaveGlove, RemoveVote, VerifyVote, Vote}; +use common::{attestation, Conviction, SignedVoteRequest, VoteRequest}; +use common::attestation::{Attestation, EnclaveInfo}; +use RuntimeError::Proxy; + +use crate::verify::{Error, try_verify_glove_result}; + +pub mod verify; + +pub async fn run(input: I) -> Result +where + I: IntoIterator, + T: Into + Clone +{ + let args = Args::parse_from(input); + + let retry_policy = ExponentialBackoff::builder() + .retry_bounds(Duration::from_secs(1), Duration::from_secs(60)) + .jitter(Jitter::Bounded) + .base(2) + .build_with_total_retry_duration(Duration::from_secs(2 * 60)); + let http_client = ClientBuilder::new(Client::builder().build()?) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) + .build(); + + let service_info = http_client + .get(url_with_path(&args.glove_url, "info")) + .send().await? + .error_for_status()? + .json::().await?; + + match args.command { + JoinGlove(cmd) => join_glove(service_info, cmd).await, + Vote(cmd) => vote(service_info, cmd, args.glove_url, http_client).await, + RemoveVote(cmd) => remove_vote(service_info, cmd, args.glove_url, http_client).await, + VerifyVote(cmd) => verify_vote(service_info, cmd).await, + LeaveGlove(cmd) => leave_glove(service_info, cmd).await, + Info => info(service_info) } +} - pub fn get_vote_balance(&self, account: &AccountId32, nonce: u32) -> Option { - self.result - .assigned_balances - .iter() - .find_map(|ab| (ab.account == *account && ab.nonce == nonce).then_some(ab.balance)) +async fn join_glove(service_info: ServiceInfo, cmd: JoinCmd) -> Result { + let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; + if is_glove_member(&network, network.account(), service_info.proxy_account.clone()).await? { + return Ok(SuccessOutput::AlreadyGloveMember); + } + let add_proxy_call = client_interface::metadata::tx() + .proxy() + .add_proxy(account_to_subxt_multi_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) + .unvalidated(); + match network.call_extrinsic(&add_proxy_call).await { + Ok(_) => Ok(SuccessOutput::JoinedGlove), + Err(Runtime(Module(module_error))) => { + match module_error.as_root_error::() { + // Unlikely, but just in case + Ok(Proxy(Duplicate)) => Ok(SuccessOutput::AlreadyGloveMember), + _ => Err(Runtime(Module(module_error)).into()) + } + }, + Err(e) => Err(e.into()) } } -/// A result of `Ok(None)` means the extrinsic was not a Glove result. -pub async fn try_verify_glove_result( - network: &SubstrateNetwork, - extrinsic: &ExtrinsicDetails, - proxy_account: &AccountId32, - poll_index: u32 -) -> Result, Error> { - let Some((glove_proof_lite, batch)) = parse_glove_proof_lite(extrinsic, proxy_account)? else { - // This extrinsic is not a Glove proof - return Ok(None); - }; - let glove_result = &glove_proof_lite.signed_result.result; +async fn vote( + service_info: ServiceInfo, + cmd: VoteCmd, + glove_url: Url, + http_client: ClientWithMiddleware, +) -> Result { + let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; + let balance = (&cmd.balance * 10u128.pow(network.token.decimals as u32)) + .to_u128() + .context("Vote balance is too big")?; + let request = VoteRequest::new( + network.account(), + network.api.genesis_hash(), + cmd.poll_index, + cmd.aye, + balance, + cmd.parse_conviction()? + ); + let nonce = request.nonce; - if glove_result.poll_index != poll_index { - // This proof is for another poll and so let's return early and avoid unnecessary - // processing, and also avoid any potential errors which wouldn't be relevant to the caller. - return Ok(None); + let signature = MultiSignature::Sr25519(network.account_key.sign(&request.encode()).0.into()); + let signed_request = SignedVoteRequest { request, signature }; + if !signed_request.verify() { + bail!("Something has gone wrong with the signature") } + let response = http_client + .post(url_with_path(&glove_url, "vote")) + .json(&signed_request) + .send().await + .context("Unable to send vote request")?; - let account_votes: HashMap> = batch.calls - .iter() - .filter_map(|call| parse_and_validate_proxy_account_vote(call, glove_result.poll_index)) - .collect::>(); - // TODO Check for duplicate on-chain votes for the same account - - // Make sure each assigned balance from the Glove proof is accounted for on-chain. - for assigned_balance in &glove_result.assigned_balances { - account_votes.get(&assigned_balance.account) - .filter(|&&account_vote| { - is_account_vote_consistent(account_vote, glove_result.direction, assigned_balance) - }) - .ok_or(Error::InconsistentVotes)?; + if response.status() != StatusCode::OK { + bail!(response.text().await?) + } + if cmd.await_glove_proof { + listen_for_glove_votes(&network, &cmd, nonce, &service_info.proxy_account).await?; } + return Ok(SuccessOutput::Voted { nonce }); +} - // It's technically possible for there to be more on-chain votes from the same proxy, for the - // same poll, which are not in the proof. This is not something the client has to worry about, - // since they can only confirm their vote request was included in the proof. +// TODO Stop waiting when the poll is closed. +async fn listen_for_glove_votes( + network: &CallableSubstrateNetwork, + vote_cmd: &VoteCmd, + nonce: u32, + proxy_account: &AccountId32 +) -> Result<()> { + network.subscribe_successful_extrinsics(|extrinsic, _| async move { + let verification_result = try_verify_glove_result( + &network, + &extrinsic, + proxy_account, + vote_cmd.poll_index, + ).await; + let verified_glove_proof = match verification_result { + Ok(None) => return Ok(()), // Not what we're looking for + Ok(Some(verified_glove_proof)) => verified_glove_proof, + Err(Error::Subxt(subxt_error)) => return Err(subxt_error.into()), + Err(error) => { + eprintln!("Error verifying Glove proof: {}", error); + return Ok(()); + } + }; + if let Some(balance) = verified_glove_proof.get_vote_balance(&network.account(), nonce) { + println!("Glove vote {:?} with balance {}", + verified_glove_proof.result.direction, network.token.amount(balance)); + if let Some(_) = &verified_glove_proof.enclave_info { + // TODO Check measurement + } else { + eprintln!("WARNING: Secure enclave wasn't used"); + } + } else { + eprintln!("WARNING: Received Glove proof for poll, but vote was not included"); + } + Ok(()) + }).await?; + Ok(()) +} - let attestation_bundle = match glove_proof_lite.attestation_location { - SubstrateRemark(remark_location) => - get_attestation_bundle_from_remark(network, remark_location).await? +async fn remove_vote( + service_info: ServiceInfo, + cmd: RemoveVoteCmd, + glove_url: Url, + http_client: ClientWithMiddleware, +) -> Result { + let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; + let request = RemoveVoteRequest { + account: network.account(), + poll_index: cmd.poll_index }; - - if attestation_bundle.attested_data.genesis_hash != network.api.genesis_hash() { - return Err(Error::ChainMismatch); + let signature = MultiSignature::Sr25519(network.account_key.sign(&request.encode()).0.into()); + let signed_request = SignedRemoveVoteRequest { request, signature }; + if !signed_request.verify() { + bail!("Something has gone wrong with the signature") } + let response = http_client + .post(url_with_path(&glove_url, "remove-vote")) + .json(&signed_request) + .send().await + .context("Unable to send remove vote request")?; + if response.status() == StatusCode::OK { + Ok(SuccessOutput::VoteRemoved) + } else { + bail!(response.text().await?) + } +} - let glove_proof = GloveProof { - signed_result: glove_proof_lite.signed_result, - attestation_bundle +async fn verify_vote(service_info: ServiceInfo, cmd: VerifyVoteCmd) -> Result { + let network = SubstrateNetwork::connect(node_endpoint(&service_info.network_name)).await?; + let Some(poll_info) = network.get_poll(cmd.poll_index).await? else { + bail!("Poll does not exist") }; - - let enclave_info = match glove_proof.verify() { - Ok(enclave_info) => Some(enclave_info), - Err(attestation::Error::InsecureMode) => None, - Err(error) => return Err(error.into()) + let subscan = Subscan::new(service_info.network_name.clone()); + let votes = subscan.get_votes(cmd.poll_index, Some(cmd.account.clone())).await?; + let Some(vote) = votes.first() else { + if matches!(poll_info, ReferendumInfoFor::Ongoing(_)) { + bail!("Glove proxy has not voted yet") + } else { + bail!("Poll is no longer active and Glove proxy did not vote") + } + }; + let Some(extrinsic) = network.get_extrinsic(vote.extrinsic_index).await? else { + bail!("Unable to find vote extrinsic at {}", vote.extrinsic_index) + }; + let verification_result = try_verify_glove_result( + &network, + &extrinsic, + &service_info.proxy_account, + cmd.poll_index + ).await; + let verified_glove_proof = match verification_result { + Ok(Some(verified_glove_proof)) => verified_glove_proof, + Ok(None) => bail!("Vote was not cast by Glove proxy"), + Err(error) => bail!("Glove proof failed verification: {}", error) + }; + let assigned_balance = verified_glove_proof + .get_assigned_balance(&cmd.account) + .ok_or_else(|| anyhow!("Account is not in Glove proof"))?; + let image_measurement = match verified_glove_proof.enclave_info { + Some(EnclaveInfo::Nitro(nitro_enclave_info)) => nitro_enclave_info.image_measurement, + None => bail!("INSECURE enclave was used to mix votes, so result cannot be trusted") }; - Ok(Some(VerifiedGloveProof { - result: glove_proof.signed_result.result, - enclave_info, - attested_data: glove_proof.attestation_bundle.attested_data - })) + if let Some(nonce) = cmd.nonce { + if nonce != assigned_balance.nonce { + bail!("Nonce in Glove proof ({}) does not match expected value. \ + Glove proxy has used an older vote request.", assigned_balance.nonce) + } + } else { + eprintln!("Nonce was not provided so cannot check if most recent vote request was used by \ + Glove proxy"); + } + + if !cmd.enclave_measurement.is_empty() { + let enclave_match = cmd.enclave_measurement + .iter() + .any(|str| hex::decode(str).ok() == Some(image_measurement.clone())); + if !enclave_match { + bail!("Unknown enclave encountered in Glove proof ({})", + hex::encode(&image_measurement)) + } + println!("Vote mixed by VERIFIED Glove enclave: {:?} with {} and conviction {:?}", + verified_glove_proof.result.direction, + network.token.amount(assigned_balance.balance), + assigned_balance.conviction); + } else { + println!("Vote mixed by POSSIBLE Glove enclave ({}): {:?} with {} and conviction {:?}", + hex::encode(&image_measurement), + verified_glove_proof.result.direction, + network.token.amount(assigned_balance.balance), + assigned_balance.conviction); + println!(); + println!("To verify this is a Glove enclave, first audit the code:"); + println!("git clone --depth 1 --branch v{} {}", + verified_glove_proof.attested_data.version, + env!("CARGO_PKG_REPOSITORY")); + println!(); + println!("And then verify 'PCR0' output is '{}':", hex::encode(&image_measurement)); + println!("./build.sh"); + } + + Ok(SuccessOutput::None) } -fn parse_glove_proof_lite( - extrinsic: &ExtrinsicDetails, - proxy_account: &AccountId32 -) -> Result, SubxtError> { - let from_proxy = account(extrinsic).filter(|account| account == proxy_account).is_some(); - if !from_proxy { - return Ok(None); +// TODO Also remove any active votes, which requires a remove-all-votes request? +async fn leave_glove(service_info: ServiceInfo, cmd: LeaveCmd) -> Result { + let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; + if !is_glove_member(&network, network.account(), service_info.proxy_account.clone()).await? { + return Ok(SuccessOutput::NotGloveMember); + } + let add_proxy_call = client_interface::metadata::tx() + .proxy() + .remove_proxy(account_to_subxt_multi_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) + .unvalidated(); + match network.call_extrinsic(&add_proxy_call).await { + Ok(_) => Ok(SuccessOutput::LeftGlove), + Err(Runtime(Module(module_error))) => { + match module_error.as_root_error::() { + // Unlikely, but just in case + Ok(Proxy(NotFound)) => Ok(SuccessOutput::NotGloveMember), + _ => Err(Runtime(Module(module_error)).into()) + } + }, + Err(e) => Err(e.into()) } +} - let Some(batch) = extrinsic.as_extrinsic::()? else { - return Ok(None); +fn info(service_info: ServiceInfo) -> Result { + let ab = &service_info.attestation_bundle; + let enclave_info = match ab.verify() { + Ok(EnclaveInfo::Nitro(enclave_info)) => { + &format!("AWS Nitro Enclave ({})", hex::encode(enclave_info.image_measurement)) + }, + Err(attestation::Error::InsecureMode) => match ab.attestation { + Attestation::Nitro(_) => "Debug AWS Nitro Enclave (INSECURE)", + Attestation::Mock => "Mock (INSECURE)" + }, + Err(attestation_error) => &format!("Error verifying attestation: {}", attestation_error) }; - let remarks = batch.calls - .iter() - .filter_map(|call| match call { - RuntimeCall::System(SystemCall::remark { remark }) => Some(remark), - _ => None - }) - .collect::>(); - - // Expecting there to be exactly one remark call - let &[remark] = remarks.as_slice() else { - return Ok(None); - }; + println!("Glove proxy account: {}", service_info.proxy_account); + println!("Enclave: {}", enclave_info); + println!("Substrate Network: {}", service_info.network_name); + println!("Genesis hash: {}", hex::encode(ab.attested_data.genesis_hash)); - Ok(GloveProofLite::decode_envelope(remark).map(|proof| (proof, batch)).ok()) + Ok(SuccessOutput::None) } -fn parse_and_validate_proxy_account_vote( - call: &RuntimeCall, - expected_poll_index: u32, -) -> Option<(AccountId32, &AccountVote)> { - let RuntimeCall::Proxy(proxy_call) = call else { - return None; - }; - let ProxyCall::proxy { real, force_proxy_type: _, call: proxied_call } = proxy_call else { - return None; - }; - let subxt_core::utils::MultiAddress::Id(real_account) = real else { - return None; - }; - let ConvictionVoting(ConvictionVotingCall::vote { poll_index, ref vote }) = **proxied_call else { - return None; - }; - if expected_poll_index != poll_index { - return None; +pub fn url_with_path(url: &Url, path: &str) -> Url { + let mut with_path = url.clone(); + with_path.set_path(path); + with_path +} + +// TODO This doesn't work for the Polkadot relay chain +pub fn node_endpoint(network_name: &str) -> String { + format!("wss://{}-rpc.polkadot.io", network_name) +} + +#[derive(Debug, Parser)] +#[command(version, about = "Glove CLI client")] +struct Args { + /// The URL of the Glove service + #[arg(long, short, verbatim_doc_comment)] + glove_url: Url, + + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Clone, clap::Args)] +struct SecretPhraseArgs { + /// The secret phrase for the Glove client account. This is a secret seed with optional + /// derivation paths. An Sr25519 key will be derived from this for signing. + /// + /// See https://wiki.polkadot.network/docs/learn-account-advanced#derivation-paths for more + /// details. + #[arg(long, verbatim_doc_comment, value_parser = client_interface::parse_secret_phrase)] + secret_phrase: Keypair +} + +impl SecretPhraseArgs { + async fn connect_to_network( + &self, + service_info: &ServiceInfo + ) -> Result { + CallableSubstrateNetwork::connect( + node_endpoint(&service_info.network_name), + self.secret_phrase.clone() + ).await } - Some((real_account.0.into(), vote)) -} - -fn is_account_vote_consistent( - account_vote: &AccountVote, - direction: VoteDirection, - assigned_balance: &AssignedBalance -) -> bool { - match direction { - VoteDirection::Aye => { - parse_standard_account_vote(account_vote) == - Some((true, assigned_balance.balance, assigned_balance.conviction)) - }, - VoteDirection::Nay => { - parse_standard_account_vote(account_vote) == - Some((false, assigned_balance.balance, assigned_balance.conviction)) - }, - VoteDirection::Abstain => { - parse_abstain_account_vote(account_vote) == Some(assigned_balance.balance) +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Add Glove as a goverance proxy to the account, if it isn't already. + #[command(verbatim_doc_comment)] + JoinGlove(JoinCmd), + /// Submit vote for inclusion in Glove mixing. The mixing process is not necessarily immediate. + /// Voting on the same poll twice will replace the previous vote. + #[command(verbatim_doc_comment)] + Vote(VoteCmd), + /// Remove a previously submitted vote. + #[command(verbatim_doc_comment)] + RemoveVote(RemoveVoteCmd), + /// Verify on-chain vote was mixed by a genuine Glove enclave + #[command(verbatim_doc_comment)] + VerifyVote(VerifyVoteCmd), + /// Remove the account from the Glove proxy. + #[command(verbatim_doc_comment)] + LeaveGlove(LeaveCmd), + /// Print information about the Glove service. + #[command(verbatim_doc_comment)] + Info +} + +#[derive(Debug, Parser)] +struct JoinCmd { + #[command(flatten)] + secret_phrase_args: SecretPhraseArgs +} + +#[derive(Debug, Parser)] +struct VoteCmd { + #[command(flatten)] + secret_phrase_args: SecretPhraseArgs, + #[arg(long, short)] + poll_index: u32, + /// Specify this to vote "aye", ommit to vote "nay" + #[arg(long, verbatim_doc_comment)] + aye: bool, + /// The amount of tokens to lock for the vote (as a decimal in the major token unit) + #[arg(long, short, verbatim_doc_comment)] + balance: BigDecimal, + /// The vote conviction multiplier + #[arg(long, short, verbatim_doc_comment, default_value_t = 0)] + conviction: u8, + /// Wait for the vote to be included in the Glove mixing process and confirmation received. + #[arg(long, short, verbatim_doc_comment)] + await_glove_proof: bool +} + +impl VoteCmd { + fn parse_conviction(&self) -> Result { + match self.conviction { + 0 => Ok(Conviction::None), + 1 => Ok(Conviction::Locked1x), + 2 => Ok(Conviction::Locked2x), + 3 => Ok(Conviction::Locked3x), + 4 => Ok(Conviction::Locked4x), + 5 => Ok(Conviction::Locked5x), + 6 => Ok(Conviction::Locked6x), + _ => bail!("Conviction must be between 0 and 6 inclusive") } } } -fn parse_standard_account_vote(vote: &AccountVote) -> Option<(bool, u128, Conviction)> { - let AccountVote::Standard { vote: Vote(direction), balance } = vote else { - return None; - }; - if *direction >= BASE_AYE { - Some((true, *balance, parse_conviction(*direction - BASE_AYE)?)) - } else { - Some((false, *balance, parse_conviction(*direction)?)) - } +#[derive(Debug, Parser)] +struct RemoveVoteCmd { + #[command(flatten)] + secret_phrase_args: SecretPhraseArgs, + #[arg(long, short)] + poll_index: u32 } -fn parse_conviction(offset: u8) -> Option { - match offset { - 0 => Some(Conviction::None), - 1 => Some(Conviction::Locked1x), - 2 => Some(Conviction::Locked2x), - 3 => Some(Conviction::Locked3x), - 4 => Some(Conviction::Locked4x), - 5 => Some(Conviction::Locked5x), - 6 => Some(Conviction::Locked6x), - _ => None - } +#[derive(Debug, Parser)] +struct VerifyVoteCmd { + /// The account on whose behalf the Glove proxy mixed the vote + #[arg(long, short, verbatim_doc_comment)] + account: AccountId32, + /// The index of the poll/referendum + #[arg(long, short, verbatim_doc_comment)] + poll_index: u32, + /// Whitelisted Glove enclave measurements. Each measurement represents a different enclave + /// version. The on-chain Glove proof associated with the vote will be checked against this + /// list. It is assumed the versions of the enclave these measurement represent have been + /// audited. + /// + /// If no enclave measurement is specified, the measurement of the Glove proof will displayed, + /// along with enclave code location, for auditing. + #[arg(long, short, verbatim_doc_comment)] + enclave_measurement: Vec, + /// Optional, the nonce value used in the most recent vote request. + #[arg(long, short, verbatim_doc_comment)] + nonce: Option } -fn parse_abstain_account_vote(account_vote: &AccountVote) -> Option { - match account_vote { - AccountVote::SplitAbstain { aye: 0, nay: 0, abstain } => Some(*abstain), - _ => None - } +#[derive(Debug, Parser)] +struct LeaveCmd { + #[command(flatten)] + secret_phrase_args: SecretPhraseArgs } -async fn get_attestation_bundle_from_remark( - network: &SubstrateNetwork, - remark_location: ExtrinsicLocation -) -> Result { - network.get_extrinsic(remark_location).await? - .ok_or_else(|| Error::ExtrinsicNotFound(remark_location))? - .as_extrinsic::()? - .ok_or_else(|| Error::InvalidAttestationBundle( - format!("Extrinsic at location {:?} is not a Remark", remark_location) - )) - .and_then(|remark| { - AttestationBundle::decode_envelope(&remark.remark).map_err(|scale_error| { - Error::InvalidAttestationBundle( - format!("Error decoding attestation bundle: {}", scale_error) - ) - }) - }) -} - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Subxt error: {0}")] - Subxt(#[from] SubxtError), - #[error("Votes are inconsistent with the Glove proof")] - InconsistentVotes, - #[error("Glove proof is for different chain")] - ChainMismatch, - #[error("Extrinsic at location {0} does not exist")] - ExtrinsicNotFound(ExtrinsicLocation), - #[error("Invalid attestation bundle: {0}")] - InvalidAttestationBundle(String), - #[error("Invalid attestation: {0}")] - Attestation(#[from] attestation::Error) -} - -// TODO Tests +#[derive(Display, Debug, PartialEq)] +pub enum SuccessOutput { + #[strum(to_string = "Account has joined Glove proxy")] + JoinedGlove, + #[strum(to_string = "Account already a member of Glove proxy")] + AlreadyGloveMember, + #[strum(to_string = "Vote successfully submitted ({nonce})")] + Voted { nonce: u32 }, + #[strum(to_string = "Vote successfully removed")] + VoteRemoved, + #[strum(to_string = "Account has left Glove proxy")] + LeftGlove, + #[strum(to_string = "Account was not a Glove proxy member")] + NotGloveMember, + None +} + +impl Termination for SuccessOutput { + fn report(self) -> ExitCode { + if self != SuccessOutput::None { + println!("{}", self); + } + ExitCode::SUCCESS + } +} diff --git a/client/src/main.rs b/client/src/main.rs index 369c853..6edc7b4 100644 --- a/client/src/main.rs +++ b/client/src/main.rs @@ -1,473 +1,10 @@ -use std::process::{ExitCode, Termination}; +use std::env; -use anyhow::{anyhow, bail, Context, Result}; -use bigdecimal::{BigDecimal, ToPrimitive}; -use clap::{Parser, Subcommand}; -use DispatchError::Module; -use reqwest::{Client, StatusCode, Url}; -use sp_core::crypto::AccountId32; -use sp_core::Encode; -use sp_runtime::MultiSignature; -use strum::Display; -use subxt::error::DispatchError; -use subxt::Error::Runtime; -use subxt_signer::sr25519::Keypair; +use anyhow::Result; -use client::{Error, try_verify_glove_result}; -use client_interface::{account_to_subxt_multi_address, CallableSubstrateNetwork, is_glove_member, SignedRemoveVoteRequest, subscan, SubstrateNetwork}; -use client_interface::metadata::referenda::storage::types::referendum_info_for::ReferendumInfoFor; -use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::Duplicate; -use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::NotFound; -use client_interface::metadata::runtime_types::polkadot_runtime::{ProxyType, RuntimeError}; -use client_interface::RemoveVoteRequest; -use client_interface::ServiceInfo; -use Command::{Info, JoinGlove, LeaveGlove, RemoveVote, VerifyVote, Vote}; -use common::{attestation, Conviction, SignedVoteRequest, VoteRequest}; -use common::attestation::{Attestation, EnclaveInfo}; -use RuntimeError::Proxy; +use client::{run, SuccessOutput}; #[tokio::main] async fn main() -> Result { - let args = Args::parse(); - - let http_client = Client::builder().build()?; - - let service_info = http_client - .get(url_with_path(&args.glove_url, "info")) - .send().await? - .error_for_status()? - .json::().await?; - - match args.command { - JoinGlove(cmd) => join_glove(service_info, cmd).await, - Vote(cmd) => vote(service_info, cmd, args.glove_url, http_client).await, - RemoveVote(cmd) => remove_vote(service_info, cmd, args.glove_url, http_client).await, - VerifyVote(cmd) => verify_vote(service_info, cmd, http_client).await, - LeaveGlove(cmd) => leave_glove(service_info, cmd).await, - Info => info(service_info) - } -} - -async fn join_glove(service_info: ServiceInfo, cmd: JoinCmd) -> Result { - let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; - if is_glove_member(&network, network.account(), service_info.proxy_account.clone()).await? { - return Ok(SuccessOutput::AlreadyGloveMember); - } - let add_proxy_call = client_interface::metadata::tx() - .proxy() - .add_proxy(account_to_subxt_multi_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) - .unvalidated(); - match network.call_extrinsic(&add_proxy_call).await { - Ok(_) => Ok(SuccessOutput::JoinedGlove), - Err(Runtime(Module(module_error))) => { - match module_error.as_root_error::() { - // Unlikely, but just in case - Ok(Proxy(Duplicate)) => Ok(SuccessOutput::AlreadyGloveMember), - _ => Err(Runtime(Module(module_error)).into()) - } - }, - Err(e) => Err(e.into()) - } -} - -async fn vote( - service_info: ServiceInfo, - cmd: VoteCmd, - glove_url: Url, - http_client: Client, -) -> Result { - let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; - let balance = (&cmd.balance * 10u128.pow(network.token.decimals as u32)) - .to_u128() - .context("Vote balance is too big")?; - let request = VoteRequest::new( - network.account(), - network.api.genesis_hash(), - cmd.poll_index, - cmd.aye, - balance, - cmd.parse_conviction()? - ); - let nonce = request.nonce; - - let signature = MultiSignature::Sr25519(network.account_key.sign(&request.encode()).0.into()); - let signed_request = SignedVoteRequest { request, signature }; - if !signed_request.verify() { - bail!("Something has gone wrong with the signature") - } - let response = http_client - .post(url_with_path(&glove_url, "vote")) - .json(&signed_request) - .send().await - .context("Unable to send vote request")?; - - if response.status() != StatusCode::OK { - bail!(response.text().await?) - } - if cmd.await_glove_proof { - listen_for_glove_votes(&network, &cmd, nonce, &service_info.proxy_account).await?; - } - return Ok(SuccessOutput::Voted { nonce }); -} - -// TODO Stop waiting when the poll is closed. -async fn listen_for_glove_votes( - network: &CallableSubstrateNetwork, - vote_cmd: &VoteCmd, - nonce: u32, - proxy_account: &AccountId32 -) -> Result<()> { - network.subscribe_successful_extrinsics(|extrinsic, _| async move { - let verification_result = try_verify_glove_result( - &network, - &extrinsic, - proxy_account, - vote_cmd.poll_index, - ).await; - let verified_glove_proof = match verification_result { - Ok(None) => return Ok(()), // Not what we're looking for - Ok(Some(verified_glove_proof)) => verified_glove_proof, - Err(Error::Subxt(subxt_error)) => return Err(subxt_error.into()), - Err(error) => { - eprintln!("Error verifying Glove proof: {}", error); - return Ok(()); - } - }; - if let Some(balance) = verified_glove_proof.get_vote_balance(&network.account(), nonce) { - println!("Glove vote {:?} with balance {}", - verified_glove_proof.result.direction, network.token.amount(balance)); - if let Some(_) = &verified_glove_proof.enclave_info { - // TODO Check measurement - } else { - eprintln!("WARNING: Secure enclave wasn't used"); - } - } else { - eprintln!("WARNING: Received Glove proof for poll, but vote was not included"); - } - Ok(()) - }).await?; - Ok(()) -} - -async fn remove_vote( - service_info: ServiceInfo, - cmd: RemoveVoteCmd, - glove_url: Url, - http_client: Client, -) -> Result { - let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; - let request = RemoveVoteRequest { - account: network.account(), - poll_index: cmd.poll_index - }; - let signature = MultiSignature::Sr25519(network.account_key.sign(&request.encode()).0.into()); - let signed_request = SignedRemoveVoteRequest { request, signature }; - if !signed_request.verify() { - bail!("Something has gone wrong with the signature") - } - let response = http_client - .post(url_with_path(&glove_url, "remove-vote")) - .json(&signed_request) - .send().await - .context("Unable to send remove vote request")?; - if response.status() == StatusCode::OK { - Ok(SuccessOutput::VoteRemoved) - } else { - bail!(response.text().await?) - } -} - -async fn verify_vote( - service_info: ServiceInfo, - cmd: VerifyVoteCmd, - http_client: Client -) -> Result { - let network = SubstrateNetwork::connect(service_info.node_endpoint).await?; - let Some(poll_info) = network.get_poll(cmd.poll_index).await? else { - bail!("Poll does not exist") - }; - let votes = subscan::get_votes( - &http_client, - &service_info.network_name, - cmd.poll_index, - Some(cmd.account.clone()) - ).await?; - let Some(vote) = votes.first() else { - if matches!(poll_info, ReferendumInfoFor::Ongoing(_)) { - bail!("Glove proxy has not voted yet") - } else { - bail!("Poll is no longer active and Glove proxy did not vote") - } - }; - let Some(extrinsic) = network.get_extrinsic(vote.extrinsic_index).await? else { - bail!("Unable to find vote extrinsic at {}", vote.extrinsic_index) - }; - let verification_result = try_verify_glove_result( - &network, - &extrinsic, - &service_info.proxy_account, - cmd.poll_index - ).await; - let verified_glove_proof = match verification_result { - Ok(Some(verified_glove_proof)) => verified_glove_proof, - Ok(None) => bail!("Vote was not cast by Glove proxy"), - Err(error) => bail!("Glove proof failed verification: {}", error) - }; - let assigned_balance = verified_glove_proof - .get_assigned_balance(&cmd.account) - .ok_or_else(|| anyhow!("Account is not in Glove proof"))?; - let image_measurement = match verified_glove_proof.enclave_info { - Some(EnclaveInfo::Nitro(nitro_enclave_info)) => nitro_enclave_info.image_measurement, - None => bail!("INSECURE enclave was used to mix votes, so result cannot be trusted") - }; - - if let Some(nonce) = cmd.nonce { - if nonce != assigned_balance.nonce { - bail!("Nonce in Glove proof ({}) does not match expected value. \ - Glove proxy has used an older vote request.", assigned_balance.nonce) - } - } else { - eprintln!("Nonce was not provided so cannot check if most recent vote request was used by \ - Glove proxy"); - } - - if !cmd.enclave_measurement.is_empty() { - let enclave_match = cmd.enclave_measurement - .iter() - .any(|str| hex::decode(str).ok() == Some(image_measurement.clone())); - if !enclave_match { - bail!("Unknown enclave encountered in Glove proof ({})", - hex::encode(&image_measurement)) - } - println!("Vote mixed by VERIFIED Glove enclave: {:?} with {} and conviction {:?}", - verified_glove_proof.result.direction, - network.token.amount(assigned_balance.balance), - assigned_balance.conviction); - } else { - println!("Vote mixed by POSSIBLE Glove enclave ({}): {:?} with {} and conviction {:?}", - hex::encode(&image_measurement), - verified_glove_proof.result.direction, - network.token.amount(assigned_balance.balance), - assigned_balance.conviction); - println!(); - println!("To verify this is a Glove enclave, first audit the code:"); - println!("git clone --depth 1 --branch v{} {}", - verified_glove_proof.attested_data.version, - env!("CARGO_PKG_REPOSITORY")); - println!(); - println!("And then verify 'PCR0' output is '{}':", hex::encode(&image_measurement)); - println!("./build.sh"); - } - - Ok(SuccessOutput::None) -} - -// TODO Also remove any active votes, which requires a remove-all-votes request? -async fn leave_glove(service_info: ServiceInfo, cmd: LeaveCmd) -> Result { - let network = cmd.secret_phrase_args.connect_to_network(&service_info).await?; - if !is_glove_member(&network, network.account(), service_info.proxy_account.clone()).await? { - return Ok(SuccessOutput::NotGloveMember); - } - let add_proxy_call = client_interface::metadata::tx() - .proxy() - .remove_proxy(account_to_subxt_multi_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) - .unvalidated(); - match network.call_extrinsic(&add_proxy_call).await { - Ok(_) => Ok(SuccessOutput::LeftGlove), - Err(Runtime(Module(module_error))) => { - match module_error.as_root_error::() { - // Unlikely, but just in case - Ok(Proxy(NotFound)) => Ok(SuccessOutput::NotGloveMember), - _ => Err(Runtime(Module(module_error)).into()) - } - }, - Err(e) => Err(e.into()) - } -} - -fn info(service_info: ServiceInfo) -> Result { - let ab = &service_info.attestation_bundle; - let enclave_info = match ab.verify() { - Ok(EnclaveInfo::Nitro(enclave_info)) => { - &format!("AWS Nitro Enclave ({})", hex::encode(enclave_info.image_measurement)) - }, - Err(attestation::Error::InsecureMode) => match ab.attestation { - Attestation::Nitro(_) => "Debug AWS Nitro Enclave (INSECURE)", - Attestation::Mock => "Mock (INSECURE)" - }, - Err(attestation_error) => &format!("Error verifying attestation: {}", attestation_error) - }; - - println!("Glove proxy account: {}", service_info.proxy_account); - println!("Enclave: {}", enclave_info); - println!("Substrate Network: {}", service_info.network_name); - println!("Genesis hash: {}", hex::encode(ab.attested_data.genesis_hash)); - - Ok(SuccessOutput::None) -} - -fn url_with_path(url: &Url, path: &str) -> Url { - let mut with_path = url.clone(); - with_path.set_path(path); - with_path -} - -#[derive(Debug, Parser)] -#[command(version, about = "Glove CLI client")] -struct Args { - /// The URL of the Glove service - #[arg(long, short, verbatim_doc_comment)] - glove_url: Url, - - #[clap(subcommand)] - command: Command, -} - -#[derive(Debug, Clone, clap::Args)] -struct SecretPhraseArgs { - /// The secret phrase for the Glove client account. This is a secret seed with optional - /// derivation paths. An Sr25519 key will be derived from this for signing. - /// - /// See https://wiki.polkadot.network/docs/learn-account-advanced#derivation-paths for more - /// details. - #[arg(long, verbatim_doc_comment, value_parser = client_interface::parse_secret_phrase)] - secret_phrase: Keypair -} - -impl SecretPhraseArgs { - async fn connect_to_network( - &self, - service_info: &ServiceInfo - ) -> Result { - CallableSubstrateNetwork::connect( - service_info.node_endpoint.clone(), - self.secret_phrase.clone() - ).await - } -} - -#[derive(Debug, Subcommand)] -enum Command { - /// Add Glove as a goverance proxy to the account, if it isn't already. - #[command(verbatim_doc_comment)] - JoinGlove(JoinCmd), - /// Submit vote for inclusion in Glove mixing. The mixing process is not necessarily immediate. - /// Voting on the same poll twice will replace the previous vote. - #[command(verbatim_doc_comment)] - Vote(VoteCmd), - /// Remove a previously submitted vote. - #[command(verbatim_doc_comment)] - RemoveVote(RemoveVoteCmd), - /// Verify on-chain vote was mixed by a genuine Glove enclave - #[command(verbatim_doc_comment)] - VerifyVote(VerifyVoteCmd), - /// Remove the account from the Glove proxy. - #[command(verbatim_doc_comment)] - LeaveGlove(LeaveCmd), - /// Print information about the Glove service. - #[command(verbatim_doc_comment)] - Info -} - -#[derive(Debug, Parser)] -struct JoinCmd { - #[command(flatten)] - secret_phrase_args: SecretPhraseArgs -} - -#[derive(Debug, Parser)] -struct VoteCmd { - #[command(flatten)] - secret_phrase_args: SecretPhraseArgs, - #[arg(long, short)] - poll_index: u32, - /// Specify this to vote "aye", ommit to vote "nay" - #[arg(long, verbatim_doc_comment)] - aye: bool, - /// The amount of tokens to lock for the vote (as a decimal in the major token unit) - #[arg(long, short, verbatim_doc_comment)] - balance: BigDecimal, - /// The vote conviction multiplier - #[arg(long, short, verbatim_doc_comment, default_value_t = 0)] - conviction: u8, - /// Wait for the vote to be included in the Glove mixing process and confirmation received. - #[arg(long, short, verbatim_doc_comment)] - await_glove_proof: bool -} - -impl VoteCmd { - fn parse_conviction(&self) -> Result { - match self.conviction { - 0 => Ok(Conviction::None), - 1 => Ok(Conviction::Locked1x), - 2 => Ok(Conviction::Locked2x), - 3 => Ok(Conviction::Locked3x), - 4 => Ok(Conviction::Locked4x), - 5 => Ok(Conviction::Locked5x), - 6 => Ok(Conviction::Locked6x), - _ => bail!("Conviction must be between 0 and 6 inclusive") - } - } -} - -#[derive(Debug, Parser)] -struct RemoveVoteCmd { - #[command(flatten)] - secret_phrase_args: SecretPhraseArgs, - #[arg(long, short)] - poll_index: u32 -} - -#[derive(Debug, Parser)] -struct VerifyVoteCmd { - /// The account on whose behalf the Glove proxy mixed the vote - #[arg(long, short, verbatim_doc_comment)] - account: AccountId32, - /// The index of the poll/referendum - #[arg(long, short, verbatim_doc_comment)] - poll_index: u32, - /// Whitelisted Glove enclave measurements. Each measurement represents a different enclave - /// version. The on-chain Glove proof associated with the vote will be checked against this - /// list. It is assumed the versions of the enclave these measurement represent have been - /// audited. - /// - /// If no enclave measurement is specified, the measurement of the Glove proof will displayed, - /// along with enclave code location, for auditing. - #[arg(long, short, verbatim_doc_comment)] - enclave_measurement: Vec, - /// Optional, the nonce value used in the most recent vote request. - #[arg(long, short, verbatim_doc_comment)] - nonce: Option -} - -#[derive(Debug, Parser)] -struct LeaveCmd { - #[command(flatten)] - secret_phrase_args: SecretPhraseArgs -} - -#[derive(Display, Debug, PartialEq)] -enum SuccessOutput { - #[strum(to_string = "Account has joined Glove proxy")] - JoinedGlove, - #[strum(to_string = "Account already a member of Glove proxy")] - AlreadyGloveMember, - #[strum(to_string = "Vote successfully submitted ({nonce})")] - Voted { nonce: u32 }, - #[strum(to_string = "Vote successfully removed")] - VoteRemoved, - #[strum(to_string = "Account has left Glove proxy")] - LeftGlove, - #[strum(to_string = "Account was not a Glove proxy member")] - NotGloveMember, - None -} - -impl Termination for SuccessOutput { - fn report(self) -> ExitCode { - if self != SuccessOutput::None { - println!("{}", self); - } - ExitCode::SUCCESS - } + run(env::args_os()).await } diff --git a/client/src/verify.rs b/client/src/verify.rs new file mode 100644 index 0000000..f94a2a4 --- /dev/null +++ b/client/src/verify.rs @@ -0,0 +1,250 @@ +use std::collections::HashMap; + +use sp_core::crypto::AccountId32; +use subxt::Error as SubxtError; + +use attestation::EnclaveInfo; +use AttestationBundleLocation::SubstrateRemark; +use client_interface::{account, ExtrinsicDetails, SubstrateNetwork}; +use client_interface::metadata::runtime_types; +use client_interface::metadata::runtime_types::frame_system::pallet::Call as SystemCall; +use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Call as ConvictionVotingCall; +use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::AccountVote; +use client_interface::metadata::runtime_types::pallet_proxy::pallet::Call as ProxyCall; +use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall; +use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall::ConvictionVoting; +use client_interface::metadata::system::calls::types::Remark; +use client_interface::metadata::utility::calls::types::Batch; +use common::{AssignedBalance, attestation, BASE_AYE, Conviction, ExtrinsicLocation, GloveResult, VoteDirection}; +use common::attestation::{AttestationBundle, AttestationBundleLocation, AttestedData, GloveProof, GloveProofLite}; +use runtime_types::pallet_conviction_voting::vote::Vote; + +#[derive(Debug, Clone, PartialEq)] +pub struct VerifiedGloveProof { + pub result: GloveResult, + /// If `None` then enclave was running in insecure mode. + pub enclave_info: Option, + pub attested_data: AttestedData +} + +// TODO API for checking EnclaveInfo for expected measurements +impl VerifiedGloveProof { + pub fn get_assigned_balance(&self, account: &AccountId32) -> Option { + self.result + .assigned_balances + .iter() + .find(|assigned_balance| assigned_balance.account == *account) + .cloned() + } + + pub fn get_vote_balance(&self, account: &AccountId32, nonce: u32) -> Option { + self.result + .assigned_balances + .iter() + .find_map(|ab| (ab.account == *account && ab.nonce == nonce).then_some(ab.balance)) + } +} + +/// A result of `Ok(None)` means the extrinsic was not a Glove result. +pub async fn try_verify_glove_result( + network: &SubstrateNetwork, + extrinsic: &ExtrinsicDetails, + proxy_account: &AccountId32, + poll_index: u32 +) -> Result, Error> { + let Some((glove_proof_lite, batch)) = parse_glove_proof_lite(extrinsic, proxy_account)? else { + // This extrinsic is not a Glove proof + return Ok(None); + }; + let glove_result = &glove_proof_lite.signed_result.result; + + if glove_result.poll_index != poll_index { + // This proof is for another poll and so let's return early and avoid unnecessary + // processing, and also avoid any potential errors which wouldn't be relevant to the caller. + return Ok(None); + } + + let account_votes: HashMap> = batch.calls + .iter() + .filter_map(|call| parse_and_validate_proxy_account_vote(call, glove_result.poll_index)) + .collect::>(); + // TODO Check for duplicate on-chain votes for the same account + + // Make sure each assigned balance from the Glove proof is accounted for on-chain. + for assigned_balance in &glove_result.assigned_balances { + account_votes.get(&assigned_balance.account) + .filter(|&&account_vote| { + is_account_vote_consistent(account_vote, glove_result.direction, assigned_balance) + }) + .ok_or(Error::InconsistentVotes)?; + } + + // It's technically possible for there to be more on-chain votes from the same proxy, for the + // same poll, which are not in the proof. This is not something the client has to worry about, + // since they can only confirm their vote request was included in the proof. + + let attestation_bundle = match glove_proof_lite.attestation_location { + SubstrateRemark(remark_location) => + get_attestation_bundle_from_remark(network, remark_location).await? + }; + + if attestation_bundle.attested_data.genesis_hash != network.api.genesis_hash() { + return Err(Error::ChainMismatch); + } + + let glove_proof = GloveProof { + signed_result: glove_proof_lite.signed_result, + attestation_bundle + }; + + let enclave_info = match glove_proof.verify() { + Ok(enclave_info) => Some(enclave_info), + Err(attestation::Error::InsecureMode) => None, + Err(error) => return Err(error.into()) + }; + + Ok(Some(VerifiedGloveProof { + result: glove_proof.signed_result.result, + enclave_info, + attested_data: glove_proof.attestation_bundle.attested_data + })) +} + +fn parse_glove_proof_lite( + extrinsic: &ExtrinsicDetails, + proxy_account: &AccountId32 +) -> Result, SubxtError> { + let from_proxy = account(extrinsic).filter(|account| account == proxy_account).is_some(); + if !from_proxy { + return Ok(None); + } + + let Some(batch) = extrinsic.as_extrinsic::()? else { + return Ok(None); + }; + + let remarks = batch.calls + .iter() + .filter_map(|call| match call { + RuntimeCall::System(SystemCall::remark { remark }) => Some(remark), + _ => None + }) + .collect::>(); + + // Expecting there to be exactly one remark call + let &[remark] = remarks.as_slice() else { + return Ok(None); + }; + + Ok(GloveProofLite::decode_envelope(remark).map(|proof| (proof, batch)).ok()) +} + +fn parse_and_validate_proxy_account_vote( + call: &RuntimeCall, + expected_poll_index: u32, +) -> Option<(AccountId32, &AccountVote)> { + let RuntimeCall::Proxy(proxy_call) = call else { + return None; + }; + let ProxyCall::proxy { real, force_proxy_type: _, call: proxied_call } = proxy_call else { + return None; + }; + let subxt_core::utils::MultiAddress::Id(real_account) = real else { + return None; + }; + let ConvictionVoting(ConvictionVotingCall::vote { poll_index, ref vote }) = **proxied_call else { + return None; + }; + if expected_poll_index != poll_index { + return None; + } + Some((real_account.0.into(), vote)) +} + +fn is_account_vote_consistent( + account_vote: &AccountVote, + direction: VoteDirection, + assigned_balance: &AssignedBalance +) -> bool { + match direction { + VoteDirection::Aye => { + parse_standard_account_vote(account_vote) == + Some((true, assigned_balance.balance, assigned_balance.conviction)) + }, + VoteDirection::Nay => { + parse_standard_account_vote(account_vote) == + Some((false, assigned_balance.balance, assigned_balance.conviction)) + }, + VoteDirection::Abstain => { + parse_abstain_account_vote(account_vote) == Some(assigned_balance.balance) + } + } +} + +fn parse_standard_account_vote(vote: &AccountVote) -> Option<(bool, u128, Conviction)> { + let AccountVote::Standard { vote: Vote(direction), balance } = vote else { + return None; + }; + if *direction >= BASE_AYE { + Some((true, *balance, parse_conviction(*direction - BASE_AYE)?)) + } else { + Some((false, *balance, parse_conviction(*direction)?)) + } +} + +fn parse_conviction(offset: u8) -> Option { + match offset { + 0 => Some(Conviction::None), + 1 => Some(Conviction::Locked1x), + 2 => Some(Conviction::Locked2x), + 3 => Some(Conviction::Locked3x), + 4 => Some(Conviction::Locked4x), + 5 => Some(Conviction::Locked5x), + 6 => Some(Conviction::Locked6x), + _ => None + } +} + +fn parse_abstain_account_vote(account_vote: &AccountVote) -> Option { + match account_vote { + AccountVote::SplitAbstain { aye: 0, nay: 0, abstain } => Some(*abstain), + _ => None + } +} + +async fn get_attestation_bundle_from_remark( + network: &SubstrateNetwork, + remark_location: ExtrinsicLocation +) -> Result { + network.get_extrinsic(remark_location).await? + .ok_or_else(|| Error::ExtrinsicNotFound(remark_location))? + .as_extrinsic::()? + .ok_or_else(|| Error::InvalidAttestationBundle( + format!("Extrinsic at location {:?} is not a Remark", remark_location) + )) + .and_then(|remark| { + AttestationBundle::decode_envelope(&remark.remark).map_err(|scale_error| { + Error::InvalidAttestationBundle( + format!("Error decoding attestation bundle: {}", scale_error) + ) + }) + }) +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Subxt error: {0}")] + Subxt(#[from] SubxtError), + #[error("Votes are inconsistent with the Glove proof")] + InconsistentVotes, + #[error("Glove proof is for different chain")] + ChainMismatch, + #[error("Extrinsic at location {0} does not exist")] + ExtrinsicNotFound(ExtrinsicLocation), + #[error("Invalid attestation bundle: {0}")] + InvalidAttestationBundle(String), + #[error("Invalid attestation: {0}")] + Attestation(#[from] attestation::Error) +} + +// TODO Tests diff --git a/common/src/lib.rs b/common/src/lib.rs index bc56aa9..b8abdb6 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -227,6 +227,7 @@ mod tests { "#; let signed_request = serde_json::from_str::(json).unwrap(); + println!("{:#?}", signed_request); assert!(signed_request.verify()); let request = signed_request.request; assert_eq!(request.account, dev::bob().public_key().0.into()); diff --git a/service/Cargo.toml b/service/Cargo.toml index 394b8fc..716edb2 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -28,6 +28,7 @@ serde.workspace = true reqwest.workspace = true aws-config.workspace = true aws-sdk-dynamodb.workspace = true +aws-smithy-runtime-api.workspace = true [target.'cfg(target_os = "linux")'.dependencies] tokio-vsock.workspace = true diff --git a/service/src/lib.rs b/service/src/lib.rs index 8341e83..556a841 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -7,13 +7,14 @@ use std::sync::atomic::{AtomicBool, Ordering}; use sp_runtime::AccountId32; use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use tracing::log::warn; +use tracing::warn; -use client_interface::{account, account_to_subxt_multi_address, CallableSubstrateNetwork, subscan, SubstrateNetwork}; +use client_interface::{account_to_subxt_multi_address, CallableSubstrateNetwork, subscan}; use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Call as ConvictionVotingCall; use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::{AccountVote, Vote}; use client_interface::metadata::runtime_types::pallet_proxy::pallet::Call as ProxyCall; use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall; +use client_interface::subscan::Subscan; use common::{AssignedBalance, BASE_AYE, BASE_NAY, Conviction, ExtrinsicLocation, GloveResult, SignedVoteRequest, VoteDirection}; use common::attestation::{AttestationBundle, AttestationBundleLocation}; @@ -25,35 +26,49 @@ pub mod mixing; pub mod dynamodb; pub mod storage; -/// Get the voters of a poll. Returns a vector of (`AccountId32`, `AccountId32`) tuples. The first -/// element is the voter, and the second is sender of the vote extrinsic. They will be different if -/// the vote was proxied. -pub async fn get_voters( - http_client: &reqwest::Client, - network: &SubstrateNetwork, - poll_index: u32 -) -> anyhow::Result> { - let mut result = Vec::new(); - // Avoid looking up the same extrinsic multiple times when we encounter the Glove vote batch - let mut extrinic_index_to_voter: HashMap = HashMap::new(); - let votes = subscan::get_votes(http_client, &network.network_name, poll_index, None).await?; - for vote in votes { - let sender = match extrinic_index_to_voter.entry(vote.extrinsic_index) { - Entry::Occupied(entry) => entry.get().clone(), - Entry::Vacant(entry) => { - let Some(extrinsic) = network.get_extrinsic(vote.extrinsic_index).await? else { - warn!("Extrinsic referenced by subscan not found: {:?}", vote); - continue; - }; - match account(&extrinsic) { - Some(sender) => entry.insert(sender).clone(), - None => continue +/// Voter lookup for a poll. +#[derive(Clone)] +pub struct VoterLookup { + poll_index: u32, + // get_voters is called frequently, so we want to avoid looking up the same extrinsic every + // time. + cache: Arc>>> +} + +impl VoterLookup { + pub fn new(poll_index: u32) -> Self { + Self { + poll_index, + cache: Arc::default() + } + } + + /// Get the all voters. Returns a vector of (`AccountId32`, `AccountId32`) tuples. The first + /// element is the voter, and the second is sender of the vote extrinsic. They will be different + /// if the vote was proxied. + pub async fn get_voters( + &self, + subscan: &Subscan + ) -> Result)>, subscan::Error> { + let mut result = Vec::new(); + let votes = subscan.get_votes(self.poll_index, None).await?; + let mut cache = self.cache.lock().await; + for vote in votes { + let sender = match cache.entry(vote.extrinsic_index) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + if let Some(extrinsic) = subscan.get_extrinsic(vote.extrinsic_index).await? { + entry.insert(extrinsic.account_display.map(|a| a.address)).clone() + } else { + warn!("Extrinsic not found: {:?}", vote.extrinsic_index); + entry.insert(None).clone() + } } - } - }; - result.push((vote.account.address, sender)); + }; + result.push((vote.account.address, sender)); + } + Ok(result) } - Ok(result) } // TODO Deal with mixed_balance of zero @@ -99,7 +114,6 @@ pub struct GloveContext { pub enclave_handle: EnclaveHandle, pub attestation_bundle: AttestationBundle, pub network: CallableSubstrateNetwork, - pub node_endpoint: String, pub regular_mix_enabled: bool, pub state: GloveState, } @@ -196,7 +210,12 @@ impl GloveState { pub async fn get_poll_state_ref(&self, poll_index: u32) -> PollStateRef { let mut poll_states = self.poll_states.lock().await; - poll_states.entry(poll_index).or_default().clone() + poll_states.entry(poll_index).or_insert_with(|| { + PollStateRef { + inner: Arc::default(), + voter_lookup: VoterLookup::new(poll_index) + } + }).clone() } async fn remove_poll_state(&self, poll_index: u32) { @@ -204,11 +223,12 @@ impl GloveState { } } -#[derive(Default, Clone)] +#[derive(Clone)] pub struct PollStateRef { // There is a read-write lock here to support concurrent addition/removal of vote requests, but // synchronized mixing of the votes. - inner: Arc> + inner: Arc>, + pub voter_lookup: VoterLookup } #[derive(Default)] diff --git a/service/src/main.rs b/service/src/main.rs index e91c2cd..b4e8217 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -4,6 +4,11 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{bail, Context}; +use aws_sdk_dynamodb::error::SdkError; +use aws_sdk_dynamodb::operation::delete_item::DeleteItemError; +use aws_sdk_dynamodb::operation::put_item::PutItemError; +use aws_sdk_dynamodb::operation::query::QueryError; +use aws_sdk_dynamodb::operation::scan::ScanError; use axum::extract::{Json, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; @@ -20,7 +25,7 @@ use tokio::spawn; use tokio::time::sleep; use tower_http::trace::TraceLayer; use tracing::{debug, error, info}; -use tracing::log::warn; +use tracing::warn; use tracing_subscriber::filter::{EnvFilter, LevelFilter}; use attestation::Error::InsecureMode; @@ -37,13 +42,15 @@ use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::NotP use client_interface::metadata::runtime_types::pallet_referenda::types::DecidingStatus; use client_interface::metadata::runtime_types::polkadot_runtime::{RuntimeCall, RuntimeError}; use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeError::Proxy; +use client_interface::subscan::Subscan; use common::{attestation, SignedGloveResult, SignedVoteRequest}; use common::attestation::{AttestationBundle, AttestationBundleLocation, GloveProofLite}; use RuntimeError::ConvictionVoting; -use service::{get_voters, GloveContext, GloveState, mixing, storage, to_proxied_vote_call}; +use service::{GloveContext, GloveState, mixing, storage, to_proxied_vote_call}; use service::dynamodb::DynamodbGloveStorage; use service::enclave::EnclaveHandle; use service::storage::{GloveStorage, InMemoryGloveStorage}; +use storage::Error as StorageError; #[derive(Parser, Debug)] #[command(version, about = "Glove proxy service")] @@ -193,17 +200,17 @@ async fn main() -> anyhow::Result<()> { enclave_handle, attestation_bundle, network, - node_endpoint: args.node_endpoint, regular_mix_enabled: args.regular_mix, state: GloveState::default() }); + let subscan = Subscan::new(glove_context.network.network_name.clone()); if args.regular_mix { warn!("Regular mixing of votes is enabled. This is not suitable for production."); } else { - mark_voted_polls_as_final(glove_context.clone()).await?; + mark_voted_polls_as_final(glove_context.clone(), &subscan).await?; } - start_background_thread(glove_context.clone()); + start_background_thread(glove_context.clone(), subscan); let router = Router::new() .route("/info", get(info)) @@ -252,13 +259,16 @@ async fn initialize_enclave(enclave_mode: EnclaveMode) -> io::Result) -> anyhow::Result<()> { - let http_client = reqwest::Client::builder().build().unwrap(); - let glove_proxy = context.network.account(); +async fn mark_voted_polls_as_final( + context: Arc, + subscan: &Subscan +) -> anyhow::Result<()> { + let glove_proxy = Some(context.network.account()); let poll_indices = context.storage.get_poll_indices().await?; debug!("Polls: {:?}", poll_indices); for poll_index in poll_indices { - let proxy_has_voted = get_voters(&http_client, &context.network, poll_index).await? + let voter_lookup = context.state.get_poll_state_ref(poll_index).await.voter_lookup; + let proxy_has_voted = voter_lookup.get_voters(&subscan).await? .into_iter() .any(|(_, sender)| sender == glove_proxy); if proxy_has_voted { @@ -270,10 +280,10 @@ async fn mark_voted_polls_as_final(context: Arc) -> anyhow::Result Ok(()) } -fn start_background_thread(context: Arc) { +fn start_background_thread(context: Arc, subscan: Subscan) { spawn(async move { loop { - if let Err(error) = run_background_task(context.clone()).await { + if let Err(error) = run_background_task(context.clone(), subscan.clone()).await { warn!("Error from background task: {:?}", error) } sleep(Duration::from_secs(60)).await; @@ -281,11 +291,10 @@ fn start_background_thread(context: Arc) { }); } -async fn run_background_task(context: Arc) -> anyhow::Result<()> { +async fn run_background_task(context: Arc, subscan: Subscan) -> anyhow::Result<()> { let network = &context.network; let storage = &context.storage; let regular_mix_enabled = context.regular_mix_enabled; - let http_client = reqwest::Client::builder().build().unwrap(); let tracks = network.get_tracks()?; for poll_index in storage.get_poll_indices().await? { @@ -295,7 +304,7 @@ async fn run_background_task(context: Arc) -> anyhow::Result<()> { continue; }; let mut mix_required = regular_mix_enabled && context.state.was_vote_added(poll_index).await; - if check_non_glove_voters(&http_client, &context, poll_index).await? { + if check_non_glove_voters(&subscan, &context, poll_index).await? { mix_required = true; } if context.state.is_mix_finalized(poll_index).await { @@ -320,14 +329,15 @@ async fn run_background_task(context: Arc) -> anyhow::Result<()> { } async fn check_non_glove_voters( - http_client: &reqwest::Client, + subscan: &Subscan, context: &GloveContext, poll_index: u32 ) -> anyhow::Result{ - let glove_proxy = context.network.account(); + let glove_proxy = Some(context.network.account()); let mut non_glove_voters = HashSet::new(); let mut mix_required = false; - for (voter, sender) in get_voters(&http_client, &context.network, poll_index).await? { + let voter_lookup = context.state.get_poll_state_ref(poll_index).await.voter_lookup; + for (voter, sender) in voter_lookup.get_voters(&subscan).await? { if sender == glove_proxy { continue; } @@ -366,7 +376,6 @@ async fn info(context: State>) -> Json { Json(ServiceInfo { proxy_account: context.network.account(), network_name: context.network.network_name.clone(), - node_endpoint: context.node_endpoint.clone(), attestation_bundle: context.attestation_bundle.clone(), version: env!("CARGO_PKG_VERSION").to_string() }) @@ -610,6 +619,48 @@ enum InternalError { Storage(#[from] storage::Error) } +impl InternalError { + fn is_too_many_requests(&self) -> bool { + let InternalError::Storage(storage_error) = self else { + return false; + }; + match storage_error { + StorageError::DynamodbPutItem(SdkError::ServiceError(error)) => matches!(error.err(), + PutItemError::ProvisionedThroughputExceededException(_) | + PutItemError::RequestLimitExceeded(_) + ), + StorageError::DynamodbDeleteItem(SdkError::ServiceError(error)) => matches!(error.err(), + DeleteItemError::ProvisionedThroughputExceededException(_) | + DeleteItemError::RequestLimitExceeded(_) + ), + StorageError::DynamodbQuery(SdkError::ServiceError(error)) => matches!(error.err(), + QueryError::ProvisionedThroughputExceededException(_) | + QueryError::RequestLimitExceeded(_) + ), + StorageError::DynamodbScan(SdkError::ServiceError(error)) => matches!(error.err(), + ScanError::ProvisionedThroughputExceededException(_) | + ScanError::RequestLimitExceeded(_) + ), + _ => false + } + } +} + +impl IntoResponse for InternalError { + fn into_response(self) -> Response { + if self.is_too_many_requests() { + warn!("Too many requests: {:?}", self); + ( + StatusCode::TOO_MANY_REQUESTS, + "Too many requests, please try again later".to_string() + ).into_response() + } else { + warn!("Unable to service request: {:?}", self); + (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".to_string()).into_response() + } + } +} + #[derive(Serialize)] struct BadRequestResponse { error: String, @@ -634,6 +685,24 @@ enum BadVoteRequestError { InsufficientBalance } +impl IntoResponse for BadVoteRequestError { + fn into_response(self) -> Response { + let error_variant = match self { + BadVoteRequestError::InvalidSignature => "InvalidSignature", + BadVoteRequestError::ChainMismatch => "ChainMismatch", + BadVoteRequestError::NotMember => "NotMember", + BadVoteRequestError::VotedOutsideGlove => "VotedOutsideGlove", + BadVoteRequestError::PollNotOngoing => "PollNotOngoing", + BadVoteRequestError::PollAlreadyMixed => "PollAlreadyMixed", + BadVoteRequestError::InsufficientBalance => "InsufficientBalance" + }.to_string(); + ( + StatusCode::BAD_REQUEST, + Json(BadRequestResponse { error: error_variant, description: self.to_string() }) + ).into_response() + } +} + #[derive(thiserror::Error, Debug)] enum VoteError { #[error("Bad request: {0}")] @@ -642,6 +711,15 @@ enum VoteError { Internal(#[from] InternalError) } +impl IntoResponse for VoteError { + fn into_response(self) -> Response { + match self { + VoteError::BadRequest(error) => error.into_response(), + VoteError::Internal(error) => error.into_response() + } + } +} + impl From for VoteError { fn from(error: SubxtError) -> Self { VoteError::Internal(error.into()) @@ -660,35 +738,6 @@ impl From for VoteError { } } -impl IntoResponse for VoteError { - fn into_response(self) -> Response { - match self { - VoteError::BadRequest(error) => { - let error_variant = match error { - BadVoteRequestError::InvalidSignature => "InvalidSignature", - BadVoteRequestError::ChainMismatch => "ChainMismatch", - BadVoteRequestError::NotMember => "NotMember", - BadVoteRequestError::VotedOutsideGlove => "VotedOutsideGlove", - BadVoteRequestError::PollNotOngoing => "PollNotOngoing", - BadVoteRequestError::PollAlreadyMixed => "PollAlreadyMixed", - BadVoteRequestError::InsufficientBalance => "InsufficientBalance" - }.to_string(); - ( - StatusCode::BAD_REQUEST, - Json(BadRequestResponse { error: error_variant, description: error.to_string() }) - ).into_response() - }, - VoteError::Internal(error) => { - warn!("Error with vote request: {:?}", error); - ( - StatusCode::INTERNAL_SERVER_ERROR, - "Internal server error".to_string() - ).into_response() - } - } - } -} - #[derive(thiserror::Error, Debug)] enum BadRemoveVoteRequestError { #[error("Signature on signed vote request is invalid")] @@ -699,6 +748,20 @@ enum BadRemoveVoteRequestError { PollAlreadyMixed } +impl IntoResponse for BadRemoveVoteRequestError { + fn into_response(self) -> Response { + let error_variant = match self { + BadRemoveVoteRequestError::InvalidSignature => "InvalidSignature", + BadRemoveVoteRequestError::NotMember => "NotMember", + BadRemoveVoteRequestError::PollAlreadyMixed => "PollAlreadyMixed" + }.to_string(); + ( + StatusCode::BAD_REQUEST, + Json(BadRequestResponse { error: error_variant, description: self.to_string() }) + ).into_response() + } +} + #[derive(thiserror::Error, Debug)] enum RemoveVoteError { #[error("Bad request: {0}")] @@ -707,6 +770,15 @@ enum RemoveVoteError { Internal(#[from] InternalError) } +impl IntoResponse for RemoveVoteError { + fn into_response(self) -> Response { + match self { + RemoveVoteError::BadRequest(error) => error.into_response(), + RemoveVoteError::Internal(error) => error.into_response() + } + } +} + impl From for RemoveVoteError { fn from(error: SubxtError) -> Self { RemoveVoteError::Internal(error.into()) @@ -724,31 +796,3 @@ impl From for RemoveVoteError { RemoveVoteError::Internal(error.into()) } } - -impl IntoResponse for RemoveVoteError { - fn into_response(self) -> Response { - match self { - RemoveVoteError::BadRequest(error) => { - let error_variant = match error { - BadRemoveVoteRequestError::InvalidSignature => "InvalidSignature", - BadRemoveVoteRequestError::NotMember => "NotMember", - BadRemoveVoteRequestError::PollAlreadyMixed => "PollAlreadyMixed" - }.to_string(); - ( - StatusCode::BAD_REQUEST, - Json(BadRequestResponse { - error: error_variant, - description: error.to_string() - }) - ).into_response() - }, - RemoveVoteError::Internal(error) => { - warn!("Error with remove-vote request: {:?}", error); - ( - StatusCode::INTERNAL_SERVER_ERROR, - "Internal server error".to_string() - ).into_response() - } - } - } -} diff --git a/service/src/mixing.rs b/service/src/mixing.rs index 90f3b26..caf49b2 100644 --- a/service/src/mixing.rs +++ b/service/src/mixing.rs @@ -1,7 +1,7 @@ use std::io; use tracing::debug; -use tracing::log::warn; +use tracing::warn; use common::{attestation, SignedGloveResult, SignedVoteRequest}; use common::attestation::{AttestationBundle, GloveProof}; diff --git a/stress-tool/Cargo.toml b/stress-tool/Cargo.toml new file mode 100644 index 0000000..4b8280b --- /dev/null +++ b/stress-tool/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "stress-tool" +homepage.workspace = true +repository.workspace = true +version.workspace = true +edition = "2021" + +[dependencies] +common = { path = "../common" } +client-interface = { path = "../client-interface" } +client = { path = "../client" } +tokio = { workspace = true, features = ["sync", "rt", "rt-multi-thread", "macros"] } +sp-runtime.workspace = true +rand.workspace = true +reqwest.workspace = true +clap.workspace = true +anyhow.workspace = true +subxt-signer.workspace = true diff --git a/stress-tool/src/main.rs b/stress-tool/src/main.rs new file mode 100644 index 0000000..6bdd5c1 --- /dev/null +++ b/stress-tool/src/main.rs @@ -0,0 +1,266 @@ +use std::io; +use std::io::Write; +use std::sync::{Arc, mpsc, Mutex}; +use std::thread::available_parallelism; + +use anyhow::{bail, Result}; +use clap::{Parser, Subcommand}; +use rand::{Rng, thread_rng}; +use reqwest::{Client, Url}; +use sp_runtime::AccountId32; +use tokio::spawn; + +use client::{node_endpoint, url_with_path}; +use client_interface::{parse_secret_phrase, ServiceInfo, SubstrateNetwork}; +use client_interface::metadata::referenda::storage::types::referendum_info_for::ReferendumInfoFor; +use client_interface::subscan::Subscan; +use common::ExtrinsicLocation; + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + match args.command { + Command::JoinGlove(cmd) => join_glove(args.glove_url, cmd).await?, + Command::Vote(cmd) => vote(args.glove_url, cmd).await?, + Command::Extrinsic(ref cmd) => extrinsic(cmd.id, args).await? + } + Ok(()) +} + +async fn join_glove(glove_url: Url, cmd: JoinGloveCmd) -> Result<()> { + let (sender, receiver) = mpsc::channel(); + let shared_derived_accounts = Arc::new(Mutex::new(cmd.accounts_args.derived_accounts()?)); + let parallelism = cmd.accounts_args.parallelism()?; + + for _ in 0..parallelism { + let shared_derived_accounts = shared_derived_accounts.clone(); + let sender = sender.clone(); + let glove_url = glove_url.clone(); + spawn(async move { + loop { + let derived_account = { + let mut shared_derived_accounts = shared_derived_accounts.lock().unwrap(); + shared_derived_accounts.pop() + }; + let Some((account_index, secret_phrase, account)) = derived_account else { + // Signal the thread is done + sender.send(()).unwrap(); + break; + }; + let output = client::run(vec![ + "client", + &format!("-g={}", glove_url), + "join-glove", + &format!("--secret-phrase={}", secret_phrase), + ]).await.unwrap(); + println!("{}. {}: {}", account_index, account, output); + } + }); + } + + // Wait for all the threads to finish + for _ in 0..parallelism { + receiver.recv()?; + } + + Ok(()) +} + +async fn vote(glove_url: Url, cmd: VoteCmd) -> Result<()>{ + let poll_indices = if cmd.poll_index.is_empty() { + let service_info = Client::new() + .get(url_with_path(&glove_url, "info")) + .send().await? + .error_for_status()? + .json::().await?; + let network = SubstrateNetwork::connect(node_endpoint(&service_info.network_name)).await?; + print!("Fetching active polls... "); + io::stdout().flush()?; + let ongoing_poll_indices = get_ongoing_poll_indices(&network).await?; + println!("{:?}", ongoing_poll_indices); + ongoing_poll_indices + } else { + cmd.poll_index + }; + + let derived_accounts = cmd.accounts_args.derived_accounts()?; + + struct VoteArg { + poll_index: u32, + aye_probability: f64, + account_index: u8, + secret_phrase: String, + account: AccountId32 + } + + let mut vote_args = Vec::new(); + + for poll_index in poll_indices { + let aye_probability = thread_rng().gen_range(0.0..1.0); + for (account_index, secret_phrase, account) in derived_accounts.clone() { + vote_args.push(VoteArg { + poll_index, + aye_probability, + account_index, + secret_phrase, + account + }); + } + } + + let shared_vote_args = Arc::new(Mutex::new(vote_args)); + + let (sender, receiver) = mpsc::channel(); + + let parallelism = cmd.accounts_args.parallelism()?; + + for _ in 0..parallelism { + let shared_vote_args = shared_vote_args.clone(); + let sender = sender.clone(); + let glove_url = glove_url.clone(); + spawn(async move { + loop { + let vote_arg = { + let mut shared_vote_args = shared_vote_args.lock().unwrap(); + shared_vote_args.pop() + }; + let Some(vote_arg) = vote_arg else { + // Signal the thread is done + sender.send(()).unwrap(); + break; + }; + let balance = thread_rng().gen_range(0.5..5.0); + let conviction = thread_rng().gen_range(0..6); + let aye = thread_rng().gen_bool(vote_arg.aye_probability); + let mut args = vec![ + "client".to_string(), + format!("-g={}", glove_url), + "vote".to_string(), + format!("--secret-phrase={}", vote_arg.secret_phrase), + format!("-p={}", vote_arg.poll_index), + format!("-b={}", balance), + format!("-c={}", conviction), + ]; + if aye { + args.push("--aye".to_string()); + } + let output = client::run(args).await.unwrap(); + println!("{}. {} poll={} balance={} conviction={} aye={}: {}", + vote_arg.account_index, vote_arg.account, vote_arg.poll_index, balance, + conviction, aye, output); + } + }); + } + + // Wait for all the threads to finish + for _ in 0..parallelism { + receiver.recv()?; + } + + Ok(()) +} + +async fn extrinsic(extrinsic_location: ExtrinsicLocation, args: Args) -> Result<()> { + let service_info = Client::new() + .get(url_with_path(&args.glove_url, "info")) + .send().await? + .error_for_status()? + .json::().await?; + let subscan = Subscan::new(service_info.network_name); + match subscan.get_extrinsic(extrinsic_location).await? { + Some(extrinsic) => println!("{:#?}", extrinsic), + None => bail!("Extrinsic not found") + } + Ok(()) +} + +async fn get_ongoing_poll_indices(network: &SubstrateNetwork) -> Result> { + // TODO Use Subscan + let mut poll_indices = Vec::new(); + let mut poll_index = 0; + loop { + match network.get_poll(poll_index).await? { + Some(ReferendumInfoFor::Ongoing(_)) => poll_indices.push(poll_index), + Some(_) => {}, + None => break, + } + poll_index += 1; + } + Ok(poll_indices) +} + +#[derive(Debug, Parser)] +#[command(version, about = "Glove service stress testing tool")] +struct Args { + #[arg(long, short, verbatim_doc_comment)] + glove_url: Url, + + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + JoinGlove(JoinGloveCmd), + Vote(VoteCmd), + Extrinsic(ExtrinsicCmd), +} + +#[derive(Debug, Parser)] +struct JoinGloveCmd { + #[command(flatten)] + accounts_args: AccountsArgs +} + +#[derive(Debug, Parser)] +struct VoteCmd { + #[command(flatten)] + accounts_args: AccountsArgs, + #[arg(long, short, verbatim_doc_comment)] + poll_index: Vec, +} + +#[derive(Debug, Parser)] +struct ExtrinsicCmd { + #[arg(long, short, verbatim_doc_comment)] + id: ExtrinsicLocation, +} + +#[derive(Debug, Clone, clap::Args)] +struct AccountsArgs { + #[arg(long, verbatim_doc_comment)] + secret_phrase: String, + + #[arg(long, short, verbatim_doc_comment, default_value_t = 0)] + start_derivation: u8, + + #[arg(long, short, verbatim_doc_comment)] + end_derivation: u8, + + #[arg(long, short, verbatim_doc_comment, default_value_t = 0)] + parallelism: u8 +} + +impl AccountsArgs { + fn derived_accounts(&self) -> Result> { + let mut keys = Vec::new(); + for i in self.start_derivation..=self.end_derivation { + let derived_phrase = format!("{}//{}", self.secret_phrase.clone(), i); + let keypair = parse_secret_phrase(&derived_phrase)?; + let account: AccountId32 = keypair.public_key().0.into(); + keys.push((i, derived_phrase, account)); + } + Ok(keys) + } + + fn parallelism(&self) -> io::Result { + if self.parallelism == 0 { + available_parallelism().map(|p| { + println!("Parallelism: {}", p); + p.get() as u8 + }) + } else { + Ok(self.parallelism) + } + } +}