diff --git a/Cargo.lock b/Cargo.lock index be56a9e..ff535a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,11 +923,13 @@ dependencies = [ "serde", "serde_json", "sp-core 33.0.1", + "sp-runtime 37.0.0", "ss58-registry", "subxt", "subxt-core", "subxt-signer", "thiserror", + "tokio", ] [[package]] @@ -4267,6 +4269,9 @@ dependencies = [ "itertools 0.13.0", "nix 0.27.1", "parity-scale-codec", + "reqwest", + "serde", + "serde_with", "sp-runtime 37.0.0", "subxt", "subxt-signer", diff --git a/README.md b/README.md index 21b525c..19c0fd6 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ Enclave Image successfully created. { "Measurements": { "HashAlgorithm": "Sha384 { ... }", - "PCR0": "6deb9e0a0fd109d1a82f998a0404fc32dffba38fc6895d768f23dc06fcbbdf4817cdb72c3144ba5fe2bb74ea874526aa", + "PCR0": "77af85a8afd3a38d4f1bcce280d886eac18b60bb8c2365906b88676fe3739458f7854c5fd8eb27d5eb4858269f6686cb", "PCR1": "4b4d5b3661b3efc12920900c80e126e4ce783c522de6c02a2a5bf7af3a2b9327b86776f188e4be1c1c404a129dbda493", - "PCR2": "5d3e7186e65b71930ae738a001ae263e1d263686e86e3f447a91b7a9bde7df03d473233016c5891680a967e671f0c27b" + "PCR2": "d97a4d6458988da4bae2cde519b75bde177fea4454d356921abc339840b28164af049e1bc68ef29fcc9dac9578f3101c" } } ``` @@ -27,7 +27,7 @@ running a Glove enclave on genuine AWS Nitro hardware. > [!NOTE] > The enclave image measurement for the latest build is -> `6deb9e0a0fd109d1a82f998a0404fc32dffba38fc6895d768f23dc06fcbbdf4817cdb72c3144ba5fe2bb74ea874526aa`. +> `77af85a8afd3a38d4f1bcce280d886eac18b60bb8c2365906b88676fe3739458f7854c5fd8eb27d5eb4858269f6686cb`. # Running Glove diff --git a/client-interface/Cargo.toml b/client-interface/Cargo.toml index 98f6a51..4869b0f 100644 --- a/client-interface/Cargo.toml +++ b/client-interface/Cargo.toml @@ -8,15 +8,17 @@ edition = "2021" common = { path = "../common" } anyhow.workspace = true sp-core.workspace = true +sp-runtime.workspace = true ss58-registry.workspace = true subxt.workspace = true subxt-core.workspace = true subxt-signer.workspace = true thiserror.workspace = true serde.workspace = true +parity-scale-codec.workspace = true +tokio = { workspace = true, features = ["sync"] } [dev-dependencies] serde_json.workspace = true hex.workspace = true rand.workspace = true -parity-scale-codec.workspace = true diff --git a/client-interface/src/lib.rs b/client-interface/src/lib.rs index f9df30e..d878baa 100644 --- a/client-interface/src/lib.rs +++ b/client-interface/src/lib.rs @@ -1,78 +1,148 @@ -use std::fmt::Debug; +use std::fmt; +use std::fmt::{Debug, Formatter}; +use std::future::Future; use std::str::FromStr; +use std::sync::Arc; use anyhow::{Context, Result}; +use parity_scale_codec::Decode; use serde::{Deserialize, Serialize}; -use sp_core::crypto::{AccountId32, Ss58Codec}; +use sp_core::crypto::AccountId32; +use sp_runtime::MultiAddress; use ss58_registry::{Ss58AddressFormat, Ss58AddressFormatRegistry, Token}; use subxt::Error as SubxtError; use subxt::ext::scale_decode::DecodeAsType; -use subxt::OnlineClient; -use subxt::utils::{AccountId32 as SubxtAccountId32, H256}; +use subxt::utils::AccountId32 as SubxtAccountId32; use subxt_core::config::PolkadotConfig; +use subxt_core::ext::sp_core::hexdisplay::AsBytesRef; use subxt_core::tx::payload::Payload; -use subxt_core::utils::MultiAddress; use subxt_signer::SecretUri; use subxt_signer::sr25519; +use tokio::sync::Mutex; use common::attestation::AttestationBundle; +use common::ExtrinsicLocation; +use metadata::proxy::events::ProxyExecuted; +use metadata::referenda::storage::types::referendum_info_for::ReferendumInfoFor; +use metadata::runtime_types::frame_support::traits::preimages::Bounded; +use metadata::runtime_types::pallet_conviction_voting::types::Tally; +use metadata::runtime_types::polkadot_runtime::OriginCaller; use metadata::runtime_types::polkadot_runtime::ProxyType; use metadata::runtime_types::polkadot_runtime::RuntimeCall; use metadata::runtime_types::polkadot_runtime::RuntimeError; use metadata::runtime_types::sp_runtime::DispatchError; +use metadata::runtime_types::sp_runtime::traits::BlakeTwo256; +use metadata::storage; use metadata::utility::events::BatchInterrupted; #[subxt::subxt(runtime_metadata_path = "../assets/polkadot-metadata.scale")] pub mod metadata {} -pub fn parse_secret_phrase(str: &str) -> Result { - Ok(sr25519::Keypair::from_uri(&SecretUri::from_str(str)?)?) -} +pub type OnlineClient = subxt::OnlineClient; +pub type Block = subxt::blocks::Block; +pub type ExtrinsicDetails = subxt::blocks::ExtrinsicDetails; +pub type ExtrinsicEvents = subxt::blocks::ExtrinsicEvents; +pub type SubxtMultiAddressId32 = subxt::utils::MultiAddress; + +pub type ReferendumStatus = metadata::runtime_types::pallet_referenda::types::ReferendumStatus< + u16, + OriginCaller, + u32, + Bounded, + u128, + Tally, + subxt::utils::AccountId32, + (u32, u32) +>; #[derive(Clone)] pub struct SubstrateNetwork { pub url: String, - pub api: OnlineClient, - pub ss58_format: Ss58AddressFormat, + pub api: OnlineClient, + pub network_name: String, pub token_decimals: u8, pub account_key: sr25519::Keypair, + submit_lock: Arc> } impl SubstrateNetwork { pub async fn connect(url: String, account_key: sr25519::Keypair) -> Result { - let api = OnlineClient::::from_url(url.clone()).await + let api = OnlineClient::from_url(url.clone()).await .with_context(|| "Unable to connect to network endpoint:")?; let ss58_address_format = api.constants() .at(&metadata::constants().system().ss58_prefix()) .map(Ss58AddressFormat::custom)?; - let ss58 = Ss58AddressFormatRegistry::try_from(ss58_address_format) + let ss58_address_format_registry = Ss58AddressFormatRegistry::try_from(ss58_address_format) .with_context(|| "Unable to determine network SS58 format")?; - let token_decimals = ss58.tokens() + let mut network_name = ss58_address_format.to_string(); + if network_name == "substrate" { + // For some reason Rococo is mapped to the generic "Substrate" network name. + network_name = "rococo".to_string(); + } + let token_decimals = ss58_address_format_registry.tokens() .first() .map(|token_registry| Token::from(*token_registry).decimals) .unwrap_or(12); - Ok(Self { url, api, ss58_format: ss58.into(), token_decimals, account_key }) + Ok(Self { + url, + api, + network_name, + token_decimals, + account_key, + submit_lock: Arc::default() + }) } pub fn account(&self) -> AccountId32 { self.account_key.public_key().0.into() } + pub async fn get_block(&self, block_number: u32) -> Result, SubxtError> { + let block_hash = self.api + .storage() + .at_latest().await? + .fetch(&storage().system().block_hash(block_number)).await?; + match block_hash { + Some(block_hash) => Ok(Some(self.api.blocks().at(block_hash).await?)), + None => Ok(None) + } + } + + pub async fn get_extrinsic( + &self, + location: ExtrinsicLocation + ) -> Result, SubxtError> { + let Some(block) = self.get_block(location.block_number).await? else { + return Ok(None); + }; + block.extrinsics().await? + .iter() + .nth(location.extrinsic_index as usize) + .transpose() + } + pub async fn call_extrinsic( &self, payload: &Call - ) -> Result<(H256, ExtrinsicEvents), SubxtError> { + ) -> Result<(ExtrinsicEvents, ExtrinsicLocation), SubxtError> { + // Submitting concurrent extrinsics causes problems with the nonce + let guard = self.submit_lock.lock().await; let tx_in_block = self.api.tx() .sign_and_submit_then_watch_default(payload, &self.account_key).await? .wait_for_finalized().await?; - let block_hash = tx_in_block.block_hash(); + // Unlock here as it's now OK for another thread to submit an extrinsic + drop(guard); let events = tx_in_block.wait_for_success().await?; - Ok((block_hash, events)) + let location = ExtrinsicLocation { + block_number: self.api.blocks().at(tx_in_block.block_hash()).await?.number(), + extrinsic_index: events.extrinsic_index() + }; + Ok((events, location)) } pub async fn batch(&self, calls: Vec) -> Result { let payload = metadata::tx().utility().batch(calls).unvalidated(); - let (_, events) = self.call_extrinsic(&payload).await?; + let (events, _) = self.call_extrinsic(&payload).await?; if let Some(batch_interrupted) = events.find_first::()? { let runtime_error = self.extract_runtime_error(&batch_interrupted.error); return if let Some(runtime_error) = runtime_error { @@ -108,12 +178,89 @@ impl SubstrateNetwork { } } - pub fn account_string(&self, account: &AccountId32) -> String { - account.to_ss58check_with_version(self.ss58_format) + pub fn confirm_proxy_executed(&self, events: &ExtrinsicEvents) -> Result<(), ProxyError> { + // Find the first proxy call which failed, if any + for (batch_index, proxy_executed) in events.find::().enumerate() { + match proxy_executed { + Ok(ProxyExecuted { result: Ok(_) }) => continue, + Ok(ProxyExecuted { result: Err(dispatch_error) }) => { + return self.extract_runtime_error(&dispatch_error) + .map_or_else( + || Err(ProxyError::Dispatch(batch_index, dispatch_error)), + |runtime_error| Err(ProxyError::Module(batch_index, runtime_error)) + ) + }, + Err(subxt_error) => return Err(subxt_error.into()) + } + } + Ok(()) + } + + pub async fn subscribe_successful_extrinsics(&self, f: F) -> Result<(), SubxtError> + where + Fut: Future>, + F: Fn(ExtrinsicDetails, ExtrinsicEvents) -> Fut + { + let mut blocks_sub = self.api.blocks().subscribe_finalized().await?; + while let Some(block) = blocks_sub.next().await { + for extrinsic in block?.extrinsics().await?.iter() { + let extrinsic = extrinsic?; + let events = extrinsic.events().await?; + if Self::is_extrinsic_successful(&events)? { + f(extrinsic, events).await?; + } + } + } + Ok(()) + } + + fn is_extrinsic_successful(events: &ExtrinsicEvents) -> Result { + for event in events.iter() { + let event = event?; + if event.pallet_name() == "System" && event.variant_name() == "ExtrinsicFailed" { + return Ok(false); + } + } + Ok(true) + } + + pub async fn account_balance(&self, account: AccountId32) -> Result { + let balance = self.api + .storage() + .at_latest().await? + .fetch(&storage().system().account(core_to_subxt(account))).await? + .map_or(0, |account| account.data.free); + Ok(balance) + } + + pub async fn get_ongoing_poll( + &self, + poll_index: u32 + ) -> Result, SubxtError> { + match self.get_poll(poll_index).await? { + Some(ReferendumInfoFor::Ongoing(status)) => Ok(Some(status)), + _ => Ok(None), + } + } + + pub async fn get_poll(&self, poll_index: u32) -> Result, SubxtError> { + self.api + .storage() + .at_latest().await? + .fetch(&storage().referenda().referendum_info_for(poll_index).unvalidated()).await } } -pub type ExtrinsicEvents = subxt::blocks::ExtrinsicEvents; +impl Debug for SubstrateNetwork { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("SubstrateNetwork") + .field("url", &self.url) + .field("network_name", &self.network_name) + .field("token_decimals", &self.token_decimals) + .field("account", &self.account()) + .finish() + } +} #[derive(thiserror::Error, Debug)] pub enum BatchError { @@ -125,12 +272,37 @@ pub enum BatchError { Subxt(#[from] SubxtError), } +#[derive(thiserror::Error, Debug)] +pub enum ProxyError { + #[error("Module error from batch index {0}: {1:?}")] + Module(usize, RuntimeError), + #[error("Dispatch error from batch index {0}: {1:?}")] + Dispatch(usize, DispatchError), + #[error("Batch error: {0}")] + Batch(#[from] BatchError), + #[error("Internal Subxt error: {0}")] + Subxt(#[from] SubxtError) +} + +impl ProxyError { + pub fn batch_index(&self) -> Option { + match self { + ProxyError::Module(batch_index, _) => Some(*batch_index), + ProxyError::Dispatch(batch_index, _) => Some(*batch_index), + ProxyError::Batch(BatchError::Module(batch_index, _)) => Some(*batch_index), + ProxyError::Batch(BatchError::Dispatch(batch_interrupted)) => + Some(batch_interrupted.index as usize), + _ => None + } + } +} + pub async fn is_glove_member( network: &SubstrateNetwork, client_account: AccountId32, glove_account: AccountId32 ) -> Result { - let proxies_query = metadata::storage() + let proxies_query = storage() .proxy() .proxies(core_to_subxt(client_account)) .unvalidated(); @@ -149,13 +321,29 @@ pub async fn is_glove_member( } } +pub fn account(extrinsic: &ExtrinsicDetails) -> Option { + extrinsic.address_bytes().and_then(parse_multi_address) +} + +pub fn parse_multi_address(bytes: &[u8]) -> Option { + type MultiAddress32 = MultiAddress; + + MultiAddress32::decode(&mut bytes.as_bytes_ref()) + .ok() + .and_then(|address| match address { + MultiAddress::Id(account) => Some(account), + _ => None + }) +} + // Annoyingly, subxt uses a different AccountId32 to sp-core. pub fn core_to_subxt(account: AccountId32) -> SubxtAccountId32 { - SubxtAccountId32::from(Into::<[u8; 32]>::into(account)) + let bytes: [u8; 32] = account.into(); + bytes.into() } -pub fn account_to_address(account: AccountId32) -> MultiAddress { - MultiAddress::Id(core_to_subxt(account)) +pub fn account_to_subxt_multi_address(account: AccountId32) -> SubxtMultiAddressId32 { + SubxtMultiAddressId32::Id(core_to_subxt(account)) } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -172,6 +360,10 @@ pub struct RemoveVoteRequest { pub poll_index: u32 } +pub fn parse_secret_phrase(str: &str) -> Result { + Ok(sr25519::Keypair::from_uri(&SecretUri::from_str(str)?)?) +} + #[cfg(test)] mod tests { use parity_scale_codec::Encode; diff --git a/client/src/lib.rs b/client/src/lib.rs index 6171861..47cc606 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -1,15 +1,11 @@ use std::collections::HashMap; use sp_core::crypto::AccountId32; -use sp_core::Decode; -use sp_runtime::MultiAddress; -use subxt::{Error as SubxtError, OnlineClient, PolkadotConfig}; -use subxt::blocks::ExtrinsicDetails; -use subxt::error::BlockError; -use subxt::ext::subxt_core::ext::sp_core::hexdisplay::AsBytesRef; +use subxt::Error as SubxtError; use attestation::EnclaveInfo; use AttestationBundleLocation::SubstrateRemark; +use client_interface::{account, ExtrinsicDetails, SubstrateNetwork}; use client_interface::metadata::runtime_types; use client_interface::metadata::runtime_types::frame_system::pallet::Call as SystemCall; use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Call as ConvictionVotingCall; @@ -19,7 +15,7 @@ use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall; use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall::ConvictionVoting; use client_interface::metadata::system::calls::types::Remark; use client_interface::metadata::utility::calls::types::Batch; -use common::{AssignedBalance, attestation, BASE_AYE, Conviction, ExtrinsicLocation, GloveResult, GloveVote}; +use common::{AssignedBalance, attestation, BASE_AYE, Conviction, ExtrinsicLocation, GloveResult, VoteDirection}; use common::attestation::{AttestationBundle, AttestationBundleLocation, GloveProof, GloveProofLite}; use runtime_types::pallet_conviction_voting::vote::Vote; @@ -41,8 +37,8 @@ impl VerifiedGloveProof { /// A result of `Ok(None)` means the extrinsic was not a Glove result. pub async fn try_verify_glove_result( - client: &OnlineClient, - extrinsic: &ExtrinsicDetails>, + network: &SubstrateNetwork, + extrinsic: &ExtrinsicDetails, proxy_account: &AccountId32, poll_index: u32 ) -> Result, Error> { @@ -68,7 +64,7 @@ pub async fn try_verify_glove_result( for assigned_balance in &glove_result.assigned_balances { account_votes.get(&assigned_balance.account) .filter(|&&account_vote| { - is_account_vote_consistent(account_vote, glove_result.vote, assigned_balance) + is_account_vote_consistent(account_vote, glove_result.direction, assigned_balance) }) .ok_or(Error::InconsistentVotes)?; } @@ -79,10 +75,10 @@ pub async fn try_verify_glove_result( let attestation_bundle = match glove_proof_lite.attestation_location { SubstrateRemark(remark_location) => - get_attestation_bundle_from_remark(client, remark_location).await? + get_attestation_bundle_from_remark(network, remark_location).await? }; - if attestation_bundle.attested_data.genesis_hash != client.genesis_hash() { + if attestation_bundle.attested_data.genesis_hash != network.api.genesis_hash() { return Err(Error::ChainMismatch); } @@ -104,14 +100,10 @@ pub async fn try_verify_glove_result( } fn parse_glove_proof_lite( - extrinsic: &ExtrinsicDetails>, + extrinsic: &ExtrinsicDetails, proxy_account: &AccountId32 ) -> Result, SubxtError> { - let from_proxy = extrinsic - .address_bytes() - .and_then(parse_multi_address) - .filter(|account| account == proxy_account) - .is_some(); + let from_proxy = account(extrinsic).filter(|account| account == proxy_account).is_some(); if !from_proxy { return Ok(None); } @@ -136,17 +128,6 @@ fn parse_glove_proof_lite( Ok(GloveProofLite::decode_envelope(remark).map(|proof| (proof, batch)).ok()) } -fn parse_multi_address(bytes: &[u8]) -> Option { - type MultiAddress32 = MultiAddress; - - MultiAddress32::decode(&mut bytes.as_bytes_ref()) - .ok() - .and_then(|address| match address { - MultiAddress::Id(account) => Some(account), - _ => None - }) -} - fn parse_and_validate_proxy_account_vote( call: &RuntimeCall, expected_poll_index: u32, @@ -171,19 +152,19 @@ fn parse_and_validate_proxy_account_vote( fn is_account_vote_consistent( account_vote: &AccountVote, - glove_vote: GloveVote, + direction: VoteDirection, assigned_balance: &AssignedBalance ) -> bool { - match glove_vote { - GloveVote::Aye => { + match direction { + VoteDirection::Aye => { parse_standard_account_vote(account_vote) == Some((true, assigned_balance.balance, assigned_balance.conviction)) }, - GloveVote::Nay => { + VoteDirection::Nay => { parse_standard_account_vote(account_vote) == Some((false, assigned_balance.balance, assigned_balance.conviction)) }, - GloveVote::Abstain => { + VoteDirection::Abstain => { parse_abstain_account_vote(account_vote) == Some(assigned_balance.balance) } } @@ -221,22 +202,14 @@ fn parse_abstain_account_vote(account_vote: &AccountVote) -> Option } async fn get_attestation_bundle_from_remark( - client: &OnlineClient, - extrinsic_location: ExtrinsicLocation + network: &SubstrateNetwork, + remark_location: ExtrinsicLocation ) -> Result { - let block_result = client.blocks().at(extrinsic_location.block_hash).await; - if let Err(SubxtError::Block(BlockError::NotFound(_))) = block_result { - return Err(Error::ExtrinsicNotFound(extrinsic_location)); - } - block_result? - .extrinsics().await? - .iter() - .nth(extrinsic_location.block_index as usize) - .transpose()? - .ok_or_else(|| Error::ExtrinsicNotFound(extrinsic_location))? + network.get_extrinsic(remark_location).await? + .ok_or_else(|| Error::ExtrinsicNotFound(remark_location))? .as_extrinsic::()? .ok_or_else(|| Error::InvalidAttestationBundle( - format!("Extrinsic at location {:?} is not a Remark", extrinsic_location) + format!("Extrinsic at location {:?} is not a Remark", remark_location) )) .and_then(|remark| { AttestationBundle::decode_envelope(&remark.remark).map_err(|scale_error| { diff --git a/client/src/main.rs b/client/src/main.rs index 14ede45..2e3ab42 100644 --- a/client/src/main.rs +++ b/client/src/main.rs @@ -14,14 +14,14 @@ use subxt::Error::Runtime; use subxt_signer::sr25519::Keypair; use client::{Error, try_verify_glove_result}; -use client_interface::{account_to_address, is_glove_member}; +use client_interface::{account_to_subxt_multi_address, is_glove_member}; use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::Duplicate; use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::NotFound; use client_interface::metadata::runtime_types::polkadot_runtime::{ProxyType, RuntimeError}; use client_interface::RemoveVoteRequest; use client_interface::ServiceInfo; use client_interface::SubstrateNetwork; -use common::{attestation, Conviction, GloveVote, SignedVoteRequest, VoteRequest}; +use common::{attestation, Conviction, VoteDirection, SignedVoteRequest, VoteRequest}; use common::attestation::{Attestation, EnclaveInfo}; use RuntimeError::Proxy; @@ -67,7 +67,7 @@ async fn join_glove(service_info: &ServiceInfo, network: &SubstrateNetwork) -> R } let add_proxy_call = client_interface::metadata::tx() .proxy() - .add_proxy(account_to_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) + .add_proxy(account_to_subxt_multi_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) .unvalidated(); match network.call_extrinsic(&add_proxy_call).await { Ok(_) => Ok(SuccessOutput::JoinedGlove), @@ -129,46 +129,42 @@ async fn listen_for_glove_votes( nonce: u32, proxy_account: &AccountId32 ) -> Result<()> { - let mut blocks_sub = network.api.blocks().subscribe_finalized().await?; - - while let Some(block) = blocks_sub.next().await { - for extrinsic in block?.extrinsics().await?.iter() { - let verification_result = try_verify_glove_result( - &network.api, - &extrinsic?, - proxy_account, - vote_cmd.poll_index, - ).await; - let verified_glove_proof = match verification_result { - Ok(None) => continue, // Not what we're looking for - Ok(Some(verified_glove_proof)) => verified_glove_proof, - Err(Error::Subxt(subxt_error)) => return Err(subxt_error.into()), - Err(error) => { - eprintln!("Error verifying Glove proof: {}", error); - continue; - } - }; - if let Some(balance) = verified_glove_proof.get_vote_balance(&network.account(), nonce) { - let balance = BigDecimal::new( - balance.into(), - network.token_decimals as i64 - ).with_scale_round(3, RoundingMode::HalfEven); - match verified_glove_proof.result.vote { - GloveVote::Aye => println!("Glove vote aye with balance {}", balance), - GloveVote::Nay => println!("Glove vote nay with balance {}", balance), - GloveVote::Abstain => println!("Glove abstained with balance {}", balance), - } - if let Some(_) = &verified_glove_proof.enclave_info { - // TODO Check measurement - } else { - eprintln!("WARNING: Secure enclave wasn't used"); - } + network.subscribe_successful_extrinsics(|extrinsic, _| async move { + let verification_result = try_verify_glove_result( + &network, + &extrinsic, + proxy_account, + vote_cmd.poll_index, + ).await; + let verified_glove_proof = match verification_result { + Ok(None) => return Ok(()), // Not what we're looking for + Ok(Some(verified_glove_proof)) => verified_glove_proof, + Err(Error::Subxt(subxt_error)) => return Err(subxt_error.into()), + Err(error) => { + eprintln!("Error verifying Glove proof: {}", error); + return Ok(()); + } + }; + if let Some(balance) = verified_glove_proof.get_vote_balance(&network.account(), nonce) { + let balance = BigDecimal::new( + balance.into(), + network.token_decimals as i64 + ).with_scale_round(3, RoundingMode::HalfEven); + match verified_glove_proof.result.direction { + VoteDirection::Aye => println!("Glove vote aye with balance {}", balance), + VoteDirection::Nay => println!("Glove vote nay with balance {}", balance), + VoteDirection::Abstain => println!("Glove abstained with balance {}", balance), + } + if let Some(_) = &verified_glove_proof.enclave_info { + // TODO Check measurement } else { - eprintln!("WARNING: Received Glove proof for poll, but vote was not included"); + eprintln!("WARNING: Secure enclave wasn't used"); } + } else { + eprintln!("WARNING: Received Glove proof for poll, but vote was not included"); } - } - + Ok(()) + }).await?; Ok(()) } @@ -200,7 +196,7 @@ async fn leave_glove(service_info: &ServiceInfo, network: &SubstrateNetwork) -> } let add_proxy_call = client_interface::metadata::tx() .proxy() - .remove_proxy(account_to_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) + .remove_proxy(account_to_subxt_multi_address(service_info.proxy_account.clone()), ProxyType::Governance, 0) .unvalidated(); match network.call_extrinsic(&add_proxy_call).await { Ok(_) => Ok(SuccessOutput::LeftGlove), diff --git a/common/src/attestation.rs b/common/src/attestation.rs index 3fa5a21..0154f13 100644 --- a/common/src/attestation.rs +++ b/common/src/attestation.rs @@ -167,7 +167,7 @@ mod tests { use Attestation::Nitro; - use crate::{AssignedBalance, Conviction, GloveResult, GloveVote, nitro}; + use crate::{AssignedBalance, Conviction, GloveResult, nitro, VoteDirection}; use crate::attestation::Attestation::Mock; use super::*; @@ -199,24 +199,24 @@ mod tests { glove_proof_lite.signed_result.result, GloveResult { poll_index: 185, - vote: GloveVote::Nay, + direction: VoteDirection::Nay, assigned_balances: vec![ AssignedBalance { account: AccountId32::from_str("28836d6f19d5cd8dd8b26da754c63ae337c6f938a7dc6a12e439ad8a1c69fb0d").unwrap(), - nonce: 1406474393, - balance: 870545507137, + nonce: 2023548734, + balance: 1076669561362, conviction: Conviction::None }, AssignedBalance { account: AccountId32::from_str("841f65d84a0ffa95b378923a0d879f188d2a4aa5cb0f97df84fb296788cb6e3e").unwrap(), - nonce: 3000217442, - balance: 7277014506261, + nonce: 3199040729, + balance: 7314891365174, conviction: Conviction::Locked1x }, AssignedBalance { account: AccountId32::from_str("ca22927dff5da60838b78763a2b5ebdf080fa4f35bcbfc8c36b3b6c59a85cd6f").unwrap(), - nonce: 3352555571, - balance: 3691529986602, + nonce: 2821157402, + balance: 3447529073464, conviction: Conviction::Locked3x } ] @@ -226,8 +226,8 @@ mod tests { assert_eq!( glove_proof_lite.attestation_location, AttestationBundleLocation::SubstrateRemark(ExtrinsicLocation { - block_hash: H256::from_str("d7663e131edda194eabbec3ac5695e5b31c9e576e144025956e0cdbf90d4f9cb").unwrap(), - block_index: 2, + block_number: 11256762, + extrinsic_index: 2 }) ); @@ -252,7 +252,7 @@ mod tests { let original_glove_result = GloveProofLite::decode_envelope(GLOVE_PROOF_LITE_BYTES).unwrap().signed_result.result; let mut modified_glove_result = original_glove_result.clone(); - modified_glove_result.vote = GloveVote::Aye; + modified_glove_result.direction = VoteDirection::Aye; assert_ne!(original_glove_result, modified_glove_result); let invalid_glove_proof = GloveProof { diff --git a/common/src/lib.rs b/common/src/lib.rs index a3e32b5..ad3e090 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,6 +1,7 @@ use fmt::Formatter; use std::fmt; use std::fmt::Display; +use std::str::FromStr; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; use rand::random; @@ -78,7 +79,7 @@ pub struct SignedGloveResult { pub struct GloveResult { #[codec(compact)] pub poll_index: u32, - pub vote: GloveVote, + pub direction: VoteDirection, pub assigned_balances: Vec } @@ -90,7 +91,7 @@ impl GloveResult { } #[derive(Debug, Copy, Clone, PartialEq, Encode, Decode)] -pub enum GloveVote { +pub enum VoteDirection { Aye, Nay, Abstain @@ -104,17 +105,28 @@ pub struct AssignedBalance { pub conviction: Conviction } -#[derive(Debug, Copy, Clone, PartialEq, Encode, Decode, MaxEncodedLen)] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, MaxEncodedLen)] pub struct ExtrinsicLocation { - pub block_hash: H256, + pub block_number: u32, /// Index of the extrinsic within the block. #[codec(compact)] - pub block_index: u32 + pub extrinsic_index: u32 } impl Display for ExtrinsicLocation { fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result { - write!(formatter, "{}-{}", self.block_hash, self.block_index) + write!(formatter, "{}-{}", self.block_number, self.extrinsic_index) + } +} + +impl FromStr for ExtrinsicLocation { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let (block_number, extrinsic_index) = s.split_once('-').ok_or("Invalid ExtrinsicLocation format")?; + let block_number = block_number.parse().map_err(|_| "Invalid block number")?; + let extrinsic_index = extrinsic_index.parse().map_err(|_| "Invalid extrinsic index")?; + Ok(ExtrinsicLocation { block_number, extrinsic_index }) } } diff --git a/common/test-resources/glove-proof-lite b/common/test-resources/glove-proof-lite index 52dd36c..80df2e3 100644 Binary files a/common/test-resources/glove-proof-lite and b/common/test-resources/glove-proof-lite differ diff --git a/common/test-resources/secure-nitro-attestation-bundle-envelope b/common/test-resources/secure-nitro-attestation-bundle-envelope index 2302a60..48d1a69 100644 Binary files a/common/test-resources/secure-nitro-attestation-bundle-envelope and b/common/test-resources/secure-nitro-attestation-bundle-envelope differ diff --git a/enclave/src/lib.rs b/enclave/src/lib.rs index d42f519..26858c3 100644 --- a/enclave/src/lib.rs +++ b/enclave/src/lib.rs @@ -6,7 +6,7 @@ use rand::distributions::{Distribution, Uniform}; use rand::thread_rng; use sp_core::H256; -use common::{AssignedBalance, GloveResult, GloveVote, SignedVoteRequest}; +use common::{AssignedBalance, GloveResult, VoteDirection, SignedVoteRequest}; pub fn mix_votes( genesis_hash: H256, @@ -99,12 +99,12 @@ pub fn mix_votes( Ok(GloveResult { poll_index, - vote: if ayes_balance > nays_balance { - GloveVote::Aye + direction: if ayes_balance > nays_balance { + VoteDirection::Aye } else if ayes_balance < nays_balance { - GloveVote::Nay + VoteDirection::Nay } else { - GloveVote::Abstain + VoteDirection::Abstain }, assigned_balances }) @@ -151,7 +151,7 @@ mod tests { mix_votes(GENESIS_HASH, &signed_requests), Ok(GloveResult { poll_index: POLL_INDEX, - vote: GloveVote::Aye, + direction: VoteDirection::Aye, assigned_balances: vec![assigned(&signed_requests[0], 10)] }) ); @@ -167,7 +167,7 @@ mod tests { mix_votes(GENESIS_HASH, &signed_requests), Ok(GloveResult { poll_index: POLL_INDEX, - vote: GloveVote::Abstain, + direction: VoteDirection::Abstain, assigned_balances: vec![assigned(&signed_requests[0], 1), assigned(&signed_requests[1], 1)] }) ) @@ -183,7 +183,7 @@ mod tests { mix_votes(GENESIS_HASH, &signed_requests), Ok(GloveResult { poll_index: POLL_INDEX, - vote: GloveVote::Aye, + direction: VoteDirection::Aye, assigned_balances: vec![assigned(&signed_requests[0], 10), assigned(&signed_requests[1], 5)] }) ); @@ -199,7 +199,7 @@ mod tests { mix_votes(GENESIS_HASH, &signed_requests), Ok(GloveResult { poll_index: POLL_INDEX, - vote: GloveVote::Nay, + direction: VoteDirection::Nay, assigned_balances: vec![assigned(&signed_requests[0], 5), assigned(&signed_requests[1], 10)] }) ); @@ -216,7 +216,7 @@ mod tests { let result = mix_votes(GENESIS_HASH, &signed_requests).unwrap(); println!("{:?}", result); - assert_eq!(result.vote, GloveVote::Aye); + assert_eq!(result.direction, VoteDirection::Aye); assert_eq!(result.assigned_balances.len(), 4); assert_eq!(result.assigned_balances.iter().map(|a| a.balance).sum::(), 20); assert(signed_requests, result.assigned_balances); @@ -230,7 +230,7 @@ mod tests { ]; let result = mix_votes(GENESIS_HASH, &signed_requests).unwrap(); - assert_eq!(result.vote, GloveVote::Nay); + assert_eq!(result.direction, VoteDirection::Nay); assert_eq!(result.assigned_balances.len(), 2); assert_eq!(result.assigned_balances.iter().map(|a| a.balance).sum::(), 17); assert(signed_requests, result.assigned_balances); diff --git a/enclave/src/main.rs b/enclave/src/main.rs index 8976c55..215320a 100644 --- a/enclave/src/main.rs +++ b/enclave/src/main.rs @@ -51,7 +51,6 @@ async fn main() -> anyhow::Result<()> { // break the loop and terminate the enclave as well. loop { let request = stream.read::().await?; - println!("Request: {:?}", request); let response = match request { EnclaveRequest::MixVotes(vote_requests) => process_mix_votes(&vote_requests, &signing_pair, genesis_hash), @@ -123,7 +122,10 @@ fn process_mix_votes( signing_key: &ed25519::Pair, genesis_hash: H256 ) -> EnclaveResponse { - println!("Received request: {:?}", vote_requests); + println!("Received mix votes request:"); + for vote_request in vote_requests { + println!(" {:?}", vote_request); + } match enclave::mix_votes(genesis_hash, &vote_requests) { Ok(glove_result) => EnclaveResponse::GloveResult(glove_result.sign(signing_key)), Err(error) => EnclaveResponse::Error(Error::Mixing(error.to_string())) diff --git a/service/Cargo.toml b/service/Cargo.toml index 4a556b0..a10b49a 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -22,6 +22,9 @@ tracing.workspace = true parity-scale-codec.workspace = true tempfile.workspace = true cfg-if.workspace = true +serde.workspace = true +serde_with.workspace = true +reqwest.workspace = true [target.'cfg(target_os = "linux")'.dependencies] tokio-vsock.workspace = true diff --git a/service/src/lib.rs b/service/src/lib.rs index 5f7f89a..7d74e7c 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::error::Error; use std::future::Future; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use itertools::Itertools; use sp_runtime::AccountId32; @@ -11,13 +11,14 @@ use common::attestation::AttestationBundleLocation; use common::SignedVoteRequest; pub mod enclave; +pub mod subscan; #[derive(Default)] pub struct GloveState { // There may be a non-trivial cost to storing the attestation bundle location, and so it's done // lazily on first poll mixing, rather than eagerly on startup. abl: Mutex>, - polls: Mutex> + polls: RwLock>, } impl GloveState { @@ -39,8 +40,8 @@ impl GloveState { } } - pub async fn get_poll(&self, poll_index: u32) -> Poll { - let mut polls = self.polls.lock().await; + pub fn get_poll(&self, poll_index: u32) -> Poll { + let mut polls = self.polls.write().unwrap(); polls .entry(poll_index) .or_insert_with(|| Poll { @@ -50,17 +51,22 @@ impl GloveState { .clone() } - pub async fn get_optional_poll(&self, poll_index: u32) -> Option { - let polls = self.polls.lock().await; + pub fn get_optional_poll(&self, poll_index: u32) -> Option { + let polls = self.polls.read().unwrap(); polls.get(&poll_index).map(Poll::clone) } - pub async fn remove_poll(&self, poll_index: u32) { - let mut polls = self.polls.lock().await; + pub fn remove_poll(&self, poll_index: u32) { + let mut polls = self.polls.write().unwrap(); polls.remove(&poll_index); } + + pub fn get_polls(&self) -> Vec { + self.polls.read().unwrap().values().cloned().collect() + } } +/// Representing a poll which the Glove proxy will participate in. #[derive(Debug, Clone)] pub struct Poll { pub index: u32, @@ -80,9 +86,9 @@ impl Poll { initiate_mix } - pub async fn remove_vote_request(&self, account: AccountId32) -> Option { + pub async fn remove_vote_request(&self, account: &AccountId32) -> Option { let mut poll = self.inner.lock().await; - let _ = poll.requests.remove(&account)?; + let _ = poll.requests.remove(account)?; let initiate_mix = !poll.pending_mix; poll.pending_mix = true; Some(initiate_mix) @@ -94,13 +100,12 @@ impl Poll { return None; } poll.pending_mix = false; - Some( - poll.requests - .clone() - .into_values() - .sorted_by(|a, b| Ord::cmp(&a.request.account, &b.request.account)) - .collect() - ) + let sorted_requests = poll.requests + .clone() + .into_values() + .sorted_by(|a, b| Ord::cmp(&a.request.account, &b.request.account)) + .collect(); + Some(sorted_requests) } } @@ -130,14 +135,14 @@ mod tests { let account = AccountId32::from([1; 32]); let vote_request = signed_vote_request(account.clone(), 1, true, 10); - let poll = glove_state.get_poll(1).await; + let poll = glove_state.get_poll(1); let pending_mix = poll.add_vote_request(vote_request.clone()).await; assert_eq!(pending_mix, true); let vote_requeats = poll.begin_mix().await; assert_eq!(vote_requeats, Some(vec![vote_request])); - let pending_mix = poll.remove_vote_request(account).await; + let pending_mix = poll.remove_vote_request(&account).await; assert_eq!(pending_mix, Some(true)); let vote_requeats = poll.begin_mix().await; assert_eq!(vote_requeats, Some(vec![])); @@ -148,8 +153,8 @@ mod tests { async fn remove_from_non_existent_poll() { let glove_state = GloveState::default(); let account = AccountId32::from([1; 32]); - let poll = glove_state.get_poll(1).await; - let pending_mix = poll.remove_vote_request(account).await; + let poll = glove_state.get_poll(1); + let pending_mix = poll.remove_vote_request(&account).await; assert_eq!(pending_mix, None); } @@ -160,10 +165,10 @@ mod tests { let account_2 = AccountId32::from([2; 32]); let vote_request = signed_vote_request(account_1.clone(), 1, true, 10); - let poll = glove_state.get_poll(1).await; + let poll = glove_state.get_poll(1); poll.add_vote_request(vote_request.clone()).await; - let pending_mix = poll.remove_vote_request(account_2).await; + let pending_mix = poll.remove_vote_request(&account_2).await; assert_eq!(pending_mix, None); } @@ -174,7 +179,7 @@ mod tests { let vote_request_1 = signed_vote_request(account.clone(), 1, true, 10); let vote_request_2 = signed_vote_request(account.clone(), 1, true, 20); - let poll = glove_state.get_poll(1).await; + let poll = glove_state.get_poll(1); let pending_mix = poll.add_vote_request(vote_request_1.clone()).await; assert_eq!(pending_mix, true); @@ -193,7 +198,7 @@ mod tests { let vote_request_1 = signed_vote_request(account_1.clone(), 1, true, 10); let vote_request_2 = signed_vote_request(account_2.clone(), 1, false, 20); - let poll = glove_state.get_poll(1).await; + let poll = glove_state.get_poll(1); let pending_mix = poll.add_vote_request(vote_request_2.clone()).await; assert_eq!(pending_mix, true); diff --git a/service/src/main.rs b/service/src/main.rs index 4108176..719b9b5 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -10,7 +10,6 @@ use axum::Router; use axum::routing::{get, post}; use cfg_if::cfg_if; use clap::{Parser, ValueEnum}; -use parity_scale_codec::Error as ScaleError; use sp_runtime::AccountId32; use subxt::Error as SubxtError; use subxt_signer::sr25519::Keypair; @@ -23,11 +22,9 @@ use tracing::log::warn; use tracing_subscriber::filter::{EnvFilter, LevelFilter}; use attestation::Error::InsecureMode; -use client_interface::{ExtrinsicEvents, is_glove_member, ServiceInfo, SubstrateNetwork}; -use client_interface::account_to_address; +use client_interface::{account, is_glove_member, ProxyError, ServiceInfo, SubstrateNetwork}; +use client_interface::account_to_subxt_multi_address; use client_interface::BatchError; -use client_interface::core_to_subxt; -use client_interface::metadata::proxy::events::ProxyExecuted; use client_interface::metadata::runtime_types::frame_system::pallet::Call as SystemCall; use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Call as ConvictionVotingCall; use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Error::{InsufficientFunds, NotOngoing, NotVoter}; @@ -35,25 +32,21 @@ use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::A use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::Vote; use client_interface::metadata::runtime_types::pallet_proxy::pallet::Call as ProxyCall; use client_interface::metadata::runtime_types::pallet_proxy::pallet::Error::NotProxy; -use client_interface::metadata::runtime_types::pallet_referenda::types::ReferendumInfo; use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeCall; use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeError; use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeError::Proxy; -use client_interface::metadata::runtime_types::sp_runtime::DispatchError as MetadataDispatchError; -use client_interface::metadata::storage; use client_interface::RemoveVoteRequest; -use common::{AssignedBalance, attestation, BASE_AYE, BASE_NAY, Conviction, ExtrinsicLocation, GloveResult, GloveVote, SignedGloveResult, SignedVoteRequest}; +use common::{AssignedBalance, attestation, BASE_AYE, BASE_NAY, Conviction, GloveResult, SignedGloveResult, SignedVoteRequest, VoteDirection}; use common::attestation::{AttestationBundle, AttestationBundleLocation, GloveProof, GloveProofLite}; use enclave_interface::{EnclaveRequest, EnclaveResponse}; use RuntimeError::ConvictionVoting; -use service::{GloveState, Poll}; +use service::{GloveState, Poll, subscan}; use service::enclave::EnclaveHandle; -use ServiceError::{NotMember, PollNotOngoing, Scale}; +use ServiceError::{NotMember, PollNotOngoing}; +use ServiceError::ChainMismatch; use ServiceError::InsufficientBalance; use ServiceError::InvalidSignature; -use crate::ServiceError::ChainMismatch; - #[derive(Parser, Debug)] #[command(version, about = "Glove proxy service")] struct Args { @@ -86,33 +79,22 @@ enum EnclaveMode { Mock } -// TODO Listen, or poll, for any member who votes directly. -// TODO And only execute on-chain vote batch without them if the service has submitted on-chain already // TODO Test what an actual limit is on batched votes // TODO Load test with ~100 accounts voting on a single poll // TODO Sign the enclave image // TODO Persist voting requests // TODO Restoring state on startup from private store and on-chain // TODO When does the mixing occur? Is it configurable? -// TODO Remove on-chain votes due to error conditions detected by the proxy // TODO Deal with RPC disconnect: -// 2024-06-19T11:36:12.533696Z DEBUG rustls::common_state: Sending warning alert CloseNotify -// 2024-06-19T11:36:12.533732Z DEBUG soketto::connection: 2d71fa53: cannot receive, connection is closed -// 2024-06-19T11:36:12.533743Z DEBUG jsonrpsee-client: Failed to read message: connection closed -// 2024-06-19T11:41:41.247725Z DEBUG request{method=GET uri=/info version=HTTP/1.1}: tower_http::trace::on_request: started processing request -// 2024-06-19T11:41:41.247763Z DEBUG request{method=GET uri=/info version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=200 -// 2024-06-19T11:41:42.195777Z DEBUG request{method=POST uri=/vote version=HTTP/1.1}: tower_http::trace::on_request: started processing request -// 2024-06-19T11:41:42.195924Z WARN request{method=POST uri=/vote version=HTTP/1.1}: service: Subxt(Rpc(ClientError(RestartNeeded(Transport(connection closed -// -// Caused by: -// connection closed))))) -// 2024-06-19T11:41:42.195944Z DEBUG request{method=POST uri=/vote version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=0 ms status=500 -// 2024-06-19T11:41:42.195957Z ERROR request{method=POST uri=/vote version=HTTP/1.1}: tower_http::trace::on_failure: response failed classification=Status code: 500 Internal Server Error latency=0 ms +// 2024-06-19T11:41:42.195924Z WARN request{method=POST uri=/vote version=HTTP/1.1}: service: Subxt(Rpc(ClientError(RestartNeeded(Transport(connection closed +// Caused by: +// connection closed))))) + #[tokio::main] async fn main() -> anyhow::Result<()> { - let filter = EnvFilter::try_new("subxt_core::events=info")? + let filter = EnvFilter::try_new("subxt_core::events=info,hyper_util=info,reqwest::connect=info")? // Set the base level to debug .add_directive(LevelFilter::DEBUG.into()); tracing_subscriber::fmt() @@ -124,7 +106,7 @@ async fn main() -> anyhow::Result<()> { let enclave_handle = initialize_enclave(args.enclave_mode).await?; let network = SubstrateNetwork::connect(args.network_url, args.proxy_secret_phrase).await?; - info!("Connected to Substrate network: {}", network.url); + info!("Connected: {:?}", network); let attestation_bundle = enclave_handle.send_receive::( &network.api.genesis_hash() @@ -143,6 +125,8 @@ async fn main() -> anyhow::Result<()> { state: GloveState::default() }); + start_background_checker(glove_context.clone()); + let router = Router::new() .route("/info", get(info)) .route("/vote", post(vote)) @@ -155,11 +139,17 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -struct GloveContext { - enclave_handle: EnclaveHandle, - attestation_bundle: AttestationBundle, - network: SubstrateNetwork, - state: GloveState +/// Start a background task which polls for Glove violators and removes them. +fn start_background_checker(context: Arc) { + spawn(async move { + loop { + debug!("Checking for Glove violators..."); + if let Err(error) = context.remove_glove_violators().await { + warn!("Error when checking for Glove violators: {:?}", error) + } + sleep(Duration::from_secs(60)).await + } + }); } async fn initialize_enclave(enclave_mode: EnclaveMode) -> io::Result { @@ -223,17 +213,17 @@ async fn vote( if !is_glove_member(network, request.account.clone(), network.account()).await? { return Err(NotMember); } - if !is_poll_ongoing(network, request.poll_index).await? { + if network.get_ongoing_poll(request.poll_index).await?.is_none() { return Err(PollNotOngoing); } // In a normal poll with multiple votes on both sides, the on-chain vote balance can be // significantly less than the vote request balance. A malicious actor could use this to scew // the poll by passing a balance value much higher than they have, knowing there's a good chance // it won't be fully utilised. - if account_balance(network, request.account.clone()).await? < request.balance { + if network.account_balance(request.account.clone()).await? < request.balance { return Err(InsufficientBalance); } - let poll = context.state.get_poll(request.poll_index).await; + let poll = context.state.get_poll(request.poll_index); let initiate_mix = poll.add_vote_request(signed_request).await; if initiate_mix { schedule_vote_mixing(context, poll); @@ -246,15 +236,16 @@ async fn remove_vote( Json(payload): Json ) -> Result<(), ServiceError> { let network = &context.network; - // TODO Does it matter if they're no longer a member? Can we remove the vote regardless? - if !is_glove_member(network, payload.account.clone(), network.account()).await? { + let account = &payload.account; + if !is_glove_member(network, account.clone(), network.account()).await? { return Err(NotMember); } - // Removing a non-existent vote request is a no-op - let Some(poll) = context.state.get_optional_poll(payload.poll_index).await else { + let Some(poll) = context.state.get_optional_poll(payload.poll_index) else { + // Removing a non-existent vote request is a no-op return Ok(()); }; - let Some(initiate_mix) = poll.remove_vote_request(payload.account.clone()).await else { + let Some(initiate_mix) = poll.remove_vote_request(&account).await else { + // Another task has already started mixing the votes return Ok(()); }; @@ -275,26 +266,6 @@ async fn remove_vote( Ok(()) } -async fn is_poll_ongoing(network: &SubstrateNetwork, poll_index: u32) -> Result { - Ok( - network.api - .storage() - .at_latest().await? - .fetch(&storage().referenda().referendum_info_for(poll_index).unvalidated()).await? - .map_or(false, |info| matches!(info, ReferendumInfo::Ongoing(_))) - ) -} - -async fn account_balance(network: &SubstrateNetwork, account: AccountId32) -> Result { - Ok( - network.api - .storage() - .at_latest().await? - .fetch(&storage().system().account(core_to_subxt(account))).await? - .map_or(0, |account| account.data.free) - ) -} - /// Schedule a background task to mix the votes and submit them on-chain after a delay. Any voting // requests which are received in the interim will be included in the mix. fn schedule_vote_mixing(context: Arc, poll: Poll) { @@ -327,7 +298,7 @@ async fn try_mix_votes(context: &GloveContext, poll: &Poll) -> Result Result { - info!("Poll {} has been detected as no longer ongoing, and so removing it", poll.index); - context.state.remove_poll(poll.index).await; + // The background thread will eventually remove the poll + info!("Poll {} is no longer ongoing, and will be removed", poll.index); Ok(true) } ProxyError::Module(batch_index, ConvictionVoting(InsufficientFunds)) => { let request = &poll_requests[batch_index].request; warn!("Insufficient funds for {:?}. Removing it from poll and trying again", request); // TODO On-chain vote needs to be removed as well - poll.remove_vote_request(request.account.clone()).await; + poll.remove_vote_request(&request.account).await; Ok(false) } ProxyError::Batch(BatchError::Module(batch_index, Proxy(NotProxy))) => { let request = &poll_requests[batch_index].request; - warn!("Account is no longer part of the proxy, removing it from poll and trying again: {:?}", + warn!("Account is no longer part of Glove, removing it from poll and trying again: {:?}", request); - // TODO How to remove on-chain vote if the account is no longer part of the proxy? - poll.remove_vote_request(request.account.clone()).await; + poll.remove_vote_request(&request.account).await; Ok(false) } proxy_error => { @@ -370,13 +340,18 @@ async fn try_mix_votes(context: &GloveContext, poll: &Poll) -> Result + vote_requests: Vec ) -> Result { - let request = EnclaveRequest::MixVotes(vote_requests.clone()); - let response = context.enclave_handle.send_receive(&request).await?; - debug!("Mixing result from enclave: {:?}", response); + let request = EnclaveRequest::MixVotes(vote_requests); + let response = context.enclave_handle.send_receive::(&request).await?; match response { EnclaveResponse::GloveResult(signed_result) => { + let result = &signed_result.result; + debug!("Glove result from enclave, poll: {}, direction: {:?}, signature: {:?}", + result.poll_index, result.direction, signed_result.signature); + for assigned_balance in &result.assigned_balances { + debug!(" {:?}", assigned_balance); + } // Double-check things all line up before committing on-chain match GloveProof::verify_components(&signed_result, &context.attestation_bundle) { Ok(_) => debug!("Glove proof verified"), @@ -385,7 +360,10 @@ async fn mix_votes_in_enclave( } Ok(signed_result) } - EnclaveResponse::Error(enclave_error) => Err(enclave_error.into()), + EnclaveResponse::Error(enclave_error) => { + warn!("Mixing error from enclave: {:?}", enclave_error); + Err(enclave_error.into()) + }, } } @@ -428,24 +406,27 @@ async fn submit_glove_result_on_chain( // the same extrinsic in the batch - there's no way of knowing which of them failed. The // `ItemCompleted` events can't be issued, since they're rolled back in light of the error. let events = context.network.batch(batched_calls).await?; - confirm_proxy_executed(&context.network, &events) + context.network.confirm_proxy_executed(&events) } // TODO Deal with mixed_balance of zero fn to_proxied_vote_call(result: &GloveResult, assigned_balance: &AssignedBalance) -> RuntimeCall { RuntimeCall::Proxy( ProxyCall::proxy { - real: account_to_address(assigned_balance.account.clone()), + real: account_to_subxt_multi_address(assigned_balance.account.clone()), force_proxy_type: None, call: Box::new(RuntimeCall::ConvictionVoting(ConvictionVotingCall::vote { poll_index: result.poll_index, - vote: to_account_vote(result.vote, assigned_balance) + vote: to_account_vote(result.direction, assigned_balance) })), } ) } -fn to_account_vote(glove_vote: GloveVote, assigned_balance: &AssignedBalance) -> AccountVote { +fn to_account_vote( + direction: VoteDirection, + assigned_balance: &AssignedBalance +) -> AccountVote { let offset = match assigned_balance.conviction { Conviction::None => 0, Conviction::Locked1x => 1, @@ -456,26 +437,20 @@ fn to_account_vote(glove_vote: GloveVote, assigned_balance: &AssignedBalance) -> Conviction::Locked6x => 6 }; let balance = assigned_balance.balance; - match glove_vote { - GloveVote::Aye => AccountVote::Standard { vote: Vote(BASE_AYE + offset), balance }, - GloveVote::Nay => AccountVote::Standard { vote: Vote(BASE_NAY + offset), balance }, - GloveVote::Abstain => AccountVote::SplitAbstain { aye: 0, nay: 0, abstain: balance } + match direction { + VoteDirection::Aye => AccountVote::Standard { vote: Vote(BASE_AYE + offset), balance }, + VoteDirection::Nay => AccountVote::Standard { vote: Vote(BASE_NAY + offset), balance }, + VoteDirection::Abstain => AccountVote::SplitAbstain { aye: 0, nay: 0, abstain: balance } } } async fn submit_attestation_bundle_location_on_chain( context: &GloveContext ) -> Result { - let compressed = context.attestation_bundle.encode_envelope(); - let payload = client_interface::metadata::tx().system().remark(compressed); + let encoded = context.attestation_bundle.encode_envelope(); let result = context.network - .call_extrinsic(&payload).await - .map(|(block_hash, events)| { - AttestationBundleLocation::SubstrateRemark(ExtrinsicLocation { - block_hash, - block_index: events.extrinsic_index(), - }) - }); + .call_extrinsic(&client_interface::metadata::tx().system().remark(encoded)).await + .map(|(_, location)| AttestationBundleLocation::SubstrateRemark(location)); info!("Stored attestation bundle: {:?}", result); result } @@ -489,7 +464,7 @@ async fn proxy_remove_vote( // error handling. let events = network.batch(vec![RuntimeCall::Proxy( ProxyCall::proxy { - real: account_to_address(account), + real: account_to_subxt_multi_address(account), force_proxy_type: None, call: Box::new(RuntimeCall::ConvictionVoting(ConvictionVotingCall::remove_vote { class: None, @@ -497,29 +472,64 @@ async fn proxy_remove_vote( })), } )]).await?; - confirm_proxy_executed(network, &events) + network.confirm_proxy_executed(&events) } -fn confirm_proxy_executed( - network: &SubstrateNetwork, - events: &ExtrinsicEvents -) -> Result<(), ProxyError> { - // Find the first proxy call which failed, if any - for (batch_index, proxy_executed) in events.find::().enumerate() { - match proxy_executed { - Ok(ProxyExecuted { result: Ok(_) }) => continue, - Ok(ProxyExecuted { result: Err(dispatch_error) }) => { - return network - .extract_runtime_error(&dispatch_error) - .map_or_else( - || Err(ProxyError::Dispatch(batch_index, dispatch_error)), - |runtime_error| Err(ProxyError::Module(batch_index, runtime_error)) - ) - }, - Err(subxt_error) => return Err(subxt_error.into()) +struct GloveContext { + enclave_handle: EnclaveHandle, + attestation_bundle: AttestationBundle, + network: SubstrateNetwork, + state: GloveState +} + +impl GloveContext { + /// Check for voters who have voted outside of Glove and remove them. + async fn remove_glove_violators(&self) -> anyhow::Result<()> { + let mut polls_need_mixing = Vec::new(); + + for poll in self.state.get_polls() { + // Use this opportunity to do some garbage collection and remove any expired polls + if self.network.get_ongoing_poll(poll.index).await?.is_none() { + self.state.remove_poll(poll.index); + continue; + } + for non_glove_voter in self.non_glove_voters(poll.index).await? { + // Remove the voter from the poll if they have submitted a Glove vote + let initiate_mix = match poll.remove_vote_request(&non_glove_voter).await { + Some(initiate_mix) => { + info!("Account {} has voted on poll {} outside of Glove and so removing them", + non_glove_voter, poll.index); + initiate_mix + }, + None => false + }; + if initiate_mix { + polls_need_mixing.push(poll.clone()); + } + } + } + + // TODO Should only be mixed if there are on-chain votes to replace + for poll in polls_need_mixing { + mix_votes(self, &poll).await; } + + Ok(()) + } + + async fn non_glove_voters(&self, poll_index: u32) -> anyhow::Result> { + let mut voters = Vec::new(); + for vote in subscan::get_votes(&self.network, poll_index).await? { + let extrinsic_account = self.network.get_extrinsic(vote.extrinsic_index).await? + .as_ref() + .and_then(account); + if extrinsic_account != Some(self.network.account()) { + // The vote wasn't cast by the Glove proxy + voters.push(vote.account.address); + } + } + Ok(voters) } - Ok(()) } #[derive(thiserror::Error, Debug)] @@ -532,31 +542,6 @@ pub enum MixingError { Attestation(#[from] attestation::Error) } -#[derive(thiserror::Error, Debug)] -pub enum ProxyError { - #[error("Module error from batch index {0}: {1:?}")] - Module(usize, RuntimeError), - #[error("Dispatch error from batch index {0}: {1:?}")] - Dispatch(usize, MetadataDispatchError), - #[error("Batch error: {0}")] - Batch(#[from] BatchError), - #[error("Internal Subxt error: {0}")] - Subxt(#[from] SubxtError) -} - -impl ProxyError { - fn batch_index(&self) -> Option { - match self { - ProxyError::Module(batch_index, _) => Some(*batch_index), - ProxyError::Dispatch(batch_index, _) => Some(*batch_index), - ProxyError::Batch(BatchError::Module(batch_index, _)) => Some(*batch_index), - ProxyError::Batch(BatchError::Dispatch(batch_interrupted)) => - Some(batch_interrupted.index as usize), - _ => None - } - } -} - #[derive(thiserror::Error, Debug)] enum ServiceError { #[error("Signature on signed vote request is invalid")] @@ -569,8 +554,6 @@ enum ServiceError { PollNotOngoing, #[error("Insufficient account balance for vote")] InsufficientBalance, - #[error("Scale decoding error: {0}")] - Scale(#[from] ScaleError), #[error("Proxy error: {0}")] Proxy(#[from] ProxyError), #[error("Internal Subxt error: {0}")] @@ -584,7 +567,6 @@ impl IntoResponse for ServiceError { NotMember => (StatusCode::BAD_REQUEST, self.to_string()), PollNotOngoing => (StatusCode::BAD_REQUEST, self.to_string()), InsufficientBalance => (StatusCode::BAD_REQUEST, self.to_string()), - Scale(_) => (StatusCode::BAD_REQUEST, self.to_string()), _ => { warn!("{:?}", self); (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error".to_string()) diff --git a/service/src/subscan.rs b/service/src/subscan.rs new file mode 100644 index 0000000..677778b --- /dev/null +++ b/service/src/subscan.rs @@ -0,0 +1,81 @@ +use serde::{Deserialize, Serialize}; +use serde_with::DisplayFromStr; +use serde_with::serde_as; +use sp_runtime::AccountId32; +use tracing::debug; + +use client_interface::SubstrateNetwork; +use common::ExtrinsicLocation; + +pub async fn get_votes( + network: &SubstrateNetwork, + poll_index: u32 +) -> Result, reqwest::Error> { + let url = format!("https://{}.api.subscan.io/api/scan/referenda/votes", network.network_name); + let http_client = reqwest::Client::new(); + + let mut all_votes = Vec::new(); + + let mut request = Request { + referendum_index: poll_index, + valid: Valid::Valid, + row: 100, + page: 0, + }; + + loop { + debug!("Fetching votes: {:?}", &request); + let response = http_client + .post(&url) + .json(&request) + .send().await? + .json::().await?; + let Some(mut votes) = response.data.list else { + break; + }; + for vote in &votes { + debug!(" {:?}", vote); + } + all_votes.append(&mut votes); + request.page += 1; + } + + Ok(all_votes) +} + +#[derive(Debug, Clone, Serialize)] +struct Request { + referendum_index: u32, + valid: Valid, + page: u32, + row: u8, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +enum Valid { + Valid, +} + +#[derive(Debug, Clone, Deserialize)] +struct Response { + data: Data, +} + +#[derive(Debug, Clone, Deserialize)] +struct Data { + list: Option>, +} + +#[serde_as] +#[derive(Debug, Clone, Deserialize)] +pub struct ConvictionVote { + pub account: Account, + #[serde_as(as = "DisplayFromStr")] + pub extrinsic_index: ExtrinsicLocation, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Account { + pub address: AccountId32, +}