From 2dda224f8cb89236c7a610773beacd579c2a6bef Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 11:26:26 +0200 Subject: [PATCH 1/8] Add bitswap support. --- client/network/Cargo.toml | 3 + client/network/src/behaviour.rs | 99 +++++++++++--- client/network/src/discovery.rs | 112 +++++++++++++-- client/network/src/gossip.rs | 10 +- client/network/src/lib.rs | 3 + client/network/src/protocol/event.rs | 20 +++ client/network/src/service.rs | 165 ++++++++++++++++++----- client/network/src/service/out_events.rs | 10 ++ client/network/src/service/tests.rs | 2 + 9 files changed, 359 insertions(+), 65 deletions(-) diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index 76d8f0a23eda2..06a7e62694a17 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -62,6 +62,9 @@ unsigned-varint = { version = "0.4.0", features = ["futures", "futures-codec"] } void = "1.0.2" wasm-timer = "0.2" zeroize = "1.0.0" +libp2p-bitswap = "0.7.0" +tiny-cid = "0.2.5" +tiny-multihash = { version = "0.4.5", default-features = false, features = ["std"] } [dependencies.libp2p] version = "0.28.1" diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 6b3cfac38ae99..1e335ade6091c 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -38,6 +38,7 @@ use std::{ task::{Context, Poll}, time::Duration, }; +use tiny_multihash::MultihashDigest; pub use crate::request_responses::{ ResponseFailure, InboundFailure, RequestFailure, OutboundFailure, RequestId, SendRequestError @@ -46,7 +47,7 @@ pub use crate::request_responses::{ /// General behaviour of the network. Combines all protocols together. #[derive(NetworkBehaviour)] #[behaviour(out_event = "BehaviourOut", poll_method = "poll")] -pub struct Behaviour { +pub struct Behaviour { /// All the substrate-specific protocols. substrate: Protocol, /// Periodically pings and identifies the nodes we are connected to, and store information in a @@ -54,6 +55,8 @@ pub struct Behaviour { peer_info: peer_info::PeerInfoBehaviour, /// Discovers nodes of the network. discovery: DiscoveryBehaviour, + /// Exchanges blocks of data with other nodes. + pub(crate) bitswap: libp2p_bitswap::Bitswap, /// Generic request-reponse protocols. request_responses: request_responses::RequestResponsesBehaviour, /// Block request handling. @@ -172,9 +175,21 @@ pub enum BehaviourOut { /// Events generated by a DHT as a response to get_value or put_value requests as well as the /// request duration. Dht(DhtEvent, Duration), + + /// Event generated by bitswap. + Bitswap(BitswapEvent) +} + +/// An event generated by bitswap. +#[derive(Clone, Debug)] +pub enum BitswapEvent { + /// A block was received. + ReceivedBlock(PeerId, tiny_cid::Cid, Box<[u8]>), + /// A WANT request was received. + ReceivedWant(PeerId, tiny_cid::Cid, i32), } -impl Behaviour { +impl Behaviour { /// Builds a new `Behaviour`. pub fn new( substrate: Protocol, @@ -197,6 +212,7 @@ impl Behaviour { finality_proof_requests, light_client_handler, events: VecDeque::new(), + bitswap: libp2p_bitswap::Bitswap::new(), role, }) } @@ -296,6 +312,21 @@ impl Behaviour { self.discovery.put_value(key, value); } + /// Starts querying the providers of a key from the DHT. Will later produce a `Providers` or `GetProvidersFailed` event. + pub fn providers(&mut self, key: &record::Key) { + self.discovery.providers(key); + } + + /// Starts providing a key from the DHT. Will later produce a `Providing` or `StartProvidingFailed` event. + pub fn provide(&mut self, key: &record::Key) { + self.discovery.provide(key); + } + + /// Stops providing a key from the DHT. + pub fn unprovide(&mut self, key: &record::Key) { + self.discovery.unprovide(key); + } + /// Issue a light client request. pub fn light_client_request(&mut self, r: light_client_handler::Request) -> Result<(), light_client_handler::Error> { self.light_client_handler.request(r) @@ -318,15 +349,15 @@ fn reported_roles_to_observed_role(local_role: &Role, remote: &PeerId, roles: Ro } } -impl NetworkBehaviourEventProcess for -Behaviour { +impl NetworkBehaviourEventProcess for +Behaviour { fn inject_event(&mut self, event: void::Void) { void::unreachable(event) } } -impl NetworkBehaviourEventProcess> for -Behaviour { +impl NetworkBehaviourEventProcess> for +Behaviour { fn inject_event(&mut self, event: CustomMessageOutcome) { match event { CustomMessageOutcome::BlockImport(origin, blocks) => @@ -398,7 +429,7 @@ Behaviour { } } -impl NetworkBehaviourEventProcess for Behaviour { +impl NetworkBehaviourEventProcess for Behaviour { fn inject_event(&mut self, event: request_responses::Event) { match event { request_responses::Event::InboundRequest { peer, protocol, result } => { @@ -419,7 +450,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess> for Behaviour { +impl NetworkBehaviourEventProcess> for Behaviour { fn inject_event(&mut self, event: block_requests::Event) { match event { block_requests::Event::AnsweredRequest { peer, total_handling_time } => { @@ -453,7 +484,7 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess> for Behaviour { +impl NetworkBehaviourEventProcess> for Behaviour { fn inject_event(&mut self, event: finality_requests::Event) { match event { finality_requests::Event::Response { peer, block_hash, proof } => { @@ -473,8 +504,8 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, event: peer_info::PeerInfoEvent) { let peer_info::PeerInfoEvent::Identified { peer_id, @@ -499,12 +530,13 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess - for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, out: DiscoveryOut) { match out { DiscoveryOut::UnroutablePeer(_peer_id) => { @@ -514,7 +546,8 @@ impl NetworkBehaviourEventProcess // implementation for `PeerInfoEvent`. } DiscoveryOut::Discovered(peer_id) => { - self.substrate.add_discovered_nodes(iter::once(peer_id)); + self.substrate.add_discovered_nodes(iter::once(peer_id.clone())); + self.bitswap.connect(peer_id); } DiscoveryOut::ValueFound(results, duration) => { self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueFound(results), duration)); @@ -528,16 +561,50 @@ impl NetworkBehaviourEventProcess DiscoveryOut::ValuePutFailed(key, duration) => { self.events.push_back(BehaviourOut::Dht(DhtEvent::ValuePutFailed(key), duration)); } + DiscoveryOut::Providers(key, providers, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::Providers(key, providers), duration)); + } + DiscoveryOut::GetProvidersFailed(key, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::GetProvidersFailed(key), duration)); + } + DiscoveryOut::Providing(key, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::Providing(key), duration)); + } + DiscoveryOut::StartProvidingFailed(key, duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::StartProvidingFailed(key), duration)); + } DiscoveryOut::RandomKademliaStarted(protocols) => { for protocol in protocols { self.events.push_back(BehaviourOut::RandomKademliaStarted(protocol)); } } + DiscoveryOut::BootstrapComplete(duration) => { + self.events.push_back(BehaviourOut::Dht(DhtEvent::BootstrapComplete, duration)); + } + } + } +} + +impl NetworkBehaviourEventProcess + for Behaviour { + fn inject_event(&mut self, event: libp2p_bitswap::BitswapEvent) { + match event { + libp2p_bitswap::BitswapEvent::ReceivedBlock(peer_id, cid, data) => { + self.events.push_back( + BehaviourOut::Bitswap(BitswapEvent::ReceivedBlock(peer_id, cid, data)) + ); + }, + libp2p_bitswap::BitswapEvent::ReceivedWant(peer_id, cid, priority) => { + self.events.push_back( + BehaviourOut::Bitswap(BitswapEvent::ReceivedWant(peer_id, cid, priority)) + ); + } + libp2p_bitswap::BitswapEvent::ReceivedCancel(..) => {}, } } } -impl Behaviour { +impl Behaviour { fn poll(&mut self, _: &mut Context, _: &mut impl PollParameters) -> Poll>> { if let Some(event) = self.events.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)) diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index 6ef97708c1336..cad33549cad02 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -54,7 +54,7 @@ use libp2p::core::{connection::{ConnectionId, ListenerId}, ConnectedPoint, Multi use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler}; use libp2p::swarm::protocols_handler::multi::MultiHandler; use libp2p::kad::{Kademlia, KademliaBucketInserts, KademliaConfig, KademliaEvent, QueryResult, Quorum, Record}; -use libp2p::kad::GetClosestPeersError; +use libp2p::kad::{BootstrapOk, BootstrapError, GetClosestPeersError}; use libp2p::kad::handler::KademliaHandler; use libp2p::kad::QueryId; use libp2p::kad::record::{self, store::{MemoryStore, RecordStore}}; @@ -173,7 +173,10 @@ impl DiscoveryConfig { } /// Create a `DiscoveryBehaviour` from this config. - pub fn finish(self) -> DiscoveryBehaviour { + pub fn finish(mut self) -> DiscoveryBehaviour { + for (_, k) in self.kademlias.iter_mut() { + k.bootstrap().ok(); + } DiscoveryBehaviour { user_defined: self.user_defined, kademlias: self.kademlias, @@ -307,6 +310,32 @@ impl DiscoveryBehaviour { } } + /// Starts providing a record. + pub fn provide(&mut self, key: &record::Key) { + for k in self.kademlias.values_mut() { + if let Err(e) = k.start_providing(key.clone()) { + warn!( + target: "sub-libp2p", + "Libp2p => Failed to start providing: {:?}", e); + self.pending_events.push_back(DiscoveryOut::StartProvidingFailed(key.clone(), Duration::from_secs(0))); + } + } + } + + /// Stops providing a record. + pub fn unprovide(&mut self, key: &record::Key) { + for k in self.kademlias.values_mut() { + k.stop_providing(key); + } + } + + /// Gets the providers of a record. + pub fn providers(&mut self, key: &record::Key) { + for k in self.kademlias.values_mut() { + k.get_providers(key.clone()); + } + } + /// Start fetching a record from the DHT. /// /// A corresponding `ValueFound` or `ValueNotFound` event will later be generated. @@ -399,6 +428,18 @@ pub enum DiscoveryOut { /// the `identify` protocol. UnroutablePeer(PeerId), + /// A set of providers found for a key. The set may be empty. + Providers(record::Key, HashSet, Duration), + + /// Getting providers for a key failed. + GetProvidersFailed(record::Key, Duration), + + /// Started providing a value in the dht. + Providing(record::Key, Duration), + + /// Providing a value in the dht failed. + StartProvidingFailed(record::Key, Duration), + /// The DHT yielded results for the record request. /// /// Returning the result grouped in (key, value) pairs as well as the request duration.. @@ -421,6 +462,9 @@ pub enum DiscoveryOut { /// Started a random Kademlia query for each DHT identified by the given `ProtocolId`s. RandomKademliaStarted(Vec), + + /// Bootstrap completed. + BootstrapComplete(Duration), } impl NetworkBehaviour for DiscoveryBehaviour { @@ -658,6 +702,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { } } KademliaEvent::QueryResult { result: QueryResult::GetRecord(res), stats, .. } => { + let duration = stats.duration().unwrap_or_else(Default::default); let ev = match res { Ok(ok) => { let results = ok.records @@ -665,28 +710,29 @@ impl NetworkBehaviour for DiscoveryBehaviour { .map(|r| (r.record.key, r.record.value)) .collect(); - DiscoveryOut::ValueFound(results, stats.duration().unwrap_or_else(Default::default)) + DiscoveryOut::ValueFound(results, duration) } Err(e @ libp2p::kad::GetRecordError::NotFound { .. }) => { trace!(target: "sub-libp2p", "Libp2p => Failed to get record: {:?}", e); - DiscoveryOut::ValueNotFound(e.into_key(), stats.duration().unwrap_or_else(Default::default)) + DiscoveryOut::ValueNotFound(e.into_key(), duration) } Err(e) => { warn!(target: "sub-libp2p", "Libp2p => Failed to get record: {:?}", e); - DiscoveryOut::ValueNotFound(e.into_key(), stats.duration().unwrap_or_else(Default::default)) + DiscoveryOut::ValueNotFound(e.into_key(), duration) } }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); } KademliaEvent::QueryResult { result: QueryResult::PutRecord(res), stats, .. } => { + let duration = stats.duration().unwrap_or_else(Default::default); let ev = match res { - Ok(ok) => DiscoveryOut::ValuePut(ok.key, stats.duration().unwrap_or_else(Default::default)), + Ok(ok) => DiscoveryOut::ValuePut(ok.key, duration), Err(e) => { warn!(target: "sub-libp2p", "Libp2p => Failed to put record: {:?}", e); - DiscoveryOut::ValuePutFailed(e.into_key(), stats.duration().unwrap_or_else(Default::default)) + DiscoveryOut::ValuePutFailed(e.into_key(), duration) } }; return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); @@ -701,9 +747,55 @@ impl NetworkBehaviour for DiscoveryBehaviour { e.key(), e) } } - // We never start any other type of query. - e => { - warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e) + KademliaEvent::QueryResult { result: QueryResult::GetProviders(res), stats, .. } => { + let duration = stats.duration().unwrap_or_else(Default::default); + let ev = match res { + Ok(ok) => DiscoveryOut::Providers(ok.key, ok.providers, duration), + Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Getting providers for {:?} failed with: {:?}", + e.key(), e); + DiscoveryOut::GetProvidersFailed(e.into_key(), duration) + } + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + KademliaEvent::QueryResult { result: QueryResult::StartProviding(res), stats, .. } => { + let duration = stats.duration().unwrap_or_else(Default::default); + let ev = match res { + Ok(ok) => DiscoveryOut::Providing(ok.key, duration), + Err(e) => { + warn!(target: "sub-libp2p", + "Libp2p => Failed to put record: {:?}", e); + DiscoveryOut::StartProvidingFailed(e.into_key(), duration) + } + }; + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + KademliaEvent::QueryResult { result: QueryResult::RepublishProvider(res), .. } => { + match res { + Ok(ok) => debug!(target: "sub-libp2p", + "Libp2p => Provider republished: {:?}", + ok.key), + Err(e) => warn!(target: "sub-libp2p", + "Libp2p => Republishing of provider {:?} failed with: {:?}", + e.key(), e) + } + } + KademliaEvent::QueryResult { result: QueryResult::Bootstrap(res), stats, .. } => { + let num_remaining = match res { + Ok(BootstrapOk { num_remaining, .. }) => Some(num_remaining), + Err(BootstrapError::Timeout { num_remaining, .. }) => num_remaining, + }; + if let Some(num_remaining) = num_remaining { + if num_remaining == 0 { + debug!(target: "sub-libp2p", "Libp2p => Bootstrap complete"); + let duration = stats.duration().unwrap_or_else(Default::default); + let ev = DiscoveryOut::BootstrapComplete(duration); + return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)); + } + } + // TODO: unhandled bootstrap events } } NetworkBehaviourAction::DialAddress { address } => diff --git a/client/network/src/gossip.rs b/client/network/src/gossip.rs index 0650e7a2f818b..eefd97954ca9f 100644 --- a/client/network/src/gossip.rs +++ b/client/network/src/gossip.rs @@ -59,6 +59,7 @@ use std::{ sync::{atomic, Arc}, time::Duration, }; +use tiny_multihash::MultihashDigest; #[cfg(test)] mod tests; @@ -75,8 +76,8 @@ impl QueuedSender { /// /// In addition to the [`QueuedSender`], also returns a `Future` whose role is to drive /// the messages sending forward. - pub fn new( - service: Arc>, + pub fn new( + service: Arc>, peer_id: PeerId, protocol: ConsensusEngineId, queue_size_limit: usize, @@ -86,6 +87,7 @@ impl QueuedSender { M: Send + 'static, B: BlockT + 'static, H: ExHashT, + MH: MultihashDigest, F: Fn(M) -> Vec + Send + 'static, { let shared = Arc::new(Shared { @@ -198,8 +200,8 @@ struct Shared { queue_size_limit: usize, } -async fn spawn_task Vec>( - service: Arc>, +async fn spawn_task Vec>( + service: Arc>, peer_id: PeerId, protocol: ConsensusEngineId, shared: Arc>, diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 3fd01c33dcf5f..881986bdeeaac 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -266,6 +266,8 @@ pub mod network_state; #[doc(inline)] pub use libp2p::{multiaddr, Multiaddr, PeerId}; +pub use libp2p::kad::record::Key; +pub use behaviour::BitswapEvent; pub use protocol::{event::{DhtEvent, Event, ObservedRole}, sync::SyncState, PeerInfo}; pub use service::{ NetworkService, NetworkWorker, RequestFailure, OutboundFailure, NotificationSender, @@ -273,6 +275,7 @@ pub use service::{ }; pub use sc_peerset::ReputationChange; +pub use tiny_multihash::MultihashDigest; use sp_runtime::traits::{Block as BlockT, NumberFor}; /// The maximum allowed number of established connections per peer. diff --git a/client/network/src/protocol/event.rs b/client/network/src/protocol/event.rs index 637bf805b5024..d1603a88daed3 100644 --- a/client/network/src/protocol/event.rs +++ b/client/network/src/protocol/event.rs @@ -21,6 +21,8 @@ use bytes::Bytes; use libp2p::core::PeerId; use libp2p::kad::record::Key; use sp_runtime::ConsensusEngineId; +use std::collections::HashSet; +use crate::behaviour::BitswapEvent; /// Events generated by DHT as a response to get_value and put_value requests. #[derive(Debug, Clone)] @@ -37,6 +39,21 @@ pub enum DhtEvent { /// An error has occurred while putting a record into the DHT. ValuePutFailed(Key), + + /// A set of providers found for a key. The set may be empty. + Providers(Key, HashSet), + + /// Getting providers for a key failed. + GetProvidersFailed(Key), + + /// Started providing a value in the dht. + Providing(Key), + + /// Providing a value in the dht failed. + StartProvidingFailed(Key), + + /// Bootstrap completed. + BootstrapComplete, } /// Type for events generated by networking layer. @@ -46,6 +63,9 @@ pub enum Event { /// Event generated by a DHT. Dht(DhtEvent), + /// Event generated by Bitswap. + Bitswap(BitswapEvent), + /// Opened a substream with the given node with the given notifications protocol. /// /// The protocol is always one of the notification protocols that have been registered. diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 59f55f01a45d1..1e7de31a21283 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -72,6 +72,7 @@ use std::{ }, task::Poll, }; +use tiny_multihash::MultihashDigest; use wasm_timer::Instant; pub use behaviour::{ResponseFailure, InboundFailure, RequestFailure, OutboundFailure}; @@ -82,7 +83,7 @@ mod out_events; mod tests; /// Substrate network service. Handles network IO and manages connectivity. -pub struct NetworkService { +pub struct NetworkService { /// Number of peers we're connected to. num_connected: Arc, /// The local external addresses. @@ -108,16 +109,16 @@ pub struct NetworkService { notifications_sizes_metric: Option, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. - _marker: PhantomData, + _marker: PhantomData<(H, M)>, } -impl NetworkWorker { +impl NetworkWorker { /// Creates the network service. /// /// Returns a `NetworkWorker` that implements `Future` and must be regularly polled in order /// for the network processing to advance. From it, you can extract a `NetworkService` using /// `worker.service()`. The `NetworkService` can be shared through the codebase. - pub fn new(params: Params) -> Result, Error> { + pub fn new(params: Params) -> Result, Error> { // Ensure the listen addresses are consistent with the transport. ensure_addresses_consistent_with_transport( params.network_config.listen_addresses.iter(), @@ -264,7 +265,7 @@ impl NetworkWorker { )?; // Build the swarm. - let (mut swarm, bandwidth): (Swarm, _) = { + let (mut swarm, bandwidth): (Swarm, _) = { let user_agent = format!( "{} ({})", params.network_config.client_version, @@ -373,14 +374,14 @@ impl NetworkWorker { // Listen on multiaddresses. for addr in ¶ms.network_config.listen_addresses { - if let Err(err) = Swarm::::listen_on(&mut swarm, addr.clone()) { + if let Err(err) = Swarm::::listen_on(&mut swarm, addr.clone()) { warn!(target: "sub-libp2p", "Can't listen on {} because: {:?}", addr, err) } } // Add external addresses. for addr in ¶ms.network_config.public_addresses { - Swarm::::add_external_address(&mut swarm, addr.clone()); + Swarm::::add_external_address(&mut swarm, addr.clone()); } let external_addresses = Arc::new(Mutex::new(Vec::new())); @@ -491,7 +492,7 @@ impl NetworkWorker { /// Return a `NetworkService` that can be shared through the code base and can be used to /// manipulate the worker. - pub fn service(&self) -> &Arc> { + pub fn service(&self) -> &Arc> { &self.service } @@ -509,14 +510,14 @@ impl NetworkWorker { /// Returns the local `PeerId`. pub fn local_peer_id(&self) -> &PeerId { - Swarm::::local_peer_id(&self.network_service) + Swarm::::local_peer_id(&self.network_service) } /// Returns the list of addresses we are listening on. /// /// Does **NOT** include a trailing `/p2p/` with our `PeerId`. pub fn listen_addresses(&self) -> impl Iterator { - Swarm::::listeners(&self.network_service) + Swarm::::listeners(&self.network_service) } /// Get network state. @@ -570,9 +571,9 @@ impl NetworkWorker { }; NetworkState { - peer_id: Swarm::::local_peer_id(&swarm).to_base58(), - listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), - external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), + peer_id: Swarm::::local_peer_id(&swarm).to_base58(), + listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), + external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), connected_peers, not_connected_peers, peerset: swarm.user_protocol_mut().peerset_debug_info(), @@ -597,9 +598,24 @@ impl NetworkWorker { pub fn add_reserved_peer(&self, peer: String) -> Result<(), String> { self.service.add_reserved_peer(peer) } + + /// Get the number of bitswap peers we are connected to. + pub fn bitswap_num_peers(&self) -> usize { + self.network_service.bitswap.peers().count() + } + + /// Get the number of bitswap peers who want a block. + pub fn bitswap_num_peers_want(&self, cid: &tiny_cid::Cid) -> usize { + self.network_service.bitswap.peers_want(cid).count() + } + + /// Get if a specific bitswap peer wants a block. + pub fn bitswap_peer_wants_cid(&self, peer_id: &PeerId, cid: &tiny_cid::Cid) -> bool { + self.network_service.bitswap.peers_want(cid).find(|id| **id == *peer_id).is_some() + } } -impl NetworkService { +impl NetworkService { /// Returns the local `PeerId`. pub fn local_peer_id(&self) -> &PeerId { &self.local_peer_id @@ -923,6 +939,27 @@ impl NetworkService { .unbounded_send(ServiceToWorkerMsg::PutValue(key, value)); } + /// Get providers of a key from the DHT. + pub fn providers(&self, key: record::Key) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::GetProviders(key)); + } + + /// Start providing a key. + pub fn provide(&self, key: record::Key) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::StartProviding(key)); + } + + /// Stop providing a key. + pub fn unprovide(&self, key: record::Key) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::StopProviding(key)); + } + /// Connect to unreserved peers and allow unreserved peers to connect. pub fn accept_unreserved_peers(&self) { self.peerset.set_reserved_only(false); @@ -1027,10 +1064,38 @@ impl NetworkService { .to_worker .unbounded_send(ServiceToWorkerMsg::OwnBlockImported(hash, number)); } + + /// Send a bitswap block to a peer. + pub fn bitswap_send_block(&self, peer_id: PeerId, cid: tiny_cid::Cid, data: Box<[u8]>) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::BitswapSendBlock(peer_id, cid, data)); + } + + /// Send a bitswap block to all peers that have the block in their wantlist. + pub fn bitswap_send_block_all(&self, cid: tiny_cid::Cid, data: Box<[u8]>) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::BitswapSendBlockAll(cid, data)); + } + + /// Send a bitswap WANT request to all peers for a block. + pub fn bitswap_want_block(&self, cid: tiny_cid::Cid, priority: i32) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::BitswapWantBlock(cid, priority)); + } + + /// Cancel a bitswap WANT request. + pub fn bitswap_cancel_block(&self, cid: tiny_cid::Cid) { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::BitswapCancelBlock(cid)); + } } -impl sp_consensus::SyncOracle - for NetworkService +impl sp_consensus::SyncOracle + for NetworkService { fn is_major_syncing(&mut self) -> bool { NetworkService::is_major_syncing(self) @@ -1041,8 +1106,8 @@ impl sp_consensus::SyncOracle } } -impl<'a, B: BlockT + 'static, H: ExHashT> sp_consensus::SyncOracle - for &'a NetworkService +impl<'a, B: BlockT + 'static, H: ExHashT, M: MultihashDigest> sp_consensus::SyncOracle + for &'a NetworkService { fn is_major_syncing(&mut self) -> bool { NetworkService::is_major_syncing(self) @@ -1053,10 +1118,11 @@ impl<'a, B: BlockT + 'static, H: ExHashT> sp_consensus::SyncOracle } } -impl NetworkStateInfo for NetworkService +impl NetworkStateInfo for NetworkService where B: sp_runtime::traits::Block, H: ExHashT, + M: MultihashDigest, { /// Returns the local external addresses. fn external_addresses(&self) -> Vec { @@ -1143,6 +1209,9 @@ enum ServiceToWorkerMsg { AnnounceBlock(B::Hash, Vec), GetValue(record::Key), PutValue(record::Key, Vec), + GetProviders(record::Key), + StartProviding(record::Key), + StopProviding(record::Key), AddKnownAddress(PeerId, Multiaddr), SyncFork(Vec, B::Hash, NumberFor), EventStream(out_events::Sender), @@ -1159,13 +1228,17 @@ enum ServiceToWorkerMsg { DisconnectPeer(PeerId), UpdateChain, OwnBlockImported(B::Hash, NumberFor), + BitswapSendBlock(PeerId, tiny_cid::Cid, Box<[u8]>), + BitswapSendBlockAll(tiny_cid::Cid, Box<[u8]>), + BitswapWantBlock(tiny_cid::Cid, i32), + BitswapCancelBlock(tiny_cid::Cid), } /// Main network worker. Must be polled in order for the network to advance. /// /// You are encouraged to poll this in a separate background thread or task. #[must_use = "The NetworkWorker must be polled in order for the network to advance"] -pub struct NetworkWorker { +pub struct NetworkWorker { /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. external_addresses: Arc>>, /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. @@ -1173,9 +1246,9 @@ pub struct NetworkWorker { /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. is_major_syncing: Arc, /// The network service that can be extracted and shared through the codebase. - service: Arc>, + service: Arc>, /// The *actual* network. - network_service: Swarm, + network_service: Swarm, /// The import queue that was passed at initialization. import_queue: Box>, /// Messages from the [`NetworkService`] that must be processed. @@ -1200,7 +1273,7 @@ pub struct NetworkWorker { peers_notifications_sinks: Arc>>, } -impl Future for NetworkWorker { +impl Future for NetworkWorker { type Output = (); fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context) -> Poll { @@ -1260,6 +1333,12 @@ impl Future for NetworkWorker { this.network_service.get_value(&key), ServiceToWorkerMsg::PutValue(key, value) => this.network_service.put_value(key, value), + ServiceToWorkerMsg::GetProviders(key) => + this.network_service.providers(&key), + ServiceToWorkerMsg::StartProviding(key) => + this.network_service.provide(&key), + ServiceToWorkerMsg::StopProviding(key) => + this.network_service.unprovide(&key), ServiceToWorkerMsg::AddKnownAddress(peer_id, addr) => this.network_service.add_known_address(peer_id, addr), ServiceToWorkerMsg::SyncFork(peer_ids, hash, number) => @@ -1301,6 +1380,14 @@ impl Future for NetworkWorker { this.network_service.user_protocol_mut().update_chain(), ServiceToWorkerMsg::OwnBlockImported(hash, number) => this.network_service.user_protocol_mut().own_block_imported(hash, number), + ServiceToWorkerMsg::BitswapSendBlock(peer_id, cid, block) => + this.network_service.bitswap.send_block(&peer_id, cid, block), + ServiceToWorkerMsg::BitswapSendBlockAll(cid, block) => + this.network_service.bitswap.send_block_all(&cid, &block), + ServiceToWorkerMsg::BitswapWantBlock(cid, priority) => + this.network_service.bitswap.want_block(cid, priority), + ServiceToWorkerMsg::BitswapCancelBlock(cid) => + this.network_service.bitswap.cancel_block(&cid), } } @@ -1499,6 +1586,11 @@ impl Future for NetworkWorker { DhtEvent::ValueNotFound(_) => "value-not-found", DhtEvent::ValuePut(_) => "value-put", DhtEvent::ValuePutFailed(_) => "value-put-failed", + DhtEvent::Providers(_, _) => "get-providers", + DhtEvent::GetProvidersFailed(_) => "get-providers-failed", + DhtEvent::Providing(_) => "start-providing", + DhtEvent::StartProvidingFailed(_) => "start-providing-failed", + DhtEvent::BootstrapComplete => "bootstrap-complete", }; metrics.kademlia_query_duration.with_label_values(&[query_type]) .observe(duration.as_secs_f64()); @@ -1506,6 +1598,9 @@ impl Future for NetworkWorker { this.event_streams.send(Event::Dht(event)); }, + Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Bitswap(ev))) => { + this.event_streams.send(Event::Bitswap(ev)); + }, Poll::Ready(SwarmEvent::ConnectionEstablished { peer_id, endpoint, num_established }) => { trace!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); @@ -1531,14 +1626,14 @@ impl Future for NetworkWorker { let reason = match cause { Some(ConnectionError::IO(_)) => "transport-error", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A(EitherError::B( - EitherError::A(PingFailure::Timeout)))))))))) => "ping-timeout", + EitherError::A(EitherError::A(EitherError::A(EitherError::A(EitherError::B( + EitherError::A(PingFailure::Timeout))))))))))) => "ping-timeout", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A(EitherError::A( - NotifsHandlerError::Legacy(LegacyConnectionKillError)))))))))) => "force-closed", + EitherError::A(EitherError::A(EitherError::A(EitherError::A(EitherError::A( + NotifsHandlerError::Legacy(LegacyConnectionKillError))))))))))) => "force-closed", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(EitherError::A(EitherError::A( - EitherError::A(EitherError::A(EitherError::A(EitherError::A( - NotifsHandlerError::SyncNotificationsClogged))))))))) => "sync-notifications-clogged", + EitherError::A(EitherError::A(EitherError::A(EitherError::A(EitherError::A( + NotifsHandlerError::SyncNotificationsClogged)))))))))) => "sync-notifications-clogged", Some(ConnectionError::Handler(NodeHandlerWrapperError::Handler(_))) => "protocol-error", Some(ConnectionError::Handler(NodeHandlerWrapperError::KeepAliveTimeout)) => "keep-alive-timeout", None => "actively-closed", @@ -1658,7 +1753,7 @@ impl Future for NetworkWorker { // Update the variables shared with the `NetworkService`. this.num_connected.store(num_connected_peers, Ordering::Relaxed); { - let external_addresses = Swarm::::external_addresses(&this.network_service).cloned().collect(); + let external_addresses = Swarm::::external_addresses(&this.network_service).cloned().collect(); *this.external_addresses.lock() = external_addresses; } @@ -1692,7 +1787,7 @@ impl Future for NetworkWorker { } } -impl Unpin for NetworkWorker { +impl Unpin for NetworkWorker { } /// Turns bytes that are potentially UTF-8 into a reasonable representable string. @@ -1707,14 +1802,14 @@ pub(crate) fn maybe_utf8_bytes_to_string(id: &[u8]) -> Cow { } /// The libp2p swarm, customized for our needs. -type Swarm = libp2p::swarm::Swarm>; +type Swarm = libp2p::swarm::Swarm>; // Implementation of `import_queue::Link` trait using the available local variables. -struct NetworkLink<'a, B: BlockT, H: ExHashT> { - protocol: &'a mut Swarm, +struct NetworkLink<'a, B: BlockT, H: ExHashT, M: MultihashDigest> { + protocol: &'a mut Swarm, } -impl<'a, B: BlockT, H: ExHashT> Link for NetworkLink<'a, B, H> { +impl<'a, B: BlockT, H: ExHashT, M: MultihashDigest> Link for NetworkLink<'a, B, H, M> { fn blocks_processed( &mut self, imported: usize, diff --git a/client/network/src/service/out_events.rs b/client/network/src/service/out_events.rs index 1b86a5fa4317d..2eb13d2cc93f2 100644 --- a/client/network/src/service/out_events.rs +++ b/client/network/src/service/out_events.rs @@ -227,6 +227,11 @@ impl Metrics { self.events_total .with_label_values(&["dht", "sent", name]) .inc_by(num); + }, + Event::Bitswap(_) => { + self.events_total + .with_label_values(&["bitswap", "sent", name]) + .inc_by(num); } Event::NotificationStreamOpened { engine_id, .. } => { self.events_total @@ -258,6 +263,11 @@ impl Metrics { .with_label_values(&["dht", "received", name]) .inc(); } + Event::Bitswap(_) => { + self.events_total + .with_label_values(&["bitswap", "received", name]) + .inc(); + } Event::NotificationStreamOpened { engine_id, .. } => { self.events_total .with_label_values(&[&format!("notif-open-{:?}", engine_id), "received", name]) diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 4b6f9dd156482..98ed7b2e2c6f2 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -271,6 +271,8 @@ fn notifications_state_consistent() { // Add new events here. future::Either::Left(Event::Dht(_)) => {} future::Either::Right(Event::Dht(_)) => {} + future::Either::Left(Event::Bitswap(_)) => {} + future::Either::Right(Event::Bitswap(_)) => {} }; } }); From 1d6becbdc0ec49d15ba8dd56b4cb7c82412951a2 Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 11:23:09 +0200 Subject: [PATCH 2/8] Remove BAD_ROLE. --- client/network/src/protocol.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index c1887ce35bfdb..17f55573ce8d6 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -118,8 +118,6 @@ mod rep { pub const GENESIS_MISMATCH: Rep = Rep::new_fatal("Genesis mismatch"); /// Peer is on unsupported protocol version. pub const BAD_PROTOCOL: Rep = Rep::new_fatal("Unsupported protocol"); - /// Peer role does not match (e.g. light peer connecting to another light peer). - pub const BAD_ROLE: Rep = Rep::new_fatal("Unsupported role"); /// Peer response data does not have requested bits. pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); } @@ -867,15 +865,7 @@ impl Protocol { } if self.config.roles.is_light() { - // we're not interested in light peers - if status.roles.is_light() { - debug!(target: "sync", "Peer {} is unable to serve light requests", who); - self.peerset_handle.report_peer(who.clone(), rep::BAD_ROLE); - self.behaviour.disconnect_peer(&who); - return CustomMessageOutcome::None; - } - - // we don't interested in peers that are far behind us + // we aren't interested in peers that are far behind us let self_best_block = self .context_data .chain From 3ca64800206bebeea5ef50b56935442bf0fdb7cd Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 11:30:42 +0200 Subject: [PATCH 3/8] Propagate generic. --- bin/node-template/node/Cargo.toml | 1 + bin/node-template/node/src/service.rs | 5 +-- bin/node/cli/Cargo.toml | 1 + bin/node/cli/src/service.rs | 5 +-- client/authority-discovery/src/worker.rs | 5 ++- .../finality-grandpa/src/communication/mod.rs | 5 +-- client/network-gossip/src/bridge.rs | 3 +- client/network-gossip/src/lib.rs | 4 +-- client/network/test/Cargo.toml | 1 + client/network/test/src/lib.rs | 4 +-- client/offchain/src/lib.rs | 7 ++-- client/service/src/builder.rs | 32 +++++++++++-------- client/service/src/lib.rs | 6 ++-- client/service/test/Cargo.toml | 1 + client/service/test/src/lib.rs | 9 +++--- 15 files changed, 54 insertions(+), 35 deletions(-) diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 8b1a47fd2bf15..a8cd74560d481 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -17,6 +17,7 @@ name = "node-template" [dependencies] structopt = "0.3.8" +tiny-multihash = "0.4.5" sc-cli = { version = "0.8.0", path = "../../../client/cli", features = ["wasmtime"] } sp-core = { version = "2.0.0", path = "../../../primitives/core" } diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 3de31dc61ab51..37e9f16efd221 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -10,6 +10,7 @@ use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; use sc_finality_grandpa::{FinalityProofProvider as GrandpaFinalityProofProvider, SharedVoterState}; +use tiny_multihash::Multihash; // Our native executor instance. native_executor_instance!( @@ -91,7 +92,7 @@ pub fn new_full(config: Configuration) -> Result { GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); let (network, network_status_sinks, system_rpc_tx, network_starter) = - sc_service::build_network(sc_service::BuildNetworkParams { + sc_service::build_network::<_, _, _, _, Multihash>(sc_service::BuildNetworkParams { config: &config, client: client.clone(), transaction_pool: transaction_pool.clone(), @@ -263,7 +264,7 @@ pub fn new_light(config: Configuration) -> Result { GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); let (network, network_status_sinks, system_rpc_tx, network_starter) = - sc_service::build_network(sc_service::BuildNetworkParams { + sc_service::build_network::<_, _, _, _, Multihash>(sc_service::BuildNetworkParams { config: &config, client: client.clone(), transaction_pool: transaction_pool.clone(), diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 39df211707eaa..def4a3c891fb7 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -43,6 +43,7 @@ rand = "0.7.2" structopt = { version = "0.3.8", optional = true } tracing = "0.1.19" parking_lot = "0.10.0" +tiny-multihash = "0.4.5" # primitives sp-authority-discovery = { version = "2.0.0", path = "../../../primitives/authority-discovery" } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index b15ace6181a8f..c69550a0723b6 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -36,6 +36,7 @@ use futures::prelude::*; use sc_client_api::{ExecutorProvider, RemoteBackend}; use sp_core::traits::BareCryptoStorePtr; use node_executor::Executor; +use tiny_multihash::Multihash; type FullClient = sc_service::TFullClient; type FullBackend = sc_service::TFullBackend; @@ -161,7 +162,7 @@ pub struct NewFullBase { pub task_manager: TaskManager, pub inherent_data_providers: InherentDataProviders, pub client: Arc, - pub network: Arc::Hash>>, + pub network: Arc::Hash, Multihash>>, pub network_status_sinks: sc_service::NetworkStatusSinks, pub transaction_pool: Arc>, } @@ -355,7 +356,7 @@ pub fn new_full(config: Configuration) pub fn new_light_base(config: Configuration) -> Result<( TaskManager, RpcHandlers, Arc, - Arc::Hash>>, + Arc::Hash, Multihash>>, Arc>> ), ServiceError> { let (client, backend, keystore, mut task_manager, on_demand) = diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs index ff4d12dadd988..c8a997044609e 100644 --- a/client/authority-discovery/src/worker.rs +++ b/client/authority-discovery/src/worker.rs @@ -42,6 +42,7 @@ use sc_network::{ DhtEvent, ExHashT, Multiaddr, + MultihashDigest, NetworkStateInfo, PeerId, }; @@ -455,6 +456,7 @@ where "Failed to put hash '{:?}' on Dht.", hash ) }, + Some(_) => {} None => { debug!(target: LOG_TARGET, "Dht event stream terminated."); return Poll::Ready(()); @@ -707,10 +709,11 @@ pub trait NetworkProvider: NetworkStateInfo { fn get_value(&self, key: &libp2p::kad::record::Key); } -impl NetworkProvider for sc_network::NetworkService +impl NetworkProvider for sc_network::NetworkService where B: BlockT + 'static, H: ExHashT, + M: MultihashDigest, { fn set_priority_group( &self, diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 9509922cf2d3a..6ebb0e491ee51 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -38,7 +38,7 @@ use std::{pin::Pin, sync::Arc, task::{Context, Poll}}; use sp_core::traits::BareCryptoStorePtr; use finality_grandpa::Message::{Prevote, Precommit, PrimaryPropose}; use finality_grandpa::{voter, voter_set::VoterSet}; -use sc_network::{NetworkService, ReputationChange}; +use sc_network::{MultihashDigest, NetworkService, ReputationChange}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; use parity_scale_codec::{Encode, Decode}; use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, NumberFor}; @@ -152,9 +152,10 @@ pub trait Network: GossipNetwork + Clone + Send + 'static fn set_sync_fork_request(&self, peers: Vec, hash: Block::Hash, number: NumberFor); } -impl Network for Arc> where +impl Network for Arc> where B: BlockT, H: sc_network::ExHashT, + M: MultihashDigest, { fn set_sync_fork_request(&self, peers: Vec, hash: B::Hash, number: NumberFor) { NetworkService::set_sync_fork_request(self, peers, hash, number) diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 70c2942597aa5..40839f6ab5ee1 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -210,7 +210,8 @@ impl Future for GossipEngine { this.forwarding_state = ForwardingState::Busy(to_forward.into()); }, - Event::Dht(_) => {} + Event::Dht(_) => {}, + Event::Bitswap(_) => {}, } // The network event stream closed. Do the same for [`GossipValidator`]. Poll::Ready(None) => return Poll::Ready(()), diff --git a/client/network-gossip/src/lib.rs b/client/network-gossip/src/lib.rs index 1d566ed3cbba2..40285df700c25 100644 --- a/client/network-gossip/src/lib.rs +++ b/client/network-gossip/src/lib.rs @@ -59,7 +59,7 @@ pub use self::state_machine::TopicNotification; pub use self::validator::{DiscardAll, MessageIntent, Validator, ValidatorContext, ValidationResult}; use futures::prelude::*; -use sc_network::{Event, ExHashT, NetworkService, PeerId, ReputationChange}; +use sc_network::{Event, ExHashT, MultihashDigest, NetworkService, PeerId, ReputationChange}; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; use std::{borrow::Cow, pin::Pin, sync::Arc}; @@ -97,7 +97,7 @@ pub trait Network { fn announce(&self, block: B::Hash, associated_data: Vec); } -impl Network for Arc> { +impl Network for Arc> { fn event_stream(&self) -> Pin + Send>> { Box::pin(NetworkService::event_stream(self, "network-gossip")) } diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 26e1631d9f1aa..0e1dee1cf7d18 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -33,3 +33,4 @@ substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtim tempfile = "3.1.0" sp-tracing = { version = "2.0.0", path = "../../../primitives/tracing" } sc-service = { version = "0.8.0", default-features = false, features = ["test-helpers"], path = "../../service" } +tiny-multihash = "0.4.7" diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 587feebe55c14..bca9ced82f3e6 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -224,7 +224,7 @@ pub struct Peer { block_import: BlockImportAdapter<()>, select_chain: Option>, backend: Option>, - network: NetworkWorker::Hash>, + network: NetworkWorker::Hash, tiny_multihash::Multihash>, imported_blocks_stream: Pin> + Send>>, finality_notification_stream: Pin> + Send>>, } @@ -397,7 +397,7 @@ impl Peer { } /// Get a reference to the network service. - pub fn network_service(&self) -> &Arc::Hash>> { + pub fn network_service(&self) -> &Arc::Hash, tiny_multihash::Multihash>> { &self.network.service() } diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 885294449fb95..2d0921132d970 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -43,7 +43,7 @@ use threadpool::ThreadPool; use sp_api::{ApiExt, ProvideRuntimeApi}; use futures::future::Future; use log::{debug, warn}; -use sc_network::{ExHashT, NetworkService, NetworkStateInfo, PeerId}; +use sc_network::{ExHashT, MultihashDigest, NetworkService, NetworkStateInfo, PeerId}; use sp_core::{offchain::{self, OffchainStorage}, ExecutionContext, traits::SpawnNamed}; use sp_runtime::{generic::BlockId, traits::{self, Header}}; use futures::{prelude::*, future::ready}; @@ -58,15 +58,16 @@ pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX}; pub trait NetworkProvider: NetworkStateInfo { /// Set the authorized peers. fn set_authorized_peers(&self, peers: HashSet); - + /// Set the authorized only flag. fn set_authorized_only(&self, reserved_only: bool); } -impl NetworkProvider for NetworkService +impl NetworkProvider for NetworkService where B: traits::Block + 'static, H: ExHashT, + M: MultihashDigest, { fn set_authorized_peers(&self, peers: HashSet) { self.set_authorized_peers(peers) diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 410198af26da3..5da71cfb95cf3 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -38,7 +38,7 @@ use jsonrpc_pubsub::manager::SubscriptionManager; use sc_keystore::Store as Keystore; use log::{info, warn}; use sc_network::config::{Role, FinalityProofProvider, OnDemand, BoxFinalityProofRequestBuilder}; -use sc_network::NetworkService; +use sc_network::{MultihashDigest, NetworkService}; use parking_lot::RwLock; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ @@ -380,7 +380,7 @@ pub fn new_client( } /// Parameters to pass into `build`. -pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend> { +pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend, M: MultihashDigest> { /// The service configuration. pub config: Configuration, /// A shared client returned by `new_full_parts`/`new_light_parts`. @@ -401,7 +401,7 @@ pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend> { /// An optional, shared remote blockchain instance. Used for light clients. pub remote_blockchain: Option>>, /// A shared network instance. - pub network: Arc::Hash>>, + pub network: Arc::Hash, M>>, /// Sinks to propagate network status updates. pub network_status_sinks: NetworkStatusSinks, /// A Sender for RPC requests. @@ -411,18 +411,19 @@ pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend> { } /// Build a shared offchain workers instance. -pub fn build_offchain_workers( +pub fn build_offchain_workers( config: &Configuration, backend: Arc, spawn_handle: SpawnTaskHandle, client: Arc, - network: Arc::Hash>>, + network: Arc::Hash, M>>, ) -> Option>> where TBl: BlockT, TBackend: sc_client_api::Backend, >::OffchainStorage: 'static, TCl: Send + Sync + ProvideRuntimeApi + BlockchainEvents + 'static, >::Api: sc_offchain::OffchainWorkerApi, + M: MultihashDigest, { let offchain_workers = match backend.offchain_storage() { Some(db) => { @@ -452,8 +453,8 @@ pub fn build_offchain_workers( } /// Spawn the tasks that are required to run a node. -pub fn spawn_tasks( - params: SpawnTasksParams, +pub fn spawn_tasks( + params: SpawnTasksParams, ) -> Result where TCl: ProvideRuntimeApi + HeaderMetadata + Chain + @@ -472,7 +473,8 @@ pub fn spawn_tasks( TBackend: 'static + sc_client_api::backend::Backend + Send, TExPool: MaintainedTransactionPool::Hash> + MallocSizeOfWasm + 'static, - TRpc: sc_rpc::RpcExtension + TRpc: sc_rpc::RpcExtension, + M: MultihashDigest, { let SpawnTasksParams { mut config, @@ -593,13 +595,14 @@ pub fn spawn_tasks( Ok(rpc_handlers) } -async fn transaction_notifications( +async fn transaction_notifications( transaction_pool: Arc, - network: Arc::Hash>> + network: Arc::Hash, M>> ) where TBl: BlockT, TExPool: MaintainedTransactionPool::Hash>, + M: MultihashDigest, { // transaction notifications transaction_pool.import_notification_stream() @@ -615,11 +618,11 @@ async fn transaction_notifications( .await; } -fn build_telemetry( +fn build_telemetry( config: &mut Configuration, endpoints: sc_telemetry::TelemetryEndpoints, telemetry_connection_sinks: TelemetryConnectionSinks, - network: Arc::Hash>>, + network: Arc::Hash, M>>, spawn_handle: SpawnTaskHandle, genesis_hash: ::Hash, ) -> sc_telemetry::Telemetry { @@ -783,11 +786,11 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { } /// Build the network service, the network status sinks and an RPC sender. -pub fn build_network( +pub fn build_network( params: BuildNetworkParams ) -> Result< ( - Arc::Hash>>, + Arc::Hash, M>>, NetworkStatusSinks, TracingUnboundedSender>, NetworkStarter, @@ -801,6 +804,7 @@ pub fn build_network( HeaderBackend + BlockchainEvents + 'static, TExPool: MaintainedTransactionPool::Hash> + 'static, TImpQu: ImportQueue + 'static, + M: MultihashDigest, { let BuildNetworkParams { config, client, transaction_pool, spawn_handle, import_queue, on_demand, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 39f1dff289a1a..e949c81bb0a18 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -61,6 +61,7 @@ pub use self::builder::{ pub use config::{ BasePath, Configuration, DatabaseConfig, PruningMode, Role, RpcMethods, TaskExecutor, TaskType, }; +pub use sc_network::MultihashDigest; pub use sc_chain_spec::{ ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension, NoExtension, ChainType, @@ -201,10 +202,11 @@ pub struct PartialComponents, - H: sc_network::ExHashT + H: sc_network::ExHashT, + M: MultihashDigest, > ( role: Role, - mut network: sc_network::NetworkWorker, + mut network: sc_network::NetworkWorker, client: Arc, status_sinks: NetworkStatusSinks, mut rpc_rx: TracingUnboundedReceiver>, diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index fde79d19ede71..de9944f106bf9 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -42,3 +42,4 @@ sc-executor = { version = "0.8.0", path = "../../executor" } sp-panic-handler = { version = "2.0.0", path = "../../../primitives/panic-handler" } parity-scale-codec = "1.3.4" sp-tracing = { version = "2.0.0", path = "../../../primitives/tracing" } +tiny-multihash = "0.4.5" diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index cfe815f174fac..fa74b79fcc7af 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -48,6 +48,7 @@ use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_transaction_pool::TransactionPool; use sc_client_api::{Backend, CallExecutor}; use parking_lot::Mutex; +use tiny_multihash::Multihash; #[cfg(test)] mod client; @@ -74,14 +75,14 @@ pub trait TestNetNode: Clone + Future + Se fn client(&self) -> Arc>; fn transaction_pool(&self) -> Arc; - fn network(&self) -> Arc::Hash>>; + fn network(&self) -> Arc::Hash, Multihash>>; } pub struct TestNetComponents { task_manager: Arc>, client: Arc>, transaction_pool: Arc, - network: Arc::Hash>>, + network: Arc::Hash, Multihash>>, } impl @@ -89,7 +90,7 @@ TestNetComponents { pub fn new( task_manager: TaskManager, client: Arc>, - network: Arc::Hash>>, + network: Arc::Hash, Multihash>>, transaction_pool: Arc, ) -> Self { Self { @@ -144,7 +145,7 @@ TestNetComponents fn transaction_pool(&self) -> Arc { self.transaction_pool.clone() } - fn network(&self) -> Arc::Hash>> { + fn network(&self) -> Arc::Hash, tiny_multihash::Multihash>> { self.network.clone() } } From 25048f72f1bfcea1ec661bbf7c121e9ab104210c Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 11:38:26 +0200 Subject: [PATCH 4/8] Update Cargo.lock --- Cargo.lock | 394 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 305 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a661ed15d242..2f2c9963cf787 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" dependencies = [ - "generic-array 0.14.3", + "generic-array 0.14.4", ] [[package]] @@ -219,37 +219,110 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5" [[package]] -name = "async-channel" +name = "async-attributes" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee81ba99bee79f3c8ae114ae4baa7eaa326f63447cf2ec65e4393618b63f8770" +checksum = "efd3d156917d94862e779f356c5acae312b08fd3121e792c857d7928c8088423" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21279cfaa4f47df10b1816007e738ca3747ef2ee53ffc51cdbf57a8bb266fee3" dependencies = [ "concurrent-queue", "event-listener", "futures-core", ] +[[package]] +name = "async-executor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d373d78ded7d0b3fa8039375718cde0aace493f2e34fb60f51cbf567562ca801" +dependencies = [ + "async-task 4.0.0", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell 1.4.1", + "vec-arena", +] + +[[package]] +name = "async-global-executor" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd4f132a18f3fe7329c7b907047684f1b06174a900c559b661b2da8bb9cad5f" +dependencies = [ + "async-executor", + "async-io", + "futures-lite", + "num_cpus", + "once_cell 1.4.1", +] + +[[package]] +name = "async-io" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38628c78a34f111c5a6b98fc87dfc056cd1590b61afe748b145be4623c56d194" +dependencies = [ + "cfg-if", + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "once_cell 1.4.1", + "parking", + "polling", + "socket2", + "vec-arena", + "waker-fn", + "wepoll-sys-stjepang", + "winapi 0.3.9", +] + +[[package]] +name = "async-mutex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66941c2577c4fa351e4ce5fdde8f86c69b88d623f3b955be1bc7362a23434632" +dependencies = [ + "event-listener", +] + [[package]] name = "async-std" -version = "1.6.2" +version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00d68a33ebc8b57800847d00787307f84a562224a14db069b0acefe4c2abbf5d" +checksum = "3c92085acfce8b32e5b261d0b59b8f3309aee69fea421ea3f271f8b93225754f" dependencies = [ - "async-task", + "async-attributes", + "async-global-executor", + "async-io", + "async-mutex", + "async-task 3.0.0", + "blocking", "crossbeam-utils", "futures-channel", "futures-core", "futures-io", - "futures-timer 3.0.2", + "futures-lite", + "gloo-timers", "kv-log-macro", "log", "memchr", "num_cpus", - "once_cell 1.4.0", + "once_cell 1.4.1", "pin-project-lite", "pin-utils", "slab", - "smol", "wasm-bindgen-futures", ] @@ -259,6 +332,12 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17772156ef2829aadc587461c7753af20b7e8db1529bc66855add962a3b35d3" +[[package]] +name = "async-task" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c37ba09c1b5185eb9897a5cef32770031f58fa92d9a5f79eb50cae5030b39c1" + [[package]] name = "async-tls" version = "0.8.0" @@ -331,6 +410,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" + [[package]] name = "base58" version = "0.1.0" @@ -471,7 +556,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ - "block-padding", + "block-padding 0.1.5", "byte-tools", "byteorder 1.3.4", "generic-array 0.12.3", @@ -483,7 +568,8 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" dependencies = [ - "generic-array 0.14.3", + "block-padding 0.2.1", + "generic-array 0.14.4", ] [[package]] @@ -492,7 +578,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa136449e765dc7faa244561ccae839c394048667929af599b5d931ebe7b7f10" dependencies = [ - "generic-array 0.14.3", + "generic-array 0.14.4", ] [[package]] @@ -504,17 +590,23 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + [[package]] name = "blocking" -version = "0.4.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2468ff7bf85066b4a3678fede6fe66db31846d753ff0adfbfab2c6a6e81612b" +checksum = "2640778f8053e72c11f621b0a5175a0560a269282aa98ed85107773ab8e2a556" dependencies = [ "async-channel", "atomic-waker", + "fastrand", "futures-lite", - "once_cell 1.4.0", - "parking", + "once_cell 1.4.1", "waker-fn", ] @@ -744,9 +836,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "1.1.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83c06aff61f2d899eb87c379df3cbf7876f14471dcab474e0b6dc90ab96c080" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" dependencies = [ "cache-padded", ] @@ -1026,7 +1118,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" dependencies = [ - "generic-array 0.14.3", + "generic-array 0.14.4", "subtle 2.2.3", ] @@ -1100,6 +1192,28 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72aa14c04dfae8dd7d8a2b1cb7ca2152618cd01336dbfe704b8dcbf8d41dbd69" +[[package]] +name = "data-encoding-macro" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de6489dde5128f5ab2f71f88f8807a237cecf08d96dc7ca4be64e0730dc7d961" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", + "proc-macro-hack", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d2d6daefd5f1d4b74a891a5d2ab7dccba028d423107c074232a0c5dc0d40a9e" +dependencies = [ + "data-encoding", + "proc-macro-hack", + "syn", +] + [[package]] name = "derive_more" version = "0.99.9" @@ -1132,7 +1246,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" dependencies = [ - "generic-array 0.14.3", + "generic-array 0.14.4", ] [[package]] @@ -1332,9 +1446,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699d84875f1b72b4da017e6b0f77dfa88c0137f089958a88974d15938cbc2976" +checksum = "e1cd41440ae7e4734bbd42302f63eaba892afc93a3912dad84006247f0dedb0e" [[package]] name = "evm" @@ -1348,7 +1462,7 @@ dependencies = [ "primitive-types", "rlp", "serde", - "sha3", + "sha3 0.8.2", ] [[package]] @@ -1379,7 +1493,7 @@ checksum = "7410f5677a52203d3fca02b0eb8f96f9799f3a45cff82946a8ed28379e6b1b04" dependencies = [ "evm-core", "primitive-types", - "sha3", + "sha3 0.8.2", ] [[package]] @@ -1427,9 +1541,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.3.3" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a9cb09840f81cd211e435d00a4e487edd263dc3c8ff815c32dd76ad668ebed" +checksum = "5c85295147490b8fcf2ea3d104080a105a8b2c63f9c319e82c02d8e952388919" [[package]] name = "fdlimit" @@ -1585,7 +1699,7 @@ dependencies = [ "frame-system", "impl-trait-for-tuples", "log", - "once_cell 1.4.0", + "once_cell 1.4.1", "parity-scale-codec", "parity-util-mem", "paste", @@ -1840,9 +1954,9 @@ checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" [[package]] name = "futures-lite" -version = "0.1.8" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180d8fc9819eb48a0c976672fbeea13a73e10999e812bdc9e14644c25ad51d60" +checksum = "5b77e08e656f472d8ea84c472fa8b0a7a917883048e1cf2d4e34a323cd0aaf63" dependencies = [ "fastrand", "futures-core", @@ -1877,7 +1991,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" dependencies = [ - "once_cell 1.4.0", + "once_cell 1.4.1", ] [[package]] @@ -1958,9 +2072,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60fb4bb6bba52f78a471264d9a3b7d026cc0af47b22cd2cffbc0b787ca003e63" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" dependencies = [ "typenum", "version_check", @@ -2736,9 +2850,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.73" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] name = "libloading" @@ -2795,6 +2909,25 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "libp2p-bitswap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d2ad2d7d2a1818954192636f6e5a056299480132c6d3bd8f9b6ef863d60b5a" +dependencies = [ + "async-std", + "fnv", + "futures 0.3.5", + "libp2p", + "log", + "prost", + "prost-build", + "thiserror", + "tiny-cid", + "tiny-multihash", + "unsigned-varint 0.5.1", +] + [[package]] name = "libp2p-core" version = "0.22.1" @@ -3051,7 +3184,7 @@ dependencies = [ "pin-project", "rand 0.7.3", "salsa20", - "sha3", + "sha3 0.8.2", ] [[package]] @@ -3522,6 +3655,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +[[package]] +name = "multibase" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b78c60039650ff12e140ae867ef5299a58e19dded4d334c849dc7177083667e2" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + [[package]] name = "multihash" version = "0.11.2" @@ -3531,9 +3675,9 @@ dependencies = [ "blake2b_simd", "blake2s_simd", "digest 0.8.1", - "sha-1", + "sha-1 0.8.2", "sha2 0.8.2", - "sha3", + "sha3 0.8.2", "unsigned-varint 0.3.3", ] @@ -3733,6 +3877,7 @@ dependencies = [ "substrate-build-script-utils", "substrate-frame-cli", "tempfile", + "tiny-multihash", "tracing", "wasm-bindgen", "wasm-bindgen-futures", @@ -3944,6 +4089,7 @@ dependencies = [ "structopt", "substrate-build-script-utils", "substrate-frame-rpc-system", + "tiny-multihash", ] [[package]] @@ -4140,11 +4286,11 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" dependencies = [ - "parking_lot 0.10.2", + "parking_lot 0.11.0", ] [[package]] @@ -4473,7 +4619,7 @@ dependencies = [ "ripemd160", "rlp", "serde", - "sha3", + "sha3 0.8.2", "sp-core", "sp-io", "sp-runtime", @@ -5192,16 +5338,16 @@ dependencies = [ "mio", "mio-extras", "rand 0.7.3", - "sha-1", + "sha-1 0.8.2", "slab", "url 2.1.1", ] [[package]] name = "parking" -version = "1.0.5" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d4a6da31f8144a32532fe38fe8fb439a6842e0ec633f0037f0144c14e7f907" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" [[package]] name = "parking_lot" @@ -5422,6 +5568,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "polling" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0720e0b9ea9d52451cf29d3413ba8a9303f8815d9d9653ef70e03ff73e65566" +dependencies = [ + "cfg-if", + "libc", + "log", + "wepoll-sys-stjepang", + "winapi 0.3.9", +] + [[package]] name = "poly1305" version = "0.6.0" @@ -5509,9 +5668,9 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc175e9777c3116627248584e8f8b3e2987405cabe1c0adf7d1dd28f09dc7880" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2", @@ -5522,14 +5681,12 @@ dependencies = [ [[package]] name = "proc-macro-error-attr" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cc9795ca17eb581285ec44936da7fc2335a3f34f2ddd13118b6f4d515435c50" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "syn", - "syn-mid", "version_check", ] @@ -6030,7 +6187,7 @@ checksum = "952cd6b98c85bbc30efa1ba5783b8abf12fec8b3287ffa52605b9432313e34e4" dependencies = [ "cc", "libc", - "once_cell 1.4.0", + "once_cell 1.4.1", "spin", "untrusted", "web-sys", @@ -6900,6 +7057,7 @@ dependencies = [ "hex", "ip_network", "libp2p", + "libp2p-bitswap", "linked-hash-map", "linked_hash_set", "log", @@ -6934,6 +7092,8 @@ dependencies = [ "substrate-test-runtime-client", "tempfile", "thiserror", + "tiny-cid", + "tiny-multihash", "unsigned-varint 0.4.0", "void", "wasm-timer", @@ -6982,6 +7142,7 @@ dependencies = [ "substrate-test-runtime", "substrate-test-runtime-client", "tempfile", + "tiny-multihash", ] [[package]] @@ -7230,6 +7391,7 @@ dependencies = [ "substrate-test-runtime", "substrate-test-runtime-client", "tempfile", + "tiny-multihash", "tokio 0.1.22", ] @@ -7553,6 +7715,19 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "sha-1" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170a36ea86c864a3f16dd2687712dd6646f7019f301e57537c7f4dc9f5916770" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpuid-bool", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + [[package]] name = "sha2" version = "0.8.2" @@ -7591,6 +7766,18 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "sha3" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81199417d4e5de3f04b1e871023acea7389672c4135918f05aa9cbf2f2fa809" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "keccak", + "opaque-debug 0.3.0", +] + [[package]] name = "sharded-slab" version = "0.0.9" @@ -7687,27 +7874,6 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" -[[package]] -name = "smol" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "620cbb3c6e34da57d3a248cda0cd01cd5848164dc062e764e65d06fe3ea7aed5" -dependencies = [ - "async-task", - "blocking", - "concurrent-queue", - "fastrand", - "futures-io", - "futures-util", - "libc", - "once_cell 1.4.0", - "scoped-tls", - "slab", - "socket2", - "wepoll-sys-stjepang", - "winapi 0.3.9", -] - [[package]] name = "snow" version = "0.7.1" @@ -7751,7 +7917,7 @@ dependencies = [ "httparse", "log", "rand 0.7.3", - "sha-1", + "sha-1 0.8.2", ] [[package]] @@ -8526,7 +8692,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09f8ed9974042b8c3672ff3030a69fcc03b74c47c3d1ecb7755e8a3626011e88" dependencies = [ - "generic-array 0.14.3", + "generic-array 0.14.4", ] [[package]] @@ -8538,6 +8704,18 @@ dependencies = [ "bytes 0.4.12", ] +[[package]] +name = "strobe-rs" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9baee171ea889cfe5333700f0948777b483db8dc805c5c519ffe92e7714783" +dependencies = [ + "bitflags", + "byteorder 1.3.4", + "subtle 2.2.3", + "tiny-keccak", +] + [[package]] name = "strsim" version = "0.8.0" @@ -8879,26 +9057,15 @@ checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "syn" -version = "1.0.35" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7f4c519df8c117855e19dd8cc851e89eb746fe7a73f0157e0d95fdec5369b0" +checksum = "6690e3e9f692504b941dc6c3b188fd28df054f7fb8469ab40680df52fdcc842b" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] -[[package]] -name = "syn-mid" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "synstructure" version = "0.12.4" @@ -9024,7 +9191,7 @@ checksum = "b0165e045cc2ae1660270ca65e1676dbaab60feb0f91b10f7d0665e9b47e31f2" dependencies = [ "failure", "hmac", - "once_cell 1.4.0", + "once_cell 1.4.1", "pbkdf2", "rand 0.7.3", "rustc-hash", @@ -9032,6 +9199,17 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "tiny-cid" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8db0f10795d9047653976779741a9f5b763a87aadc7bf0c80be5a2361ecc7a9" +dependencies = [ + "multibase", + "tiny-multihash", + "unsigned-varint 0.5.1", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -9041,6 +9219,38 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tiny-multihash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632824c586db700f0fab27c2cca5b6bd7bf8e49b1f8ddce77f736c9fbe9401f2" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "digest 0.9.0", + "generic-array 0.14.4", + "sha-1 0.9.1", + "sha2 0.9.1", + "sha3 0.9.1", + "strobe-rs", + "tiny-multihash-derive", + "unsigned-varint 0.5.1", +] + +[[package]] +name = "tiny-multihash-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcc7989e2269ece9a93256298139f1f28b5d2cc483cd033b60c0c76f36d9377" +dependencies = [ + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "tinytemplate" version = "1.1.0" @@ -9595,7 +9805,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" dependencies = [ - "generic-array 0.14.3", + "generic-array 0.14.4", "subtle 2.2.3", ] @@ -9661,6 +9871,12 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" +[[package]] +name = "vec-arena" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d" + [[package]] name = "vec_map" version = "0.8.2" From 1db72aacf7f7a6e44d12dc5fc6a7bd028f675bcc Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 12:03:05 +0200 Subject: [PATCH 5/8] Make build_network a method on BuildNetworkParams. --- bin/node-template/node/src/service.rs | 8 +- bin/node/cli/src/service.rs | 8 +- client/service/src/builder.rs | 238 +++++++++++++------------- client/service/src/lib.rs | 2 +- 4 files changed, 129 insertions(+), 127 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 37e9f16efd221..409f3924788e8 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -92,7 +92,7 @@ pub fn new_full(config: Configuration) -> Result { GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); let (network, network_status_sinks, system_rpc_tx, network_starter) = - sc_service::build_network::<_, _, _, _, Multihash>(sc_service::BuildNetworkParams { + sc_service::BuildNetworkParams { config: &config, client: client.clone(), transaction_pool: transaction_pool.clone(), @@ -102,7 +102,7 @@ pub fn new_full(config: Configuration) -> Result { block_announce_validator_builder: None, finality_proof_request_builder: None, finality_proof_provider: Some(finality_proof_provider.clone()), - })?; + }.build_network::()?; if config.offchain_worker.enabled { sc_service::build_offchain_workers( @@ -264,7 +264,7 @@ pub fn new_light(config: Configuration) -> Result { GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); let (network, network_status_sinks, system_rpc_tx, network_starter) = - sc_service::build_network::<_, _, _, _, Multihash>(sc_service::BuildNetworkParams { + sc_service::BuildNetworkParams { config: &config, client: client.clone(), transaction_pool: transaction_pool.clone(), @@ -274,7 +274,7 @@ pub fn new_light(config: Configuration) -> Result { block_announce_validator_builder: None, finality_proof_request_builder: Some(finality_proof_request_builder), finality_proof_provider: Some(finality_proof_provider), - })?; + }.build_network::()?; if config.offchain_worker.enabled { sc_service::build_offchain_workers( diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index c69550a0723b6..368e90fa1c780 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -184,7 +184,7 @@ pub fn new_full_base( let (shared_voter_state, finality_proof_provider) = rpc_setup; let (network, network_status_sinks, system_rpc_tx, network_starter) = - sc_service::build_network(sc_service::BuildNetworkParams { + sc_service::BuildNetworkParams { config: &config, client: client.clone(), transaction_pool: transaction_pool.clone(), @@ -194,7 +194,7 @@ pub fn new_full_base( block_announce_validator_builder: None, finality_proof_request_builder: None, finality_proof_provider: Some(finality_proof_provider.clone()), - })?; + }.build_network::()?; if config.offchain_worker.enabled { sc_service::build_offchain_workers( @@ -406,7 +406,7 @@ pub fn new_light_base(config: Configuration) -> Result<( GrandpaFinalityProofProvider::new_for_service(backend.clone(), client.clone()); let (network, network_status_sinks, system_rpc_tx, network_starter) = - sc_service::build_network(sc_service::BuildNetworkParams { + sc_service::BuildNetworkParams { config: &config, client: client.clone(), transaction_pool: transaction_pool.clone(), @@ -416,7 +416,7 @@ pub fn new_light_base(config: Configuration) -> Result<( block_announce_validator_builder: None, finality_proof_request_builder: Some(finality_proof_request_builder), finality_proof_provider: Some(finality_proof_provider), - })?; + }.build_network::()?; network_starter.start_network(); if config.offchain_worker.enabled { diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 5da71cfb95cf3..a5112f8b970c1 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -785,132 +785,134 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { pub finality_proof_provider: Option>>, } -/// Build the network service, the network status sinks and an RPC sender. -pub fn build_network( - params: BuildNetworkParams -) -> Result< - ( - Arc::Hash, M>>, - NetworkStatusSinks, - TracingUnboundedSender>, - NetworkStarter, - ), - Error -> - where - TBl: BlockT, - TCl: ProvideRuntimeApi + HeaderMetadata + Chain + - BlockBackend + BlockIdTo + ProofProvider + - HeaderBackend + BlockchainEvents + 'static, - TExPool: MaintainedTransactionPool::Hash> + 'static, - TImpQu: ImportQueue + 'static, - M: MultihashDigest, +impl<'a, TBl, TExPool, TImpQu, TCl> BuildNetworkParams<'a, TBl, TExPool, TImpQu, TCl> +where + TBl: BlockT, + TCl: ProvideRuntimeApi + HeaderMetadata + Chain + + BlockBackend + BlockIdTo + ProofProvider + + HeaderBackend + BlockchainEvents + 'static, + TExPool: MaintainedTransactionPool::Hash> + 'static, + TImpQu: ImportQueue + 'static, { - let BuildNetworkParams { - config, client, transaction_pool, spawn_handle, import_queue, on_demand, - block_announce_validator_builder, finality_proof_request_builder, finality_proof_provider, - } = params; - - let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { - imports_external_transactions: !matches!(config.role, Role::Light), - pool: transaction_pool, - client: client.clone(), - }); - - let protocol_id = { - let protocol_id_full = match config.chain_spec.protocol_id() { - Some(pid) => pid, - None => { - warn!("Using default protocol ID {:?} because none is configured in the \ - chain specs", DEFAULT_PROTOCOL_ID - ); - DEFAULT_PROTOCOL_ID - } + /// Build the network service, the network status sinks and an RPC sender. + pub fn build_network( + self + ) -> Result< + ( + Arc::Hash, TMh>>, + NetworkStatusSinks, + TracingUnboundedSender>, + NetworkStarter, + ), + Error + > + { + let BuildNetworkParams { + config, client, transaction_pool, spawn_handle, import_queue, on_demand, + block_announce_validator_builder, finality_proof_request_builder, finality_proof_provider, + } = self; + + let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { + imports_external_transactions: !matches!(config.role, Role::Light), + pool: transaction_pool, + client: client.clone(), + }); + + let protocol_id = { + let protocol_id_full = match config.chain_spec.protocol_id() { + Some(pid) => pid, + None => { + warn!("Using default protocol ID {:?} because none is configured in the \ + chain specs", DEFAULT_PROTOCOL_ID + ); + DEFAULT_PROTOCOL_ID + } + }; + sc_network::config::ProtocolId::from(protocol_id_full) }; - sc_network::config::ProtocolId::from(protocol_id_full) - }; - let block_announce_validator = if let Some(f) = block_announce_validator_builder { - f(client.clone()) - } else { - Box::new(DefaultBlockAnnounceValidator) - }; - - let network_params = sc_network::config::Params { - role: config.role.clone(), - executor: { - let spawn_handle = Clone::clone(&spawn_handle); - Some(Box::new(move |fut| { - spawn_handle.spawn("libp2p-node", fut); - })) - }, - network_config: config.network.clone(), - chain: client.clone(), - finality_proof_provider, - finality_proof_request_builder, - on_demand: on_demand, - transaction_pool: transaction_pool_adapter as _, - import_queue: Box::new(import_queue), - protocol_id, - block_announce_validator, - metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()) - }; - - let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); - let network_mut = sc_network::NetworkWorker::new(network_params)?; - let network = network_mut.service().clone(); - let network_status_sinks = NetworkStatusSinks::new(); + let block_announce_validator = if let Some(f) = block_announce_validator_builder { + f(client.clone()) + } else { + Box::new(DefaultBlockAnnounceValidator) + }; - let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc"); + let network_params = sc_network::config::Params { + role: config.role.clone(), + executor: { + let spawn_handle = Clone::clone(&spawn_handle); + Some(Box::new(move |fut| { + spawn_handle.spawn("libp2p-node", fut); + })) + }, + network_config: config.network.clone(), + chain: client.clone(), + finality_proof_provider, + finality_proof_request_builder, + on_demand: on_demand, + transaction_pool: transaction_pool_adapter as _, + import_queue: Box::new(import_queue), + protocol_id, + block_announce_validator, + metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()) + }; - let future = build_network_future( - config.role.clone(), - network_mut, - client, - network_status_sinks.clone(), - system_rpc_rx, - has_bootnodes, - config.announce_block, - ); + let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); + let network_mut = sc_network::NetworkWorker::new(network_params)?; + let network = network_mut.service().clone(); + let network_status_sinks = NetworkStatusSinks::new(); + + let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc"); + + let future = build_network_future( + config.role.clone(), + network_mut, + client, + network_status_sinks.clone(), + system_rpc_rx, + has_bootnodes, + config.announce_block, + ); - // TODO: Normally, one is supposed to pass a list of notifications protocols supported by the - // node through the `NetworkConfiguration` struct. But because this function doesn't know in - // advance which components, such as GrandPa or Polkadot, will be plugged on top of the - // service, it is unfortunately not possible to do so without some deep refactoring. To bypass - // this problem, the `NetworkService` provides a `register_notifications_protocol` method that - // can be called even after the network has been initialized. However, we want to avoid the - // situation where `register_notifications_protocol` is called *after* the network actually - // connects to other peers. For this reason, we delay the process of the network future until - // the user calls `NetworkStarter::start_network`. - // - // This entire hack should eventually be removed in favour of passing the list of protocols - // through the configuration. - // - // See also https://github.com/paritytech/substrate/issues/6827 - let (network_start_tx, network_start_rx) = oneshot::channel(); - - // The network worker is responsible for gathering all network messages and processing - // them. This is quite a heavy task, and at the time of the writing of this comment it - // frequently happens that this future takes several seconds or in some situations - // even more than a minute until it has processed its entire queue. This is clearly an - // issue, and ideally we would like to fix the network future to take as little time as - // possible, but we also take the extra harm-prevention measure to execute the networking - // future using `spawn_blocking`. - spawn_handle.spawn_blocking("network-worker", async move { - if network_start_rx.await.is_err() { - debug_assert!(false); - log::warn!( - "The NetworkStart returned as part of `build_network` has been silently dropped" - ); - // This `return` might seem unnecessary, but we don't want to make it look like - // everything is working as normal even though the user is clearly misusing the API. - return; - } + // TODO: Normally, one is supposed to pass a list of notifications protocols supported by the + // node through the `NetworkConfiguration` struct. But because this function doesn't know in + // advance which components, such as GrandPa or Polkadot, will be plugged on top of the + // service, it is unfortunately not possible to do so without some deep refactoring. To bypass + // this problem, the `NetworkService` provides a `register_notifications_protocol` method that + // can be called even after the network has been initialized. However, we want to avoid the + // situation where `register_notifications_protocol` is called *after* the network actually + // connects to other peers. For this reason, we delay the process of the network future until + // the user calls `NetworkStarter::start_network`. + // + // This entire hack should eventually be removed in favour of passing the list of protocols + // through the configuration. + // + // See also https://github.com/paritytech/substrate/issues/6827 + let (network_start_tx, network_start_rx) = oneshot::channel(); + + // The network worker is responsible for gathering all network messages and processing + // them. This is quite a heavy task, and at the time of the writing of this comment it + // frequently happens that this future takes several seconds or in some situations + // even more than a minute until it has processed its entire queue. This is clearly an + // issue, and ideally we would like to fix the network future to take as little time as + // possible, but we also take the extra harm-prevention measure to execute the networking + // future using `spawn_blocking`. + spawn_handle.spawn_blocking("network-worker", async move { + if network_start_rx.await.is_err() { + debug_assert!(false); + log::warn!( + "The NetworkStart returned as part of `build_network` has been silently dropped" + ); + // This `return` might seem unnecessary, but we don't want to make it look like + // everything is working as normal even though the user is clearly misusing the API. + return; + } - future.await - }); + future.await + }); - Ok((network, network_status_sinks, system_rpc_tx, NetworkStarter(network_start_tx))) + Ok((network, network_status_sinks, system_rpc_tx, NetworkStarter(network_start_tx))) + } } /// Object used to start the network. diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index e949c81bb0a18..3f2a5273b953c 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -53,7 +53,7 @@ use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver, pub use self::error::Error; pub use self::builder::{ new_full_client, new_client, new_full_parts, new_light_parts, - spawn_tasks, build_network, BuildNetworkParams, NetworkStarter, build_offchain_workers, + spawn_tasks, BuildNetworkParams, NetworkStarter, build_offchain_workers, SpawnTasksParams, TFullClient, TLightClient, TFullBackend, TLightBackend, TLightBackendWithHash, TLightClientWithBackend, TFullCallExecutor, TLightCallExecutor, RpcExtensionBuilder, NoopRpcExtensionBuilder, From 93d02fb97af898fe87fc23a89b2d85d412624aa5 Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 12:17:40 +0200 Subject: [PATCH 6/8] Fix tests. --- client/network/src/gossip/tests.rs | 1 + client/network/src/service/tests.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs index 9ba44f564e132..d4489d3162e4f 100644 --- a/client/network/src/gossip/tests.rs +++ b/client/network/src/gossip/tests.rs @@ -26,6 +26,7 @@ use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _ type TestNetworkService = NetworkService< substrate_test_runtime_client::runtime::Block, substrate_test_runtime_client::runtime::Hash, + tiny_multihash::Multihash, >; /// Builds a full node to be used for testing. Returns the node service and its associated events diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 98ed7b2e2c6f2..1ad522553bc6f 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -27,6 +27,7 @@ use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _ type TestNetworkService = NetworkService< substrate_test_runtime_client::runtime::Block, substrate_test_runtime_client::runtime::Hash, + tiny_multihash::Multihash, >; /// Builds a full node to be used for testing. Returns the node service and its associated events From 6c04d780f1d3b746fca366bdd1e0075044695298 Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 12:48:50 +0200 Subject: [PATCH 7/8] Fix line width. --- client/network/src/behaviour.rs | 12 ++++++++---- client/network/src/discovery.rs | 3 ++- client/network/src/service.rs | 3 ++- client/network/test/src/lib.rs | 5 +++-- client/service/test/src/lib.rs | 10 ++++++---- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 1e335ade6091c..55749d0c5c0a6 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -312,7 +312,8 @@ impl Behaviour { self.discovery.put_value(key, value); } - /// Starts querying the providers of a key from the DHT. Will later produce a `Providers` or `GetProvidersFailed` event. + /// Starts querying the providers of a key from the DHT. Will later produce a `Providers` or + /// `GetProvidersFailed` event. pub fn providers(&mut self, key: &record::Key) { self.discovery.providers(key); } @@ -429,7 +430,8 @@ Behaviour { } } -impl NetworkBehaviourEventProcess for Behaviour { +impl NetworkBehaviourEventProcess + for Behaviour { fn inject_event(&mut self, event: request_responses::Event) { match event { request_responses::Event::InboundRequest { peer, protocol, result } => { @@ -450,7 +452,8 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess> for Behaviour { +impl NetworkBehaviourEventProcess> + for Behaviour { fn inject_event(&mut self, event: block_requests::Event) { match event { block_requests::Event::AnsweredRequest { peer, total_handling_time } => { @@ -484,7 +487,8 @@ impl NetworkBehaviourEventProcess NetworkBehaviourEventProcess> for Behaviour { +impl NetworkBehaviourEventProcess> + for Behaviour { fn inject_event(&mut self, event: finality_requests::Event) { match event { finality_requests::Event::Response { peer, block_hash, proof } => { diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index cad33549cad02..80a4a489c9d67 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -317,7 +317,8 @@ impl DiscoveryBehaviour { warn!( target: "sub-libp2p", "Libp2p => Failed to start providing: {:?}", e); - self.pending_events.push_back(DiscoveryOut::StartProvidingFailed(key.clone(), Duration::from_secs(0))); + let ev = DiscoveryOut::StartProvidingFailed(key.clone(), Duration::from_secs(0)); + self.pending_events.push_back(ev); } } } diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 1e7de31a21283..6626ece6dad4a 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -570,10 +570,11 @@ impl NetworkWorker .collect() }; + let external_addresses = Swarm::::external_addresses(&swarm).cloned().collect(); NetworkState { peer_id: Swarm::::local_peer_id(&swarm).to_base58(), listened_addresses: Swarm::::listeners(&swarm).cloned().collect(), - external_addresses: Swarm::::external_addresses(&swarm).cloned().collect(), + external_addresses, connected_peers, not_connected_peers, peerset: swarm.user_protocol_mut().peerset_debug_info(), diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index bca9ced82f3e6..490db1e801135 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -61,6 +61,7 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use sp_runtime::{ConsensusEngineId, Justification}; use substrate_test_runtime_client::{self, AccountKeyring}; use sc_service::client::Client; +use tiny_multihash::Multihash; pub use sc_network::config::EmptyTransactionPool; pub use substrate_test_runtime_client::runtime::{Block, Extrinsic, Hash, Transfer}; pub use substrate_test_runtime_client::{TestClient, TestClientBuilder, TestClientBuilderExt}; @@ -224,7 +225,7 @@ pub struct Peer { block_import: BlockImportAdapter<()>, select_chain: Option>, backend: Option>, - network: NetworkWorker::Hash, tiny_multihash::Multihash>, + network: NetworkWorker::Hash, Multihash>, imported_blocks_stream: Pin> + Send>>, finality_notification_stream: Pin> + Send>>, } @@ -397,7 +398,7 @@ impl Peer { } /// Get a reference to the network service. - pub fn network_service(&self) -> &Arc::Hash, tiny_multihash::Multihash>> { + pub fn network_service(&self) -> &Arc::Hash, Multihash>> { &self.network.service() } diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index fa74b79fcc7af..bcc5474cbde73 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -66,6 +66,8 @@ struct TestNet { nodes: usize, } +type TestNetworkService = sc_network::NetworkService::Hash, Multihash>; + pub trait TestNetNode: Clone + Future + Send + 'static { type Block: BlockT; type Backend: Backend; @@ -75,14 +77,14 @@ pub trait TestNetNode: Clone + Future + Se fn client(&self) -> Arc>; fn transaction_pool(&self) -> Arc; - fn network(&self) -> Arc::Hash, Multihash>>; + fn network(&self) -> Arc>; } pub struct TestNetComponents { task_manager: Arc>, client: Arc>, transaction_pool: Arc, - network: Arc::Hash, Multihash>>, + network: Arc>, } impl @@ -90,7 +92,7 @@ TestNetComponents { pub fn new( task_manager: TaskManager, client: Arc>, - network: Arc::Hash, Multihash>>, + network: Arc>, transaction_pool: Arc, ) -> Self { Self { @@ -145,7 +147,7 @@ TestNetComponents fn transaction_pool(&self) -> Arc { self.transaction_pool.clone() } - fn network(&self) -> Arc::Hash, tiny_multihash::Multihash>> { + fn network(&self) -> Arc> { self.network.clone() } } From de30ac0729452807730ea2fdc9761b7521cac038 Mon Sep 17 00:00:00 2001 From: David Craven Date: Mon, 21 Sep 2020 16:10:30 +0200 Subject: [PATCH 8/8] Fix tests. --- client/network/src/discovery.rs | 1 + client/network/src/gossip.rs | 2 +- client/service/test/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index 80a4a489c9d67..2bc8738cddc1d 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -947,6 +947,7 @@ mod tests { to_discover[swarm_n].remove(&other); }, DiscoveryOut::RandomKademliaStarted(_) => {}, + DiscoveryOut::BootstrapComplete(_) => {}, e => {panic!("Unexpected event: {:?}", e)}, } continue 'polling diff --git a/client/network/src/gossip.rs b/client/network/src/gossip.rs index eefd97954ca9f..ffba2108eaa0a 100644 --- a/client/network/src/gossip.rs +++ b/client/network/src/gossip.rs @@ -82,7 +82,7 @@ impl QueuedSender { protocol: ConsensusEngineId, queue_size_limit: usize, messages_encode: F - ) -> (Self, impl Future + Send + 'static) + ) -> (Self, impl Future + 'static) where M: Send + 'static, B: BlockT + 'static, diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index bcc5474cbde73..ff806ce2a3444 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -403,7 +403,7 @@ pub fn connectivity( const NUM_LIGHT_NODES: usize = 5; let expected_full_connections = NUM_FULL_NODES - 1 + NUM_LIGHT_NODES; - let expected_light_connections = NUM_FULL_NODES; + let expected_light_connections = NUM_FULL_NODES - 1 + NUM_LIGHT_NODES; { let temp = tempdir_with_prefix("substrate-connectivity-test");