From 5dd095ea9dd6bb35a59ccba67a2e132f6f16d842 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Sun, 8 Jan 2023 22:00:13 -0500 Subject: [PATCH 01/16] con't with intro section improvements and combine fundamentals --- content/concepts/fundamentals/dht.md | 7 ------- content/concepts/fundamentals/overview.md | 10 ---------- .../concepts/introduction/core-abstractions/_index.md | 5 +++++ .../core-abstractions}/addressing.md | 0 .../introduction/core-abstractions/connections.md | 7 +++++++ .../concepts/introduction/core-abstractions/keys.md | 7 +++++++ .../core-abstractions}/peers.md | 2 +- content/concepts/introduction/overview.md | 2 +- .../{fundamentals => introduction/protocols}/_index.md | 4 ++-- content/concepts/introduction/protocols/dht.md | 9 +++++++++ content/concepts/introduction/protocols/fetch.md | 7 +++++++ content/concepts/introduction/protocols/identify.md | 7 +++++++ content/concepts/introduction/protocols/multistream.md | 7 +++++++ .../protocols/overview.md} | 0 content/concepts/introduction/protocols/ping.md | 2 +- content/concepts/introduction/protocols/protobuf.md | 7 +++++++ 16 files changed, 61 insertions(+), 22 deletions(-) delete mode 100644 content/concepts/fundamentals/dht.md delete mode 100644 content/concepts/fundamentals/overview.md create mode 100644 content/concepts/introduction/core-abstractions/_index.md rename content/concepts/{fundamentals => introduction/core-abstractions}/addressing.md (100%) create mode 100644 content/concepts/introduction/core-abstractions/connections.md create mode 100644 content/concepts/introduction/core-abstractions/keys.md rename content/concepts/{fundamentals => introduction/core-abstractions}/peers.md (99%) rename content/concepts/{fundamentals => introduction/protocols}/_index.md (74%) create mode 100644 content/concepts/introduction/protocols/dht.md create mode 100644 content/concepts/introduction/protocols/fetch.md create mode 100644 content/concepts/introduction/protocols/identify.md create mode 100644 content/concepts/introduction/protocols/multistream.md rename content/concepts/{fundamentals/protocols.md => introduction/protocols/overview.md} (100%) create mode 100644 content/concepts/introduction/protocols/protobuf.md diff --git a/content/concepts/fundamentals/dht.md b/content/concepts/fundamentals/dht.md deleted file mode 100644 index e500da61..00000000 --- a/content/concepts/fundamentals/dht.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "The DHT" -description: Flexible networks need flexible addressing systems. Since libp2p is designed to work across a wide variety of networks, we need a way to work with a lot of different addressing schemes in a consistent way. -weight: 40 ---- - -Check out the following Launchpad guide on the [DHT](https://curriculum.pl-launchpad.io/curriculum/libp2p/dht/). diff --git a/content/concepts/fundamentals/overview.md b/content/concepts/fundamentals/overview.md deleted file mode 100644 index 0dd05b46..00000000 --- a/content/concepts/fundamentals/overview.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title : "Overview" -description: "Learn about the core pillars that compose each libp2p peer and a libp2p network." -weight: 20 -aliases: - - "/fundamentals" - - "/concepts/fundamentals" ---- - -libp2p includes a few default primitives that each stack is based on. diff --git a/content/concepts/introduction/core-abstractions/_index.md b/content/concepts/introduction/core-abstractions/_index.md new file mode 100644 index 00000000..91555da0 --- /dev/null +++ b/content/concepts/introduction/core-abstractions/_index.md @@ -0,0 +1,5 @@ +--- +title : "Core Abstractions" +description: "Learn about the core pillars that compose each libp2p peer and a libp2p network." +weight: 3 +--- diff --git a/content/concepts/fundamentals/addressing.md b/content/concepts/introduction/core-abstractions/addressing.md similarity index 100% rename from content/concepts/fundamentals/addressing.md rename to content/concepts/introduction/core-abstractions/addressing.md diff --git a/content/concepts/introduction/core-abstractions/connections.md b/content/concepts/introduction/core-abstractions/connections.md new file mode 100644 index 00000000..c12128e4 --- /dev/null +++ b/content/concepts/introduction/core-abstractions/connections.md @@ -0,0 +1,7 @@ +--- +title: "Connections" +description: "" +weight: 33 +--- + +Coming soon! diff --git a/content/concepts/introduction/core-abstractions/keys.md b/content/concepts/introduction/core-abstractions/keys.md new file mode 100644 index 00000000..5c8973a4 --- /dev/null +++ b/content/concepts/introduction/core-abstractions/keys.md @@ -0,0 +1,7 @@ +--- +title: "Keys" +description: "" +weight: 34 +--- + +Coming soon! diff --git a/content/concepts/fundamentals/peers.md b/content/concepts/introduction/core-abstractions/peers.md similarity index 99% rename from content/concepts/fundamentals/peers.md rename to content/concepts/introduction/core-abstractions/peers.md index 90278a88..6f3f5475 100644 --- a/content/concepts/fundamentals/peers.md +++ b/content/concepts/introduction/core-abstractions/peers.md @@ -1,7 +1,7 @@ --- title: "Peers" description: Peers are what make up a libp2p network. They include unique identifiers and storage methods to facilitate peer-to-peer connections in libp2p. -weight: 60 +weight: 32 aliases: - "/concepts/peers" - "/concepts/fundamentals/peers" diff --git a/content/concepts/introduction/overview.md b/content/concepts/introduction/overview.md index 3a8c1849..0d017d40 100644 --- a/content/concepts/introduction/overview.md +++ b/content/concepts/introduction/overview.md @@ -1,7 +1,7 @@ --- title: "What is libp2p" description: libp2p is a modular system of protocols, specifications, and libraries that enable the development of peer-to-peer network applications. -weight: 10 +weight: 2 aliases: - "/introduction/what-is-libp2p" --- diff --git a/content/concepts/fundamentals/_index.md b/content/concepts/introduction/protocols/_index.md similarity index 74% rename from content/concepts/fundamentals/_index.md rename to content/concepts/introduction/protocols/_index.md index d770ec6f..7516c6f4 100644 --- a/content/concepts/fundamentals/_index.md +++ b/content/concepts/introduction/protocols/_index.md @@ -1,5 +1,5 @@ --- -title : "Fundamentals" +title : "Protocol Basics" description: "Learn about the core pillars that compose each libp2p peer and a libp2p network." -weight: 2 +weight: 4 --- diff --git a/content/concepts/introduction/protocols/dht.md b/content/concepts/introduction/protocols/dht.md new file mode 100644 index 00000000..74ce9109 --- /dev/null +++ b/content/concepts/introduction/protocols/dht.md @@ -0,0 +1,9 @@ +--- +title: "Kademlia DHT" +description: "" +weight: 25 +--- + +Coming soon! + +For now, check out the following [Launchpad guide on the DHT](https://curriculum.pl-launchpad.io/curriculum/libp2p/dht/). diff --git a/content/concepts/introduction/protocols/fetch.md b/content/concepts/introduction/protocols/fetch.md new file mode 100644 index 00000000..5bf8f098 --- /dev/null +++ b/content/concepts/introduction/protocols/fetch.md @@ -0,0 +1,7 @@ +--- +title: "Fetch" +description: "" +weight: 24 +--- + +Coming soon! diff --git a/content/concepts/introduction/protocols/identify.md b/content/concepts/introduction/protocols/identify.md new file mode 100644 index 00000000..63026035 --- /dev/null +++ b/content/concepts/introduction/protocols/identify.md @@ -0,0 +1,7 @@ +--- +title: "Identify" +description: "A way for nodes in libp2p to query and inform other nodes about their metadata." +weight: 23 +--- + +Coming soon! diff --git a/content/concepts/introduction/protocols/multistream.md b/content/concepts/introduction/protocols/multistream.md new file mode 100644 index 00000000..77d6a62f --- /dev/null +++ b/content/concepts/introduction/protocols/multistream.md @@ -0,0 +1,7 @@ +--- +title: "Multistream Select" +description: "" +weight: 26 +--- + +Coming soon! diff --git a/content/concepts/fundamentals/protocols.md b/content/concepts/introduction/protocols/overview.md similarity index 100% rename from content/concepts/fundamentals/protocols.md rename to content/concepts/introduction/protocols/overview.md diff --git a/content/concepts/introduction/protocols/ping.md b/content/concepts/introduction/protocols/ping.md index e849f98e..628fd4c3 100644 --- a/content/concepts/introduction/protocols/ping.md +++ b/content/concepts/introduction/protocols/ping.md @@ -1,7 +1,7 @@ --- title: "Ping" description: "The ping protocol is a simple request response protocol." -weight: 20 +weight: 22 --- ## Ping in libp2p diff --git a/content/concepts/introduction/protocols/protobuf.md b/content/concepts/introduction/protocols/protobuf.md new file mode 100644 index 00000000..d9776f14 --- /dev/null +++ b/content/concepts/introduction/protocols/protobuf.md @@ -0,0 +1,7 @@ +--- +title: "Protobufs" +description: "libp2p protocols use protocol buffers (aka protobuf) to define message schemas" +weight: 21 +--- + +Coming soon! From 4d2c6c04730c6e86f98c789cd7e2a1a196f2050f Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Sun, 8 Jan 2023 22:06:58 -0500 Subject: [PATCH 02/16] edits --- content/concepts/introduction/protocols/overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/concepts/introduction/protocols/overview.md b/content/concepts/introduction/protocols/overview.md index 507cee74..78788226 100644 --- a/content/concepts/introduction/protocols/overview.md +++ b/content/concepts/introduction/protocols/overview.md @@ -1,7 +1,7 @@ --- -title: "Protocols" +title: "Overview" description: There are protocols everywhere you look when you're writing network applications, and libp2p is especially thick with them. -weight: 50 +weight: 20 aliases: - "/concepts/protocols" - "/concepts/fundamentals/protocols" From fdd06cd48995956aed1870fe0d4bc49612c34c4e Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Sun, 8 Jan 2023 22:13:19 -0500 Subject: [PATCH 03/16] lint --- .../introduction/protocols/overview.md | 44 ++++++++----------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/content/concepts/introduction/protocols/overview.md b/content/concepts/introduction/protocols/overview.md index 78788226..3a161e3c 100644 --- a/content/concepts/introduction/protocols/overview.md +++ b/content/concepts/introduction/protocols/overview.md @@ -35,13 +35,13 @@ By convention, protocol ids have a path-like structure, with a version number as ``` Breaking changes to your protocol's wire format or semantics should result in a new version -number. See the [protocol negotiation section](#protocol-neotiation) for more information about +number. See the [protocol negotiation section](#protocol-negotiation) for more information about how version selection works during the dialing and listening process. {{< alert icon="💡" context="info">}} While libp2p will technically accept any string as a valid protocol id, using the recommended path structure with a version component is both -developer-friendly and enables [easier matching by version](#match-using-semver). +developer-friendly and enables easier matching by version. {{< /alert >}} #### Handler functions @@ -50,8 +50,8 @@ To accept connections, a libp2p application will register handler functions for [switch][definition_switch] (aka "swarm"), or a higher level interface such as [go's Host interface](https://github.com/libp2p/go-libp2p-core/blob/master/host/host.go). The handler function will be invoked when an incoming stream is tagged with the registered protocol id. -If you register your handler with a [match function](#using-a-match-function), you can choose whether -to accept non-exact string matches for protocol ids, for example, to match on [semantic major versions](#match-using-semver). +If you register your handler with a match function, you can choose whether +to accept non-exact string matches for protocol ids, for example, to match on semantic major versions. #### Binary streams @@ -131,10 +131,6 @@ length is encoded as a [protobuf varint](https://developers.google.com/protocol- |--------------------|------|---------------|---------------|-------------------| | `/ipfs/ping/1.0.0` | N/A | [go][ping_go] | [js][ping_js] | [rust][ping_rust] | -[ping_go]: https://github.com/libp2p/go-libp2p/tree/master/p2p/protocol/ping -[ping_js]: https://github.com/libp2p/js-libp2p-ping -[ping_rust]: https://github.com/libp2p/rust-libp2p/blob/master/protocols/ping/src/lib.rs - The ping protocol is a simple liveness check that peers can use to quickly see if another peer is online. After the initial protocol negotiation, the dialing peer sends 32 bytes of random binary data. The listening @@ -147,12 +143,6 @@ the latency between request and response. |------------------|--------------------------------|-------------------|-------------------|-----------------------| | `/ipfs/id/1.0.0` | [identify spec][spec_identify] | [go][identify_go] | [js][identify_js] | [rust][identify_rust] | - -[spec_identify]: https://github.com/libp2p/specs/pull/97/files -[identify_go]: https://github.com/libp2p/go-libp2p/tree/master/p2p/protocol/identify -[identify_js]: https://github.com/libp2p/js-libp2p-identify -[identify_rust]: https://github.com/libp2p/rust-libp2p/tree/master/protocols/identify/src - The `identify` protocol allows peers to exchange information about each other, most notably their public keys and known network addresses. @@ -166,8 +156,6 @@ Importantly, the `Identify` message includes an `observedAddr` field that contai the peer observed the request coming in on. This helps peers determine their NAT status, since it allows them to see what other peers observe as their public address and compare it to their own view of the network. -[identify_proto]: https://github.com/libp2p/go-libp2p/blob/master/p2p/protocol/identify/pb/identify.proto - #### identify/push | **Protocol id** | spec & implementations | @@ -189,26 +177,30 @@ makes it more likely that other peers will discover the new address. libp2p uses the DHT as the foundation of its [peer routing](/concepts/peer-routing/) and [content routing](/concepts/content-routing/) functionality. To learn more about DHT and the Kademlia algorithm, check out the [Distributed Hash Tables guide][dht] on the IPFS documentation site. In addition, check out the [libp2p implementations page](https://libp2p.io/implementations/) for updates on all the kad-libp2p implementations. - - -[wiki_dht]: https://en.wikipedia.org/wiki/Distributed_hash_table -[wiki_kad]: https://en.wikipedia.org/wiki/Kademlia -[dht]: https://docs.ipfs.tech/concepts/dht/ - ### Circuit Relay | **Protocol id** | spec | | implementations | |-------------------------------|----------------------------------|----------------|-----------------| | `/libp2p/circuit/relay/0.1.0` | [circuit relay spec][spec_relay] | [go][relay_go] | [js][relay_js] | -[spec_relay]: https://github.com/libp2p/specs/tree/master/relay -[relay_js]: https://github.com/libp2p/js-libp2p-circuit -[relay_go]: https://github.com/libp2p/go-libp2p-circuit - As described in the [Circuit Relay article](/concepts/circuit-relay/), libp2p provides a protocol for tunneling traffic through relay peers when two peers are unable to connect to each other directly. See the article for more information on working with relays, including notes on relay addresses and how to enable automatic relay connection when behind an intractable NAT. +[ping_go]: https://github.com/libp2p/go-libp2p/tree/master/p2p/protocol/ping +[ping_js]: https://github.com/libp2p/js-libp2p-ping +[ping_rust]: https://github.com/libp2p/rust-libp2p/blob/master/protocols/ping/src/lib.rs +[spec_identify]: https://github.com/libp2p/specs/pull/97/files +[identify_go]: https://github.com/libp2p/go-libp2p/tree/master/p2p/protocol/identify +[identify_js]: https://github.com/libp2p/js-libp2p-identify +[identify_rust]: https://github.com/libp2p/rust-libp2p/tree/master/protocols/identify/src +[identify_proto]: https://github.com/libp2p/go-libp2p/blob/master/p2p/protocol/identify/pb/identify.proto +[spec_relay]: https://github.com/libp2p/specs/tree/master/relay +[relay_js]: https://github.com/libp2p/js-libp2p-circuit +[relay_go]: https://github.com/libp2p/go-libp2p-circuit [definition_switch]: /reference/glossary/#switch [definition_multiaddr]: /reference/glossary/#multiaddr +[wiki_dht]: https://en.wikipedia.org/wiki/Distributed_hash_table +[wiki_kad]: https://en.wikipedia.org/wiki/Kademlia +[dht]: https://docs.ipfs.tech/concepts/dht/ From 2b26cabb2d7bae3ff3a02e5cb50afe6d804c8872 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Sun, 8 Jan 2023 22:18:16 -0500 Subject: [PATCH 04/16] split what is libp2p --- .../introduction/core-abstractions/_index.md | 2 +- content/concepts/introduction/overview.md | 73 ------------------ content/concepts/introduction/why-libp2p.md | 76 +++++++++++++++++++ 3 files changed, 77 insertions(+), 74 deletions(-) create mode 100644 content/concepts/introduction/why-libp2p.md diff --git a/content/concepts/introduction/core-abstractions/_index.md b/content/concepts/introduction/core-abstractions/_index.md index 91555da0..0cc56ac8 100644 --- a/content/concepts/introduction/core-abstractions/_index.md +++ b/content/concepts/introduction/core-abstractions/_index.md @@ -1,5 +1,5 @@ --- title : "Core Abstractions" description: "Learn about the core pillars that compose each libp2p peer and a libp2p network." -weight: 3 +weight: 4 --- diff --git a/content/concepts/introduction/overview.md b/content/concepts/introduction/overview.md index 0d017d40..f98b77b9 100644 --- a/content/concepts/introduction/overview.md +++ b/content/concepts/introduction/overview.md @@ -75,76 +75,3 @@ runtimes and networking environments. libp2p was designed to address the limitations of traditional P2P networking approaches and these existing network models, with the goal of enabling the distributed web. {{< /alert >}} - -## Why libp2p? - -There are several reasons to consider using libp2p as a networking layer to create a robust P2P application: - -- **Modularity**: libp2p is designed to be modular, allowing developers to mix and match different components - to meet the needs of their particular application. This makes it easy to customize the networking stack - to fit the specific requirements of any P2P application. - -- **Extensive transport configurability**: libp2p provides a set of specifications that can be adapted to - support various [transport protocols](/concepts/transports/overview), allowing libp2p applications to operate - in various runtime and networking environments as the wealth of transport protocol choices makes it possible - to use libp2p in a variety of scenarios. - -- **Versatility**: In addition to supporting a wide range of transports, libp2p offers a range of discovery - mechanisms, data storage and retrieval patterns, and is also - [implemented in many programming languages](https://libp2p.io/implementations/), providing - developers with great flexibility when building P2P applications. - -- **Security**: libp2p includes [several security features](/concepts/security/security-considerations), - such as peer identity verification using public key cryptography and - [encrypted communication](/concepts/secure-comm/overview) between peers using modern cryptographic algorithms. - -- **Robustness**: libp2p is a robust and reliable networking protocol that is designed to withstand stress, - disturbance, and change. Its features and design choices ensure that it is able to function effectively - and efficiently in a wide range of environments, and it is able to recover quickly from disruptions or - failures. It also offers protection against network attacks through the use of - [mitigation techniques](/concepts/security/dos-mitigation). - -- **Resiliency**: P2P networks are often more resilient than traditional client-server networks, - as there is no single point of failure. libp2p includes features such as - [peer discovery](/concepts/discovery/overview) and [content routing](/concepts/routing/overview) that help - to ensure that the network remains available and accessible even if some peers are offline or unreachable. - -- **Efficiency**: P2P networks can be more efficient in resource utilization, as data is - distributed across multiple peers rather than stored on a central server. libp2p includes various storage - and retrieval patterns that allow developers to distribute data efficiently across the network, making it - possible to store and retrieve data in a cost-effective and scalable way. - -- **Piercing NAT Barriers**: libp2p is equipped with capabilities for [NAT traversal](/concepts/nat/overview), - which allows P2P communication between peers even when they are behind NAT devices or firewalls. This - helps to maintain the connectivity of the network and ensure that it remains accessible despite the - presence of these obstacles. - -- **Message Distribution and Dissemination**: One such pattern libp2p uses is - [publish/subscribe (pubsub)](/concepts/pubsub/overview), which allows a sender (publisher) to send a message - to multiple recipients (subscribers) without the publisher having to know who the subscribers are. - libp2p implements pubsub through the use of protocols like [gossipsub](/concepts/pubsub/gossipsub), providing - developers with a flexible and efficient means of exchanging data and messages within their P2P - applications. - -- **Interoperability**: libp2p implementations in different programming languages and libp2p releases across - versions are designed to be interoperable with one another. This enables applications /from different - language ecosystems to communicate seamlessly. This helps to promote a healthy, interconnected ecosystem - of P2P applications. - -- **Decentralization**: One of the main advantages of P2P networks is their decentralized nature, allowing - them to operate without a central authority. libp2p is designed to facilitate decentralized - communication between peers, making it possible to build P2P applications resistant to censorship and more - resilient in the face of network disruptions. - -## Related projects - -libp2p remains an integral component in IPFS and can be easily integrated with other projects in the -IPFS "family". Check their sites for specific information and references: - -- [IPFS](https://libp2p.io) is the InterPlanetary File System, which uses libp2p as - its networking layer. -- [Multiformats](https://multiformats.io) is a variety of *self-describing* data formats. -- [IPLD](https://ipld.io) is a set of tools for describing links between content-addressed - data, like IPFS files, Git commits, or Ethereum blocks. -- [The Permissive License Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack) - is a licensing strategy for software development that embraces open-source values. diff --git a/content/concepts/introduction/why-libp2p.md b/content/concepts/introduction/why-libp2p.md new file mode 100644 index 00000000..2cd4dac9 --- /dev/null +++ b/content/concepts/introduction/why-libp2p.md @@ -0,0 +1,76 @@ +--- +title: "Why libp2p" +description: libp2p is a modular system of protocols, specifications, and libraries that enable the development of peer-to-peer network applications. +weight: 3 +--- + +There are several reasons to consider using libp2p as a networking layer to create a robust P2P application: + +- **Modularlity**: libp2p is designed to be modular, allowing developers to mix and match different components + to meet the needs of their particular application. This makes it easy to customize the networking stack + to fit the specific requirements of any P2P application. + +- **Extensive transport configurability**: libp2p provides a set of specifications that can be adapted to + support various [transport protocols](../transports/overview.md), allowing libp2p applications to operate + in various runtime and networking environments as the wealth of transport protocol choices makes it possible + to use libp2p in a variety of scenarios. + +- **Versatility**: In addition to supporting a wide range of transports, libp2p offers a range of discovery + mechanisms, data storage and retrieval patterns, and is also + [implemented in many programming languages](https://libp2p.io/implementations/), providing + developers with great flexibility when building P2P applications. + +- **Security**: libp2p includes [several security features](../security/security-considerations.md), + such as peer identity verification using public key cryptography and + [encrypted communication](../secure-comm/overview.md) between peers using modern cryptographic algorithms. + +- **Robustness**: libp2p is a robust and reliable networking protocol that is designed to withstand stress, + disturbance, and change. Its features and design choices ensure that it is able to function effectively + and efficiently in a wide range of environments, and it is able to recover quickly from disruptions or + failures. It also offers protection against network attacks through the use of + [mitigation techniques](../security/dos-mitigation.md). + +- **Resiliency**: P2P networks are often more resilient than traditional client-server networks, + as there is no single point of failure. libp2p includes features such as + [peer discovery](../discovery/overview.md) and [content routing](../routing/overview.md) that help + to ensure that the network remains available and accessible even if some peers are offline or unreachable. + +- **Efficiency**: P2P networks can be more efficient in resource utilization, as data is + distributed across multiple peers rather than stored on a central server. libp2p includes various storage + and retrieval patterns that allow developers to distribute data efficiently across the network, making it + possible to store and retrieve data in a cost-effective and scalable way. + +- **Piercing NAT Barriers**: libp2p is equipped with capabilities for [NAT traversal](../nat/overview.md), + which allows P2P communication between peers even when they are behind NAT devices or firewalls. This + helps to maintain the connectivity of the network and ensure that it remains accessible despite the + presence of these obstacles. + +- **Message Distribution and Dissemination**: One such pattern libp2p uses is + [publish/subscribe (pubsub)](../pubsub/overview.md), which allows a sender (publisher) to send a message + to multiple recipients (subscribers) without the publisher having to know who the subscribers are. + libp2p implements pubsub through the use of protocols like [gossipsub](../pubsub/gossipsub.md), providing + developers with a flexible and efficient means of exchanging data and messages within their P2P + applications. + +- **Interoperability**: libp2p implementations in different programming languages and libp2p releases across + versions are designed to be interoperable with one another. This enables applications /from different + language ecosystems to communicate seamlessly. This helps to promote a healthy, interconnected ecosystem + of P2P applications. + +- **Decentralization**: One of the main advantages of P2P networks is their decentralized nature, allowing + them to operate without a central authority. libp2p is designed to facilitate decentralized + communication between peers, making it possible to build P2P applications resistant to censorship and more + resilient in the face of network disruptions. + +## Related projects + +libp2p remains an integral component in IPFS and can be easily integrated with other projects in the +IPFS "family". Check their sites for specific information and references: + +- [IPFS](https://libp2p.io) is the InterPlanetary File System, which uses libp2p as + its networking layer. +- [Multiformats](https://multiformats.io) is a variety of *self-describing* data formats. +- [IPLD](https://ipld.io) is a set of tools for describing links between content-addressed + data, like IPFS files, Git commits, or Ethereum blocks. +- [The Permissive License Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack) + is a licensing strategy for software development that embraces open-source values. From 2356d22720f15cc5879e80655c72e67b6a010436 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Wed, 11 Jan 2023 20:21:21 -0500 Subject: [PATCH 05/16] Apply suggestions from code review Co-authored-by: Marco Munizaga --- content/concepts/introduction/why-libp2p.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/concepts/introduction/why-libp2p.md b/content/concepts/introduction/why-libp2p.md index 2cd4dac9..ab5ce725 100644 --- a/content/concepts/introduction/why-libp2p.md +++ b/content/concepts/introduction/why-libp2p.md @@ -53,7 +53,7 @@ There are several reasons to consider using libp2p as a networking layer to crea applications. - **Interoperability**: libp2p implementations in different programming languages and libp2p releases across - versions are designed to be interoperable with one another. This enables applications /from different + versions are designed to be interoperable with one another. This enables applications from different language ecosystems to communicate seamlessly. This helps to promote a healthy, interconnected ecosystem of P2P applications. From 1b1dfd07298304b3850bc74c95884ae385a8f4c3 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Thu, 12 Jan 2023 04:41:23 -0500 Subject: [PATCH 06/16] incorporate PR feedback into why libp2p --- content/concepts/introduction/why-libp2p.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/content/concepts/introduction/why-libp2p.md b/content/concepts/introduction/why-libp2p.md index ab5ce725..cd3f61b3 100644 --- a/content/concepts/introduction/why-libp2p.md +++ b/content/concepts/introduction/why-libp2p.md @@ -1,6 +1,6 @@ --- title: "Why libp2p" -description: libp2p is a modular system of protocols, specifications, and libraries that enable the development of peer-to-peer network applications. +description: "The reasons that make libp2p a uniquely powerful networking stack." weight: 3 --- @@ -61,16 +61,3 @@ There are several reasons to consider using libp2p as a networking layer to crea them to operate without a central authority. libp2p is designed to facilitate decentralized communication between peers, making it possible to build P2P applications resistant to censorship and more resilient in the face of network disruptions. - -## Related projects - -libp2p remains an integral component in IPFS and can be easily integrated with other projects in the -IPFS "family". Check their sites for specific information and references: - -- [IPFS](https://libp2p.io) is the InterPlanetary File System, which uses libp2p as - its networking layer. -- [Multiformats](https://multiformats.io) is a variety of *self-describing* data formats. -- [IPLD](https://ipld.io) is a set of tools for describing links between content-addressed - data, like IPFS files, Git commits, or Ethereum blocks. -- [The Permissive License Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack) - is a licensing strategy for software development that embraces open-source values. From 08a77ebbf5b9fdf58f012a82f249a6b90400540e Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Mon, 2 Jan 2023 21:11:57 -0500 Subject: [PATCH 07/16] new connections doc --- .../core-abstractions/connections.md | 199 +++++++++++++++++- 1 file changed, 197 insertions(+), 2 deletions(-) diff --git a/content/concepts/introduction/core-abstractions/connections.md b/content/concepts/introduction/core-abstractions/connections.md index c12128e4..d526f67d 100644 --- a/content/concepts/introduction/core-abstractions/connections.md +++ b/content/concepts/introduction/core-abstractions/connections.md @@ -1,7 +1,202 @@ --- title: "Connections" -description: "" +description: "A libp2p connection allows two peers to read and write data to each other." weight: 33 --- -Coming soon! +## Background + +A libp2p connection allows two peers to read and write data to each other. +Peers connect over [transport protocols](../transports/overview.md), which are +core abstractions of libp2p and offer extensibility. As a modular networking stack, +libp2p is transport-agnostic and does not enforce the implementation of a specific +transport protocol. Though the support of modern transport protocols is a primary +focus for libp2p, implementers may support multiple types of transport. This document +does not cover establishing "transport level" connections, for example, opening "raw" +TCP sockets, as those semantics are specific to each transport. Rather, this document +explains the process that occurs after making the initial transport-level connection, +up to the point where "application level" streams are opened. Learn about transport +protocols in libp2p in the [transport section](../transports/overview/). + +## Life of a libp2p connection + +The following points present an overview for a standard libp2p connection: + +- A libp2p node (peer) initiates a connection with another node by sending + a request over the underlying transport protocol (e.g., TCP). This request + is referred to as "dialing." +- The receiving node, known as the "listener", accepts the incoming connection + request and responds with a handshake message to initiate the protocol negotiation + process. +- Both nodes exchange protocol information and select the protocols to use for the + connection, including protocols for security and stream multiplexing if necessary. + The multistream-select protocol is used for this negotiation process. +- If the underlying transport protocol does not natively support security and multiplexing, + a connection upgrade is performed to add these capabilities to the raw transport connection. + The peers may use NAT traversal techniques such as UPnP or hole punching to allow nodes + behind NAT devices to establish connections. +- Once the connection is established, and protocols are selected, the nodes can open multiple + streams over the connection for various interactions, each using its own protocol. +- Data is exchanged over the streams using the negotiated protocols. +- When a node wants to close the connection or a stream, it sends a close message to the other + node, which responds with an acknowledgment message. +- Once both nodes have sent and received the close message and acknowledgment, the connection + or stream is closed. + +### Connection and stream management + +libp2p handles the management of connections and streams in a few ways: + +- **Tracking**: libp2p keeps track of which connections and streams are open and updates + this information as connections and streams are opened or closed. +- **Timeouts and retries**: If a connection or stream fails, libp2p will automatically retry + to establish the connection or stream up to a certain number of times before giving up. +- **Opening and closing**: libp2p applications can specify when to open and close connections + and streams based on their specific requirements. For example, an application may open a new + stream for each file transfer rather than reusing a single stream. + +### NAT traversal + +NAT traversal allows nodes behind NAT devices (e.g., routers) to establish connections with +nodes on the public internet. Some transport protocols and connection upgrade protocols in +libp2p have built-in NAT traversal capabilities, while others require additional mechanisms +such as UPnP or hole punching. Learn more about NAT traversal in the +[NAT section](../nat/overview.md). + +### Multiplexing and performance + +Stream multiplexing allows multiple streams to share a single connection, which can be more +efficient than opening a separate connection for each stream. This can be especially helpful +in applications that simultaneously maintain many connections or streams, such as a peer-to-peer +file-sharing application. Multiplexing also allows for better utilization of network resources +and can improve overall performance. For example, if one stream is idle while another stream +is receiving a burst of data, the idle stream will not block the active stream from using the +connection. + +### Protocol support + +libp2p supports a variety of protocols for security and stream multiplexing, including: + +- Security protocols: + - noise: Provides encryption, authentication, and forward secrecy for libp2p connections. + - tls: Provides encryption, authentication, and forward secrecy for libp2p connections. + +Learn more about secure connections in the +[secure communications section](../secure-comm/overview.md). + +- Stream multiplexing protocols: + - yamux: Provides multiplexing of streams over a single connection. + - mplex: Provides multiplexing of streams over a single connection. + +Learn more about multiplexing in the +[stream multiplexing section](../multiplex/overview.md). + +New protocols can be added to libp2p as needed. When adding a new protocol, it is +important to consider compatibility with existing protocols, performance, and security. + +### Upgrading connections + +libp2p is designed to support a variety of transport protocols, including those that do +not natively support the core libp2p capabilities of security and stream multiplexing. +The process of layering capabilities onto "raw" transport connections is called "upgrading" +the connection. + +The listener's multiaddr determines the security protocol to be used on a connection +(see [addressing specification]). The multiplexing protocol is determined using protocol +negotiation, with the multistream-select protocol used for this negotiation process +(as described in the [Protocol Negotiation section](#protocol-negotiation)). When raw +connections need both security and multiplexing, security is always established first, +and the negotiation for stream multiplexing takes place over the encrypted channel. + +Here is an example of the connection upgrade process: + +- The dialing peer sends a request to initiate a connection to the listening + peer over the underlying transport protocol (e.g., TCP). +- The listening peer accepts the incoming connection request and sends the security + protocol ID (e.g., Noise) in its multiaddr to indicate the security protocol to use. +- The dialing peer responds with the security handshake message to initiate the Noise + protocol. +- If the security handshake is successful, the peers exchange the multistream protocol + ID to establish that they will use multistream-select to negotiate protocols for the + connection upgrade. +- The peers negotiate which stream multiplexer to use by proposing and responding with + protocol IDs. If a proposed multiplexer is unsupported, the listening peer responds + with "na". +- Once security and stream multiplexing are established, the connection upgrade process + is complete, and both peers can use the resulting libp2p connection to open new secure + multiplexed streams. + +> In the case where both peers initially act as initiators, e.g., during NAT hole punching, +> tie-breaking is done via the multistream-select simultaneous open protocol extension. + +### Protocol negotiation + +One of libp2p's core design goals is to be adaptable to many network environments, +including those that still need to be created. To provide this flexibility, the connection +upgrade process supports multiple protocols for connection security and stream multiplexing +and allows peers to select which to use for each connection. + +The process of selecting protocols is called protocol negotiation. In addition to its role in +the connection upgrade process, protocol negotiation is used whenever a new stream is opened +over an existing connection. This allows libp2p applications to route application-specific +protocols to the correct handler functions. + +Each protocol supported by a peer is identified using a unique string called a protocol ID. +While any string can be used, the conventional format is a path-like structure containing a +short name and a version number, separated by "/" characters. + +For example: `/yamux/1.0.0` identifies version `1.0.0` of yamux. multistream-select itself has +a protocol ID of `/multistream/1.0.0`. Including a version number in the protocol ID simplifies +the case where you want to concurrently support multiple protocol versions, perhaps a stable +version and an in-development version. By default, libp2p peers should support the latest protocol +version, but they may also choose to support older versions for compatibility. More information on +protocols is available on the [protocols document](protocols.md). + +Protocol negotiation works by one peer sending a message containing the protocol ID of the +protocol it wishes to use, and the other either echoing back the same id if it supports +the protocol or sending a different id if it does not. This process continues until both +peers agree on a common protocol to use or determine that no common protocol is available. + +For example, consider a case where two libp2p nodes are attempting to upgrade a +raw transport connection to a secure libp2p connection using the Noise protocol: + +- The dialing peer sends a message containing the protocol ID `/noise/1.0.0` to +- indicate its desire to use the Noise protocol. +- The listening peer supports the Noise protocol, +which echoes the same protocol ID. +- Both peers perform the Noise handshake to establish a secure connection. + +> If the listening peer did not support the Noise protocol, it would have responded +> with a different protocol ID or with "na" to indicate that no common protocol +> could be found. The dialing peer could then try a different protocol or terminate +> the connection attempt. + +### Opening new streams over a connection + +Once a libp2p connection is established, either through a new connection or +by upgrading a raw transport connection, new streams can be opened over the +connection as needed. The protocol negotiation process described above is used to +identify the protocol for each stream and route the stream data to the appropriate +handler functions. + +For example, consider a libp2p node that wants to send a file to another node over +a libp2p connection. The sending node might open a new stream and negotiate the +`/filetransfer/1.0.0` protocol with the receiving node to handle the file transfer. +The receiving node would then route the stream data to its file transfer handler +function. + +### Closing connections and streams + +Connections and streams can be closed by the dialing or listening node by sending +a close message. The close message indicates that the sender will not send more data +on the connection or stream, but it may still receive data until the other peer +closes the connection or stream. Once both peers have closed a connection or stream, +the connection or stream is considered fully closed, and all resources can be released. + +{{< alert icon="" context="">}} +It is important to properly close connections and streams to free up resources and avoid +resource leaks. However, it is also essential to implement timeouts and retries to handle +cases where a close message may not be received due to network issues. +{{< /alert >}} + +{{< alert icon="💡" context="note" text="See the connections technical specification for more details." />}} From e6e8b5f1ba74be6ab190045efea006b3f7d5457e Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Fri, 6 Jan 2023 06:42:41 -0500 Subject: [PATCH 08/16] edits --- .../core-abstractions/connections.md | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/content/concepts/introduction/core-abstractions/connections.md b/content/concepts/introduction/core-abstractions/connections.md index d526f67d..6ee6caa8 100644 --- a/content/concepts/introduction/core-abstractions/connections.md +++ b/content/concepts/introduction/core-abstractions/connections.md @@ -8,38 +8,40 @@ weight: 33 A libp2p connection allows two peers to read and write data to each other. Peers connect over [transport protocols](../transports/overview.md), which are -core abstractions of libp2p and offer extensibility. As a modular networking stack, -libp2p is transport-agnostic and does not enforce the implementation of a specific -transport protocol. Though the support of modern transport protocols is a primary -focus for libp2p, implementers may support multiple types of transport. This document -does not cover establishing "transport level" connections, for example, opening "raw" -TCP sockets, as those semantics are specific to each transport. Rather, this document -explains the process that occurs after making the initial transport-level connection, -up to the point where "application level" streams are opened. Learn about transport -protocols in libp2p in the [transport section](../transports/overview/). +core abstractions of libp2p and offer extensibility. +This document does not cover establishing "transport level" connections, for example, +opening "raw" TCP sockets, as those semantics are specific to each transport. Rather, +this document explains the process that occurs after making the initial transport-level +connection, up to the point where "application level" streams are opened. +Learn about transport protocols in libp2p in the [transport section](../transports/overview/). ## Life of a libp2p connection -The following points present an overview for a standard libp2p connection: +The following points present an overview for a standard libp2p connection; we'll then dive +more into the various components later in the document. - A libp2p node (peer) initiates a connection with another node by sending a request over the underlying transport protocol (e.g., TCP). This request - is referred to as "dialing." + is referred to as "dialing", and the peer is referred to as the "dialer." - The receiving node, known as the "listener", accepts the incoming connection request and responds with a handshake message to initiate the protocol negotiation process. - Both nodes exchange protocol information and select the protocols to use for the - connection, including protocols for security and stream multiplexing if necessary. - The multistream-select protocol is used for this negotiation process. + connection, including protocols for [security](../secure-comm/overview.md) and + [stream multiplexing](../multiplex/overview.md) if necessary. + The [multistream-select](https://github.com/multiformats/multistream-select) is used + for this negotiation process. - If the underlying transport protocol does not natively support security and multiplexing, a connection upgrade is performed to add these capabilities to the raw transport connection. - The peers may use NAT traversal techniques such as UPnP or hole punching to allow nodes - behind NAT devices to establish connections. + The peers may use NAT traversal techniques such as + [UPnP](https://en.wikipedia.org/wiki/Universal_Plug_and_Play) or + [hole punching](../nat/hole-punching.md) to allow nodes behind NAT devices to establish + connections. - Once the connection is established, and protocols are selected, the nodes can open multiple streams over the connection for various interactions, each using its own protocol. - Data is exchanged over the streams using the negotiated protocols. -- When a node wants to close the connection or a stream, it sends a close message to the other - node, which responds with an acknowledgment message. +- When a node wants to close the connection or a stream, it sends a "close" message to the other + node, which responds with an "acknowledgment" message. - Once both nodes have sent and received the close message and acknowledgment, the connection or stream is closed. @@ -98,8 +100,8 @@ important to consider compatibility with existing protocols, performance, and se libp2p is designed to support a variety of transport protocols, including those that do not natively support the core libp2p capabilities of security and stream multiplexing. -The process of layering capabilities onto "raw" transport connections is called "upgrading" -the connection. +The process of layering capabilities onto "raw" transport connections is known as +"upgrading" the connection. The listener's multiaddr determines the security protocol to be used on a connection (see [addressing specification]). The multiplexing protocol is determined using protocol @@ -153,17 +155,16 @@ version, but they may also choose to support older versions for compatibility. M protocols is available on the [protocols document](protocols.md). Protocol negotiation works by one peer sending a message containing the protocol ID of the -protocol it wishes to use, and the other either echoing back the same id if it supports -the protocol or sending a different id if it does not. This process continues until both +protocol it wishes to use, and the other either echoing back the same ID if it supports +the protocol or sending a different ID if it does not. This process continues until both peers agree on a common protocol to use or determine that no common protocol is available. For example, consider a case where two libp2p nodes are attempting to upgrade a raw transport connection to a secure libp2p connection using the Noise protocol: - The dialing peer sends a message containing the protocol ID `/noise/1.0.0` to -- indicate its desire to use the Noise protocol. -- The listening peer supports the Noise protocol, -which echoes the same protocol ID. + indicate its desire to use the Noise protocol. +- The listening peer supports the Noise protocol, which echoes the same protocol ID. - Both peers perform the Noise handshake to establish a secure connection. > If the listening peer did not support the Noise protocol, it would have responded From 90e9f11bdac4e9641d35e82401d078eca9004582 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Wed, 11 Jan 2023 10:14:21 -0500 Subject: [PATCH 09/16] incorporate PR feedback, add references to docs + edits --- .../core-abstractions/connections.md | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/content/concepts/introduction/core-abstractions/connections.md b/content/concepts/introduction/core-abstractions/connections.md index 6ee6caa8..934ef308 100644 --- a/content/concepts/introduction/core-abstractions/connections.md +++ b/content/concepts/introduction/core-abstractions/connections.md @@ -40,7 +40,8 @@ more into the various components later in the document. - Once the connection is established, and protocols are selected, the nodes can open multiple streams over the connection for various interactions, each using its own protocol. - Data is exchanged over the streams using the negotiated protocols. -- When a node wants to close the connection or a stream, it sends a "close" message to the other +- Based on the transport being used, when a node wants to close the connection or a stream, + it can send a "close" message to the other. node, which responds with an "acknowledgment" message. - Once both nodes have sent and received the close message and acknowledgment, the connection or stream is closed. @@ -51,18 +52,31 @@ libp2p handles the management of connections and streams in a few ways: - **Tracking**: libp2p keeps track of which connections and streams are open and updates this information as connections and streams are opened or closed. -- **Timeouts and retries**: If a connection or stream fails, libp2p will automatically retry - to establish the connection or stream up to a certain number of times before giving up. +- **Timeouts and retries**: When a connection or stream fails, the application would be notified + and it is expected to handle the reconnection process. libp2p only reconnects if you attempt to + open a new stream It is possible to handle the reconnection logic by writing a custom reconnection + manager. This manager can listen for closed stream events, and when a stream is closed, it can + attempt to reconnect to the peer. Additionally, you can add your own retry and timeout logic. - **Opening and closing**: libp2p applications can specify when to open and close connections and streams based on their specific requirements. For example, an application may open a new stream for each file transfer rather than reusing a single stream. + > Implementations may include a connection manager (like in go-libp2p), which is is responsible + > for maintaining the number of connections to other peers within a specific range, known as the + > low and high watermark. The low watermark is the minimum number of connections that the connection + > manager tries to maintain, while the high watermark is the maximum number of connections that the + > connection manager allows. + > There is also functionality for connection protection, which is used to protect certain connections + > from being closed by the connection manager. When a connection is protected, the connection manager + > will not close it, even if the number of connections exceeds the high watermark. This allows for + > important connections, such as those used for critical communication or for relaying traffic, to be + > preserved. ### NAT traversal NAT traversal allows nodes behind NAT devices (e.g., routers) to establish connections with -nodes on the public internet. Some transport protocols and connection upgrade protocols in -libp2p have built-in NAT traversal capabilities, while others require additional mechanisms -such as UPnP or hole punching. Learn more about NAT traversal in the +nodes on the public internet or other nodes behind NATs. Some transport protocols and connection +upgrade protocols in libp2p have built-in NAT traversal capabilities, while others require additional +mechanisms such as UPnP or hole punching. Learn more about NAT traversal in the [NAT section](../nat/overview.md). ### Multiplexing and performance @@ -80,15 +94,15 @@ connection. libp2p supports a variety of protocols for security and stream multiplexing, including: - Security protocols: - - noise: Provides encryption, authentication, and forward secrecy for libp2p connections. - - tls: Provides encryption, authentication, and forward secrecy for libp2p connections. + - [noise](../../secure-comm/noise.md): Provides encryption, authentication, and forward secrecy for libp2p connections. + - [tls](../../secure-comm/tls.md): Provides encryption, authentication, and forward secrecy for libp2p connections. Learn more about secure connections in the [secure communications section](../secure-comm/overview.md). - Stream multiplexing protocols: - - yamux: Provides multiplexing of streams over a single connection. - - mplex: Provides multiplexing of streams over a single connection. + - [yamux](../../multiplex/yamux.md): Provides multiplexing of streams over a single connection and backpressue. + - [mplex](../../multiplex/mplex.md): Provides multiplexing of streams over a single connection. Learn more about multiplexing in the [stream multiplexing section](../multiplex/overview.md). @@ -103,19 +117,21 @@ not natively support the core libp2p capabilities of security and stream multipl The process of layering capabilities onto "raw" transport connections is known as "upgrading" the connection. -The listener's multiaddr determines the security protocol to be used on a connection -(see [addressing specification]). The multiplexing protocol is determined using protocol -negotiation, with the multistream-select protocol used for this negotiation process -(as described in the [Protocol Negotiation section](#protocol-negotiation)). When raw -connections need both security and multiplexing, security is always established first, -and the negotiation for stream multiplexing takes place over the encrypted channel. +Eventually, the listener's multiaddr will determine the security protocol to be used on a +connection ([see the specification](https://github.com/libp2p/specs/pull/353)). The +multiplexing protocol is determined using protocol negotiation, with the multistream-select +protocol used for this negotiation process (as described in the +[Protocol Negotiation section](#protocol-negotiation)). When raw connections need both +security and multiplexing, security is always established first, and the negotiation for +stream multiplexing takes place over the encrypted channel. Here is an example of the connection upgrade process: - The dialing peer sends a request to initiate a connection to the listening peer over the underlying transport protocol (e.g., TCP). - The listening peer accepts the incoming connection request and sends the security - protocol ID (e.g., Noise) in its multiaddr to indicate the security protocol to use. + protocol ID (e.g., Noise) using multistream-select to indicate the security protocol + to use. - The dialing peer responds with the security handshake message to initiate the Noise protocol. - If the security handshake is successful, the peers exchange the multistream protocol @@ -123,7 +139,9 @@ Here is an example of the connection upgrade process: connection upgrade. - The peers negotiate which stream multiplexer to use by proposing and responding with protocol IDs. If a proposed multiplexer is unsupported, the listening peer responds - with "na". + with "na". In some cases, the peer may include the stream muxer in the security handshake to + save this roundtrip. This is known as + [early multiplexer negotiation](../../multiplex/early-negotiation). - Once security and stream multiplexing are established, the connection upgrade process is complete, and both peers can use the resulting libp2p connection to open new secure multiplexed streams. From 52e6e05d65e7e9add691a51f7ee943fcee4a3d71 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Wed, 11 Jan 2023 22:39:56 -0500 Subject: [PATCH 10/16] additional refereneces and edits --- .../introduction/core-abstractions/connections.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/content/concepts/introduction/core-abstractions/connections.md b/content/concepts/introduction/core-abstractions/connections.md index 934ef308..418269c1 100644 --- a/content/concepts/introduction/core-abstractions/connections.md +++ b/content/concepts/introduction/core-abstractions/connections.md @@ -33,7 +33,7 @@ more into the various components later in the document. for this negotiation process. - If the underlying transport protocol does not natively support security and multiplexing, a connection upgrade is performed to add these capabilities to the raw transport connection. - The peers may use NAT traversal techniques such as +- The peers may use NAT traversal techniques such as [UPnP](https://en.wikipedia.org/wiki/Universal_Plug_and_Play) or [hole punching](../nat/hole-punching.md) to allow nodes behind NAT devices to establish connections. @@ -147,7 +147,9 @@ Here is an example of the connection upgrade process: multiplexed streams. > In the case where both peers initially act as initiators, e.g., during NAT hole punching, -> tie-breaking is done via the multistream-select simultaneous open protocol extension. +> tie-breaking is done via the +> [multistream-select](https://github.com/libp2p/specs/tree/master/connections#multistream-select) +> simultaneous open protocol extension. ### Protocol negotiation @@ -186,9 +188,9 @@ raw transport connection to a secure libp2p connection using the Noise protocol: - Both peers perform the Noise handshake to establish a secure connection. > If the listening peer did not support the Noise protocol, it would have responded -> with a different protocol ID or with "na" to indicate that no common protocol -> could be found. The dialing peer could then try a different protocol or terminate -> the connection attempt. +> with "na" to indicate that no common protocol could be found. The dialing peer could +> then try a different protocol or terminate the connection attempt. More details are +> available [here](https://github.com/libp2p/specs/tree/master/connections#multistream-select). ### Opening new streams over a connection From 09cdcfc7f0bfe6be01fa229477af9261efb42e12 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Thu, 12 Jan 2023 04:38:19 -0500 Subject: [PATCH 11/16] edit stream manage --- .../introduction/core-abstractions/connections.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/concepts/introduction/core-abstractions/connections.md b/content/concepts/introduction/core-abstractions/connections.md index 418269c1..93c88c1c 100644 --- a/content/concepts/introduction/core-abstractions/connections.md +++ b/content/concepts/introduction/core-abstractions/connections.md @@ -53,10 +53,10 @@ libp2p handles the management of connections and streams in a few ways: - **Tracking**: libp2p keeps track of which connections and streams are open and updates this information as connections and streams are opened or closed. - **Timeouts and retries**: When a connection or stream fails, the application would be notified - and it is expected to handle the reconnection process. libp2p only reconnects if you attempt to - open a new stream It is possible to handle the reconnection logic by writing a custom reconnection - manager. This manager can listen for closed stream events, and when a stream is closed, it can - attempt to reconnect to the peer. Additionally, you can add your own retry and timeout logic. + and it is expected to handle the reconnection process. It is possible to handle the reconnection + logic by writing a custom reconnection manager. This manager can listen for closed stream events, + and when a stream is closed, it can attempt to reconnect to the peer. Additionally, you can add + your own retry and timeout logic. - **Opening and closing**: libp2p applications can specify when to open and close connections and streams based on their specific requirements. For example, an application may open a new stream for each file transfer rather than reusing a single stream. From 7f1b73417c1d14bfb314549e2986d884c0153080 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Sun, 15 Jan 2023 13:54:36 -0500 Subject: [PATCH 12/16] rmv keys and protobuf for now --- content/concepts/introduction/core-abstractions/keys.md | 7 ------- content/concepts/introduction/protocols/protobuf.md | 7 ------- 2 files changed, 14 deletions(-) delete mode 100644 content/concepts/introduction/core-abstractions/keys.md delete mode 100644 content/concepts/introduction/protocols/protobuf.md diff --git a/content/concepts/introduction/core-abstractions/keys.md b/content/concepts/introduction/core-abstractions/keys.md deleted file mode 100644 index 5c8973a4..00000000 --- a/content/concepts/introduction/core-abstractions/keys.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Keys" -description: "" -weight: 34 ---- - -Coming soon! diff --git a/content/concepts/introduction/protocols/protobuf.md b/content/concepts/introduction/protocols/protobuf.md deleted file mode 100644 index d9776f14..00000000 --- a/content/concepts/introduction/protocols/protobuf.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Protobufs" -description: "libp2p protocols use protocol buffers (aka protobuf) to define message schemas" -weight: 21 ---- - -Coming soon! From 4c340fb13d480987bab0d2331543a3c69c06eeca Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Fri, 10 Feb 2023 10:23:44 -0500 Subject: [PATCH 13/16] feat: kad-dht (#290) --- .../concepts/introduction/protocols/dht.md | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/content/concepts/introduction/protocols/dht.md b/content/concepts/introduction/protocols/dht.md index 74ce9109..0fd20cee 100644 --- a/content/concepts/introduction/protocols/dht.md +++ b/content/concepts/introduction/protocols/dht.md @@ -1,9 +1,74 @@ --- title: "Kademlia DHT" -description: "" +description: "The libp2p Kad-DHT subsystem is an implementation of the Kademlia +DHT, a distributed hash table." weight: 25 --- -Coming soon! +## Overview -For now, check out the following [Launchpad guide on the DHT](https://curriculum.pl-launchpad.io/curriculum/libp2p/dht/). +The Kademlia Distributed Hash Table (DHT), or Kad-DHT, is a distributed hash table +that is designed for P2P networks. + +Kad-DHT in libp2p is a subsystem based on the +[Kademlia whitepaper](https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf). + +Kad-DHT offers a way to find nodes and data on the network by using a +[routing table](https://en.wikipedia.org/wiki/Routing_table) that organizes peers based +on how similar their keys are. + +
+ A deeper look + + The routing table is organized based on a prefix length and a distance metric. + The prefix length helps to group similar keys, and the distance metric helps to + find the closest peers to a specific key in the routing table. The table maintains + a list of `k` closest peers for each possible prefix length between `0` and `L-1`, + where `L` is the length of the keyspace, determined by the length of the hash + function used. **Kad-DHT uses SHA-256**, with a keyspace of 256 bits, trying to maintain + `k` peers with a shared key prefix for every prefix length between `0` and `255` in + its routing table. + + The prefix length measures the proximity of two keys in the routing table and + divides the keyspace into smaller subspaces, called "buckets", each containing nodes + that share a common prefix of bits in their SHA-256 hash. The prefix length is the + number of bits that are the same in the two keys' SHA-256 hash. The more leading bits + that are the same, the longer the prefix length and the closer the proximity of the + two keys are considered to be. + + The distance metric is a way to calculate the distance between two keys by + taking the bitwise exclusive-or (XOR) of the SHA-256 hash of the two keys. The + result is a measure of the distance between the two keys, where a distance of + `0` means the keys are identical, and a distance of `1` means that only one + bit is different, meaning the two keys are close to each other (i.e. their + SHA-256 hashes are similar). + + This design allows for efficient and effective lookups in the routing table when + trying to find nodes or data that share similar prefixes. + +
+ +## Peer routing + +The Kad-DHT uses a process called "peer routing" to discover nodes in the +network. When looking for a peer, the local node contacts the `k` closest nodes to +the remote peer's ID asking them for closer nodes. The local node repeats the +process until it finds the peer or determines that it is not in the network. + +## Content provider routing + +Kad-DHT also includes a feature for content provider discovery, where nodes can +look up providers for a given key. The local node again contacts the `k` closest +nodes to the key asking them for either providers of the key and/or closer nodes +to the key. The local node repeats the process until it finds providers for the +key or determines that it is not in the network. + +## Bootstrap process + +To maintain a healthy routing table and discover new nodes, the Kad-DHT includes +a bootstrap process that runs periodically. The process starts by generating a random peer +ID and looking it up via the peer routing process. The node then adds the closest peers it +discovers to its routing table and repeats the process multiple times. This process also +includes looking up its own peer ID to improve awareness of nodes close to itself. + +{{< alert icon="💡" context="note" text="See the Kademlia DHT technical specification for more details." />}} From 94572fcdbb650f43ca1186098940603cbc6f5a90 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Mon, 20 Feb 2023 11:10:58 -0500 Subject: [PATCH 14/16] feat: identify (#286) --- .../introduction/protocols/identify.md | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/content/concepts/introduction/protocols/identify.md b/content/concepts/introduction/protocols/identify.md index 63026035..db00d771 100644 --- a/content/concepts/introduction/protocols/identify.md +++ b/content/concepts/introduction/protocols/identify.md @@ -1,7 +1,40 @@ --- title: "Identify" -description: "A way for nodes in libp2p to query and inform other nodes about their metadata." +description: "A way for nodes in libp2p to query and inform remote nodes about their metadata." weight: 23 --- -Coming soon! +The libp2p Identify protocol allows nodes in the libp2p network to share +and query information about each other. It is usually run right after a new libp2p +connection has been established and, at certain times, during the lifetime of the +connection. + +The information exchanged includes: + +- the protocol version: a network identifier, e.g. `/my-network/0.1.0`. +- the user agent: a free-form string identifying the local node's implementation, usually + in the format "agent-name/version". +- the public key: cryptographic identifier of the local node in the format of a peer ID. +- the observed address: the remote node's public IP address as observed by the local node. +- the listen addresses: the multiaddresses at which the local node can be reached. +- the list of [protocols](/concepts/introduction/protocols/overview) that the + local node supports. + +There are two variations of the protocol: `identify` and `identify/push`. + +The Identify protocol, for historical reasons identified by the protocol ID `/ipfs/id/1.0.0`, +is used to actively query a remote node for its metadata. The identify/push protocol, identified +by the protocol ID `/ipfs/id/push/1.0.0`, allows a node to push Identify data to the remote node +without querying. This is used whenever changes to the data sent in Identify happen, +e.g., when the node starts listening on a new address and adds or removes support +for a protocol. + +{{< alert icon="" context="note">}} +It's worth noting that go-libp2p used to include a feature called 'Identify Delta' +() for some time. This protocol was used to reduce the size of +Identify Pushes, but proved to be less useful (and a lot more complex) than expected. +It has since been deprecated, but many nodes on the IPFS network still support +this protocol. +{{< /alert >}} + +{{< alert icon="💡" context="note" text="See the Identify technical specification for more details." />}} From 7f69298f309097ec5054625af75bd03447905472 Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Thu, 23 Feb 2023 13:57:17 -0500 Subject: [PATCH 15/16] feat: host (#296) --- .../introduction/core-abstractions/_index.md | 2 +- content/concepts/introduction/host.md | 24 +++++++++++++++++++ .../concepts/introduction/protocols/_index.md | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 content/concepts/introduction/host.md diff --git a/content/concepts/introduction/core-abstractions/_index.md b/content/concepts/introduction/core-abstractions/_index.md index 0cc56ac8..0756195f 100644 --- a/content/concepts/introduction/core-abstractions/_index.md +++ b/content/concepts/introduction/core-abstractions/_index.md @@ -1,5 +1,5 @@ --- title : "Core Abstractions" description: "Learn about the core pillars that compose each libp2p peer and a libp2p network." -weight: 4 +weight: 5 --- diff --git a/content/concepts/introduction/host.md b/content/concepts/introduction/host.md new file mode 100644 index 00000000..fca0de26 --- /dev/null +++ b/content/concepts/introduction/host.md @@ -0,0 +1,24 @@ +--- +title: "libp2p Host" +description: "A libp2p host is a program or process that runs on a peer and allows it to participate in a libp2p network." +weight: 4 +--- + +A libp2p host is a program or process that allows a peer to participate +in the libp2p network. + +{{< alert icon="" context="note">}} +There are [several implementations](https://libp2p.io/implementations/) of the +libp2p host, and not all of the implementations support the features mentioned below. +{{< /alert >}} + +A libp2p host has a +[unique identity](/concepts/introduction/core-abstractions/peers#peer-id) +and can listen on different [transport protocols](/concepts/transports/overview), +[dial connections to other hosts](/concepts/transports/listen-and-dial), and detect +if it's a publicly reachable node or if it's behind a NAT/firewall with +[AutoNAT](../nat/overview.md##automatic-router-configuration). +The host can apply [hole punching techniques](/concepts/nat/hole-punching) to overcome +NATs, find peers through a routing protocol like +[DHT](/concepts/introduction/protocols/dht), and send messages across the network +using [Gossipsub](/concepts/pubsub/gossipsub). diff --git a/content/concepts/introduction/protocols/_index.md b/content/concepts/introduction/protocols/_index.md index 7516c6f4..2734b640 100644 --- a/content/concepts/introduction/protocols/_index.md +++ b/content/concepts/introduction/protocols/_index.md @@ -1,5 +1,5 @@ --- title : "Protocol Basics" description: "Learn about the core pillars that compose each libp2p peer and a libp2p network." -weight: 4 +weight: 6 --- From 1c35abedd4e93b29d014b8c7985d1a12d333edae Mon Sep 17 00:00:00 2001 From: Danny Salman Date: Fri, 24 Feb 2023 13:28:01 -0500 Subject: [PATCH 16/16] feat: multistream-select (#297) --- .../introduction/protocols/multistream.md | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/content/concepts/introduction/protocols/multistream.md b/content/concepts/introduction/protocols/multistream.md index 77d6a62f..cb2d8328 100644 --- a/content/concepts/introduction/protocols/multistream.md +++ b/content/concepts/introduction/protocols/multistream.md @@ -1,7 +1,23 @@ --- title: "Multistream Select" -description: "" +description: "Multistream Select is used to negotiate the protocol to be spoken on a connection or stream." weight: 26 --- -Coming soon! +## Overview + +[Multistream Select](https://github.com/multiformats/multistream-select) is a +protocol negotiation protocol. It allows two peers to negotiate the next spoken +protocol on a connection or stream. + +Peers use Multistream Select at various places to select the protocols to use on +a libp2p connection or stream. Most prominently Multistream Select is used to +select the [security](../../secure-comm/overview.md) followed by [stream +multiplexing](../../multiplex/overview.md) protocol spoken on a connection. In +addition it is used to select the application protocol (e.g. Kademlia) on each +new stream on a given connection. The [protocol +negotiation](../core-abstractions/connections#protocol-negotiation) and [upgrade +process](../core-abstractions/connections#upgrading-connections) are explained +further in the [connections document](../core-abstractions/connections.md). + +{{< alert icon="💡" context="note" text="See the multistream-select technical specification for more details." />}}