Merge branch 'master' into docs-release-v0.17.0

.circleci/main.yml

@@ -224,7 +224,7 @@ jobs:
     docker:
       - image: cimg/go:1.19.1-node
     parallelism: 4
-    resource_class: large
+    resource_class: 2xlarge+
     steps:
       - *make_out_dirs
       - attach_workspace:
@@ -324,6 +324,7 @@ jobs:
             - ~/.cache/go-build/
   ipfs-webui:
     executor: node-browsers
+    resource_class: 2xlarge+
     steps:
       - *make_out_dirs
       - attach_workspace:

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Kubo Changelogs
 
+- [v0.17](docs/changelogs/v0.17.md)
 - [v0.16](docs/changelogs/v0.16.md)
 - [v0.15](docs/changelogs/v0.15.md)
 - [v0.14](docs/changelogs/v0.14.md)

Dockerfile

@@ -1,4 +1,3 @@
-# Note: when updating the go minor version here, also update the go-channel in snap/snapcraft.yml
 FROM golang:1.19.1-buster
 LABEL maintainer="Steven Allen <[email protected]>"
 

README.md

@@ -7,7 +7,7 @@
 
 ## What is Kubo?
 
-Kubo was the first IPFS implementation and is the most widely used one today. Implementing the *Interplanetary Filesystem* - the Web3 standard and contender to replace https. Thus powered by IPLD's data models and the libp2p for network communication. Kubo is written in Go.
+Kubo was the first IPFS implementation and is the most widely used one today. Implementing the *Interplanetary Filesystem* - the Web3 standard for content-addressing, interoperable with HTTP. Thus powered by IPLD's data models and the libp2p for network communication. Kubo is written in Go.
 
 Featureset
 - Runs an IPFS-Node as a network service
@@ -216,22 +216,7 @@ $ guix package -i go-ipfs
 
 #### Snap
 
-> ⚠️ **SNAP USE IS DISCOURAGED**
-> 
-> If you want something more sophisticated to escape the Snap confinement, we recommend using a different method to install Kubo so that it is not subject to snap confinement.
-
-
-With snap, in any of the [supported Linux distributions](https://snapcraft.io/docs/core/install):
-
-```
-$ sudo snap install ipfs
-```
-
-The snap sets `IPFS_PATH` to `SNAP_USER_COMMON`, which is usually `~/snap/ipfs/common`. If you want to use `~/.ipfs` instead, you can bind-mount it to `~/snap/ipfs/common` like this:
-
-```
-$ sudo mount --bind ~/.ipfs ~/snap/ipfs/common
-```
+No longer supported, see rationale in [kubo#8688](https://github.com/ipfs/kubo/issues/8688).
 
 #### MacPorts
 
@@ -266,11 +251,7 @@ $ brew install --formula ipfs
 
 #### Chocolatey
 
-[![Chocolatey Version](https://img.shields.io/chocolatey/v/go-ipfs?color=00a4ef&label=go-ipfs&logo=windows&style=flat-square&cacheSeconds=3600)](https://chocolatey.org/packages/go-ipfs)
-
-```Powershell
-PS> choco install go-ipfs
-```
+No longer supported, see rationale in [kubo#9341](https://github.com/ipfs/kubo/issues/9341).
 
 #### Scoop
 
@@ -378,6 +359,10 @@ Basic proof of 'ipfs working' locally:
     # QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o
     ipfs cat <that hash>
 
+### HTTP/RPC clients
+
+For programmatic interaction with Kubo, see our [list of HTTP/RPC clients](docs/http-rpc-clients.md).
+
 ### Troubleshooting
 
 If you have previously installed IPFS before and you are running into problems getting a newer version to work, try deleting (or backing up somewhere else) your IPFS config directory (~/.ipfs by default) and rerunning `ipfs init`. This will reinitialize the config file to its defaults and clear out the local datastore of any bad entries.

config/init.go

@@ -79,14 +79,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
 			Interval: "12h",
 			Strategy: "all",
 		},
-		Swarm: SwarmConfig{
-			ConnMgr: ConnMgr{
-				LowWater:    DefaultConnMgrLowWater,
-				HighWater:   DefaultConnMgrHighWater,
-				GracePeriod: DefaultConnMgrGracePeriod.String(),
-				Type:        "basic",
-			},
-		},
 		Pinning: Pinning{
 			RemoteServices: map[string]RemotePinningService{},
 		},
@@ -114,6 +106,10 @@ const DefaultConnMgrLowWater = 600
 // grace period
 const DefaultConnMgrGracePeriod = time.Second * 20
 
+// DefaultConnMgrType is the default value for the connection managers
+// type.
+const DefaultConnMgrType = "basic"
+
 func addressesConfig() Addresses {
 	return Addresses{
 		Swarm: []string{

config/profile.go

@@ -178,9 +178,13 @@ fetching may be degraded.
 			c.AutoNAT.ServiceMode = AutoNATServiceDisabled
 			c.Reprovider.Interval = "0"
 
-			c.Swarm.ConnMgr.LowWater = 20
-			c.Swarm.ConnMgr.HighWater = 40
-			c.Swarm.ConnMgr.GracePeriod = time.Minute.String()
+			lowWater := int64(20)
+			highWater := int64(40)
+			gracePeriod := time.Minute
+			c.Swarm.ConnMgr.Type = NewOptionalString("basic")
+			c.Swarm.ConnMgr.LowWater = &OptionalInteger{value: &lowWater}
+			c.Swarm.ConnMgr.HighWater = &OptionalInteger{value: &highWater}
+			c.Swarm.ConnMgr.GracePeriod = &OptionalDuration{&gracePeriod}
 			return nil
 		},
 	},

config/pubsub.go

@@ -11,4 +11,8 @@ type PubsubConfig struct {
 
 	// Enable pubsub (--enable-pubsub-experiment)
 	Enabled Flag `json:",omitempty"`
+
+	// SeenMessagesTTL configures the duration after which a previously seen
+	// message ID can be forgotten about.
+	SeenMessagesTTL *OptionalDuration `json:",omitempty"`
 }

config/swarm.go

@@ -131,10 +131,10 @@ type Transports struct {
 
 // ConnMgr defines configuration options for the libp2p connection manager
 type ConnMgr struct {
-	Type        string
-	LowWater    int
-	HighWater   int
-	GracePeriod string
+	Type        *OptionalString   `json:",omitempty"`
+	LowWater    *OptionalInteger  `json:",omitempty"`
+	HighWater   *OptionalInteger  `json:",omitempty"`
+	GracePeriod *OptionalDuration `json:",omitempty"`
 }
 
 // ResourceMgr defines configuration options for the libp2p Network Resource Manager
@@ -144,8 +144,8 @@ type ResourceMgr struct {
 	Enabled Flag               `json:",omitempty"`
 	Limits  *rcmgr.LimitConfig `json:",omitempty"`
 
-	MaxMemory          OptionalString  `json:",omitempty"`
-	MaxFileDescriptors OptionalInteger `json:",omitempty"`
+	MaxMemory          *OptionalString  `json:",omitempty"`
+	MaxFileDescriptors *OptionalInteger `json:",omitempty"`
 
 	// A list of multiaddrs that can bypass normal system limits (but are still
 	// limited by the allowlist scope). Convenience config around

core/node/groups.go

@@ -38,33 +38,20 @@ var BaseLibP2P = fx.Options(
 )
 
 func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option {
-	// parse ConnMgr config
-
-	grace := config.DefaultConnMgrGracePeriod
-	low := config.DefaultConnMgrLowWater
-	high := config.DefaultConnMgrHighWater
-
-	connmgr := fx.Options()
-
-	if cfg.Swarm.ConnMgr.Type != "none" {
-		switch cfg.Swarm.ConnMgr.Type {
-		case "":
-			// 'default' value is the basic connection manager
-			break
-		case "basic":
-			var err error
-			grace, err = time.ParseDuration(cfg.Swarm.ConnMgr.GracePeriod)
-			if err != nil {
-				return fx.Error(fmt.Errorf("parsing Swarm.ConnMgr.GracePeriod: %s", err))
-			}
-
-			low = cfg.Swarm.ConnMgr.LowWater
-			high = cfg.Swarm.ConnMgr.HighWater
-		default:
-			return fx.Error(fmt.Errorf("unrecognized ConnMgr.Type: %q", cfg.Swarm.ConnMgr.Type))
-		}
-
+	var connmgr fx.Option
+
+	// set connmgr based on Swarm.ConnMgr.Type
+	connMgrType := cfg.Swarm.ConnMgr.Type.WithDefault(config.DefaultConnMgrType)
+	switch connMgrType {
+	case "none":
+		connmgr = fx.Options() // noop
+	case "", "basic":
+		grace := cfg.Swarm.ConnMgr.GracePeriod.WithDefault(config.DefaultConnMgrGracePeriod)
+		low := int(cfg.Swarm.ConnMgr.LowWater.WithDefault(config.DefaultConnMgrLowWater))
+		high := int(cfg.Swarm.ConnMgr.HighWater.WithDefault(config.DefaultConnMgrHighWater))
 		connmgr = fx.Provide(libp2p.ConnectionManager(low, high, grace))
+	default:
+		return fx.Error(fmt.Errorf("unrecognized Swarm.ConnMgr.Type: %q", connMgrType))
 	}
 
 	// parse PubSub config
@@ -77,6 +64,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option {
 		pubsubOptions = append(
 			pubsubOptions,
 			pubsub.WithMessageSigning(!cfg.Pubsub.DisableSigning),
+			pubsub.WithSeenMessagesTTL(cfg.Pubsub.SeenMessagesTTL.WithDefault(pubsub.TimeCacheDuration)),
 		)
 
 		switch cfg.Pubsub.Router {

core/node/libp2p/rcmgr.go

@@ -52,18 +52,22 @@ func ResourceManager(cfg config.SwarmConfig) interface{} {
 				return nil, opts, fmt.Errorf("opening IPFS_PATH: %w", err)
 			}
 
-			limits, err := createDefaultLimitConfig(cfg)
+			limitConfig, err := createDefaultLimitConfig(cfg)
 			if err != nil {
 				return nil, opts, err
 			}
 
+			// The logic for defaults and overriding with specified SwarmConfig.ResourceMgr.Limits
+			// is documented in docs/config.md.
+			// Any changes here should be reflected there.
 			if cfg.ResourceMgr.Limits != nil {
 				l := *cfg.ResourceMgr.Limits
-				l.Apply(limits)
-				limits = l
+				// This effectively overrides the computed default LimitConfig with any vlues from cfg.ResourceMgr.Limits
+				l.Apply(limitConfig)
+				limitConfig = l
 			}
 
-			limiter := rcmgr.NewFixedLimiter(limits)
+			limiter := rcmgr.NewFixedLimiter(limitConfig)
 
 			str, err := rcmgrObs.NewStatsTraceReporter()
 			if err != nil {

core/node/libp2p/rcmgr_defaults.go

@@ -44,51 +44,8 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{
 // This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled
 
 // createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager.
-// libp2p's resource manager provides tremendous flexibility but also adds a lot of complexity.
-// The intent of the default config here is to provide good defaults,
-// and where the defaults aren't good enough,
-// to expose a good set of higher-level "knobs" to users to satisfy most use cases
-// without requiring users to wade into all the intricacies of libp2p's resource manager.
-//
-// The inputs one can specify in SwarmConfig are:
-//   - cfg.ResourceMgr.MaxMemory:  This is the max amount of memory in bytes to allow libp2p to use.
-//     libp2p's resource manager will prevent additional resource creation while this limit is hit.
-//     If this value isn't specified, 1/8th of the total system memory is used.
-//   - cfg.ResourceMgr.MaxFileDescriptors:  This is the maximum number of file descriptors to allow libp2p to use.
-//     libp2p's resource manager will prevent additional file descriptor consumption while this limit is hit.
-//     If this value isn't specified, the maximum between 1/2 of system FD limit and 4096 is used.
-//   - Swarm.ConnMgr.HighWater: If a connection manager is specified, libp2p's resource manager
-//     will allow 2x more connections than the HighWater mark
-//     so the connection manager has "space and time" to close "least useful" connections.
-//
-// With these inputs defined, limits are created at the system, transient, and peer scopes.
-// Other scopes are ignored (by being set to infinity).
-// The reason these scopes are chosen is because:
-//   - system - This gives us the coarse-grained control we want so we can reason about the system as a whole.
-//     It is the backstop, and allows us to reason about resource consumption more easily
-//     since don't have think about the interaction of many other scopes.
-//   - transient - Limiting connections that are in process of being established provides backpressure so not too much work queues up.
-//   - peer - The peer scope doesn't protect us against intentional DoS attacks.
-//     It's just as easy for an attacker to send 100 requests/second with 1 peerId vs. 10 requests/second with 10 peers.
-//     We are reliant on the system scope for protection here in the malicious case.
-//     The reason for having a peer scope is to protect against unintentional DoS attacks
-//     (e.g., bug in a peer which is causing it to "misbehave").
-//     In the unintional case, we want to make sure a "misbehaving" node doesn't consume more resources than necessary.
-//
-// Within these scopes, limits are just set on memory, FD, and inbound connections/streams.
-// Limits are set based on the inputs above.
-// We trust this node to behave properly and thus ignore outbound connection/stream limits.
-// We apply any limits that libp2p has for its protocols/services
-// since we assume libp2p knows best here.
-//
-// This leaves 3 levels of resource management protection:
-//  1. The user who does nothing and uses defaults - In this case they get some sane defaults
-//     based on the amount of memory and file descriptors their system has.
-//     This should protect the node from many attacks.
-//  2. Slightly more advanced user - They can tweak the above by passing in config on
-//     maxMemory, maxFD, or maxConns with Swarm.HighWater.ConnMgr.
-//  3. Power user - They specify all the limits they want set via Swarm.ResourceMgr.Limits
-//     and we don't do any defaults/overrides. We pass that config blindly into libp2p resource manager.
+// The defaults follow the documentation in docs/config.md.
+// Any changes in the logic here should be reflected there.
 func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) {
 	maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 8)
 	maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString)
@@ -105,7 +62,6 @@ func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error)
 			FD:     int(numFD),
 
 			// By default, we just limit connections on the inbound side.
-			// Note that the limit gets adjusted below if "cfg.ConnMgr.HighWater" is set.
 			Conns:         bigEnough,
 			ConnsInbound:  rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default
 			ConnsOutbound: bigEnough,
@@ -132,9 +88,31 @@ func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error)
 			StreamsOutbound: 0,
 		},
 
-		// Just go with what libp2p does
-		TransientBaseLimit:     rcmgr.DefaultLimits.TransientBaseLimit,
-		TransientLimitIncrease: rcmgr.DefaultLimits.TransientLimitIncrease,
+		TransientBaseLimit: rcmgr.BaseLimit{
+			Memory: rcmgr.DefaultLimits.TransientBaseLimit.Memory,
+			FD:     rcmgr.DefaultLimits.TransientBaseLimit.FD,
+
+			Conns:         bigEnough,
+			ConnsInbound:  rcmgr.DefaultLimits.TransientBaseLimit.ConnsInbound,
+			ConnsOutbound: bigEnough,
+
+			Streams:         bigEnough,
+			StreamsInbound:  rcmgr.DefaultLimits.TransientBaseLimit.StreamsInbound,
+			StreamsOutbound: bigEnough,
+		},
+
+		TransientLimitIncrease: rcmgr.BaseLimitIncrease{
+			Memory:     rcmgr.DefaultLimits.TransientLimitIncrease.Memory,
+			FDFraction: rcmgr.DefaultLimits.TransientLimitIncrease.FDFraction,
+
+			Conns:         0,
+			ConnsInbound:  rcmgr.DefaultLimits.TransientLimitIncrease.ConnsInbound,
+			ConnsOutbound: 0,
+
+			Streams:         0,
+			StreamsInbound:  rcmgr.DefaultLimits.TransientLimitIncrease.StreamsInbound,
+			StreamsOutbound: 0,
+		},
 
 		// Lets get out of the way of the allow list functionality.
 		// If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through.
@@ -197,12 +175,5 @@ func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error)
 
 	defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), int(numFD))
 
-	// If a high water mark is set:
-	if cfg.ConnMgr.Type == "basic" {
-		// set the connection limit higher than high water mark so that the ConnMgr has "space and time" to close "least useful" connections.
-		defaultLimitConfig.System.Conns = 2 * cfg.ConnMgr.HighWater
-		log.Info("adjusted default resource manager System.Conns limits to match ConnMgr.HighWater value of %s", cfg.ConnMgr.HighWater)
-	}
-
 	return defaultLimitConfig, nil
 }

core/node/libp2p/rcmgr_logging.go

@@ -50,7 +50,7 @@ func (n *loggingResourceManager) start(ctx context.Context) {
 				n.limitExceededErrs = make(map[string]int)
 
 				for e, count := range errs {
-					n.logger.Errorf("Resource limits were exceeded %d times with error %q.", count, e)
+					n.logger.Errorf("Protected from exceeding resource limits %d times: %q.", count, e)
 				}
 
 				if len(errs) != 0 {

core/node/libp2p/rcmgr_logging_test.go

@@ -55,7 +55,7 @@ func TestLoggingResourceManager(t *testing.T) {
 			if oLogs.Len() == 0 {
 				continue
 			}
-			require.Equal(t, "Resource limits were exceeded 2 times with error \"system: cannot reserve inbound connection: resource limit exceeded\".", oLogs.All()[0].Message)
+			require.Equal(t, "Protected from exceeding resource limits 2 times: \"system: cannot reserve inbound connection: resource limit exceeded\".", oLogs.All()[0].Message)
 			return
 		}
 	}

docs/EARLY_TESTERS.md

@@ -31,7 +31,7 @@ We will ask early testers to participate at two points in the process:
 - [ ] Siderus (@koalalorenzo)
 - [ ] Charity Engine (@rytiss, @tristanolive)
 - [ ] Fission (@bmann)
-- [ ] OrbitDB (@tabcat)
+- [ ] OrbitDB (@aphelionz)
 
 ## How to sign up?
 

docs/PATCH_RELEASE_TEMPLATE.md

@@ -24,8 +24,6 @@ This process handles patch releases from version `vX.Y.Z` to `vX.Y.Z+1` assuming
 - [ ] Release published
   - [ ] to [dist.ipfs.tech](https://dist.ipfs.tech)
   - [ ] to [npm-go-ipfs](https://www.npmjs.com/package/go-ipfs) (should be done by [ipfs/npm-go-ipfs](https://github.com/ipfs/npm-go-ipfs), but ok to dispatch [this job](https://github.com/ipfs/npm-go-ipfs/actions/workflows/main.yml) manually)
-  - [ ] to [chocolatey](https://chocolatey.org/packages/go-ipfs) (should be done by [ipfs/choco-go-ipfs](https://github.com/ipfs/choco-go-ipfs/), but ok to dispatch [this job](https://github.com/ipfs/choco-go-ipfs/actions/workflows/main.yml) manually)
-  - [ ] to [snap](https://snapcraft.io/ipfs) (should happen automatically, see [snap/snapcraft.yaml](https://github.com/ipfs/kubo/blob/master/snap/snapcraft.yaml))
   - [ ] to [github](https://github.com/ipfs/kubo/releases)
   - [ ] to [arch](https://www.archlinux.org/packages/community/x86_64/go-ipfs/) (flag it out of date)
 - [ ] Cut a new ipfs-desktop release