diff --git a/config/init.go b/config/init.go index 4a86aa5183e..133ddd4776b 100644 --- a/config/init.go +++ b/config/init.go @@ -138,6 +138,28 @@ func DefaultDatastoreConfig() Datastore { } } +func pebbleSpec() map[string]interface{} { + return map[string]interface{}{ + "type": "measure", + "prefix": "pebble.datastore", + "child": map[string]interface{}{ + "type": "pebbleds", + "path": "pebbleds", + "bytesPerSync": 0, + "bisableWAL": false, + "cacheSize": 0, + "l0CompactionThreshold": 0, + "l0StopWritesThreshold": 0, + "lBaseMaxBytes": 0, + "maxConcurrentCompactions": 0, + "memTableSize": 0, + "memTableStopWritesThreshold": 0, + "walBytesPerSync": 0, + "walMinSyncSeconds": 0, + }, + } +} + func badgerSpec() map[string]interface{} { return map[string]interface{}{ "type": "measure", diff --git a/config/profile.go b/config/profile.go index 63e76d8ec1b..7c7e3007835 100644 --- a/config/profile.go +++ b/config/profile.go @@ -144,6 +144,27 @@ This profile may only be applied when first initializing the node. return nil }, }, + "pebbleds": { + Description: `Configures the node to use the pebble high-performance datastore. + +Pebble is a LevelDB/RocksDB inspired key-value store focused on performance +and internal usage by CockroachDB. +You should use this datastore if: + +- You need a datastore that is focused on performance. +- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical. +- This datastore is good for multi-terrabyte data sets. +- May benefit from tuning depeending on read/write patterns and throughput. +- Performance is helped significnatly by running on a system with plenty of memory. + +This profile may only be applied when first initializing the node.`, + + InitOnly: true, + Transform: func(c *Config) error { + c.Datastore.Spec = pebbleSpec() + return nil + }, + }, "badgerds": { Description: `Configures the node to use the legacy badgerv1 datastore. diff --git a/docs/config.md b/docs/config.md index 5500128abf1..f07267fc03f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -181,6 +181,7 @@ config file at runtime. - [`local-discovery` profile](#local-discovery-profile) - [`default-networking` profile](#default-networking-profile) - [`flatfs` profile](#flatfs-profile) + - [`pebbleds` profile](#pebbleds-profile) - [`badgerds` profile](#badgerds-profile) - [`lowpower` profile](#lowpower-profile) - [`legacy-cid-v0` profile](#legacy-cid-v0-profile) @@ -2416,6 +2417,21 @@ You should use this datastore if: This profile may only be applied when first initializing the node. +### `pebbleds` profile + +Configures the node to use the pebble high-performance datastore. + +Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB. +You should use this datastore if: + +- You need a datastore that is focused on performance. +- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical. +- This datastore is good for multi-terrabyte data sets. +- May benefit from tuning depeending on read/write patterns and throughput. +- Performance is helped significnatly by running on a system with plenty of memory. + +This profile may only be applied when first initializing the node. + ### `badgerds` profile Configures the node to use the legacy badgerv1 datastore. diff --git a/docs/datastores.md b/docs/datastores.md index c18ecb0c7c5..63eb797f1f2 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -35,6 +35,32 @@ Uses a leveldb database to store key value pairs. } ``` +## pebbleds + +Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store. + +#### The following options are availble for tuning pebble. If they are not configured (or assigned their zero-valued), then default values are used. Start using pebble with only default values and configure tuning items are needed for your needs. For a more complete description of these values, see: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options + +* `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB) +* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) +* `cacheSize`: Size of pebble's shared block cache. (default: 8MB) +* `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction. +* `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels. +* `lBaseMaxBytes`: int, Maximum number of bytes for LBase. The base level is the level which L0 is compacted into. +* `maxConcurrentCompactions`: int, Maximum number of concurrent compactions. (default: 1) +* `memTableSize`: int, Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize (default: 4MB) +* `memTableStopWritesThreshold`: int, Limit on the number of queued of MemTables. (default: 2) +* `walBytesPerSync`: int: Sets the number of bytes to write to a WAL before calling Sync on it in the background. (default: 0, no background syncing) +* `walMinSyncSeconds`: int: Sets the minimum duration between syncs of the WAL. (default: 0) + + +```json +{ + "type": "pebbleds", + "path": "", +} +``` + ## badgerds Uses [badger](https://github.com/dgraph-io/badger) as a key value store. diff --git a/go.mod b/go.mod index e188c196852..e89d51cb6a0 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/ceramicnetwork/go-dag-jose v0.1.0 github.com/cheggaaa/pb v1.0.29 + github.com/cockroachdb/pebble v1.1.2 github.com/coreos/go-systemd/v22 v22.5.0 github.com/dustin/go-humanize v1.0.1 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 @@ -28,6 +29,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ds-measure v0.2.0 + github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-ipfs-cmds v0.13.0 github.com/ipfs/go-ipld-cbor v0.1.0 @@ -93,12 +95,18 @@ require ( require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect github.com/cskr/pubsub v1.0.2 // indirect @@ -113,6 +121,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -149,6 +158,8 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -162,7 +173,7 @@ require ( github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect - github.com/mattn/go-colorable v0.1.6 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect @@ -208,6 +219,7 @@ require ( github.com/quic-go/quic-go v0.45.2 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/cors v1.10.1 // indirect github.com/samber/lo v1.46.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 246eef96b5e..3bb39d88954 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -104,6 +106,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= @@ -177,10 +193,14 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -365,6 +385,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ= github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= +github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a h1:OouXE/TSvgFP3Scv/3WA1brxKC+FM40EeusWnLGilgY= +github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= @@ -558,15 +580,15 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= @@ -683,6 +705,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= @@ -725,6 +749,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I= github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -783,6 +808,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= @@ -1198,6 +1224,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index 2ad84e59489..75e21270c92 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -8,6 +8,7 @@ import ( pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git" pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds" pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs" + pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds" pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog" ) @@ -21,6 +22,7 @@ func init() { Preload(pluginbadgerds.Plugins...) Preload(pluginflatfs.Plugins...) Preload(pluginlevelds.Plugins...) + Preload(pluginpebbleds.Plugins...) Preload(pluginpeerlog.Plugins...) Preload(pluginfxtest.Plugins...) Preload(pluginnopfs.Plugins...) diff --git a/plugin/loader/preload_list b/plugin/loader/preload_list index 462a3f39337..190cc65d716 100644 --- a/plugin/loader/preload_list +++ b/plugin/loader/preload_list @@ -9,6 +9,7 @@ iplddagjose github.com/ipfs/kubo/plugin/plugins/dagjose * badgerds github.com/ipfs/kubo/plugin/plugins/badgerds * flatfs github.com/ipfs/kubo/plugin/plugins/flatfs * levelds github.com/ipfs/kubo/plugin/plugins/levelds * +pebbleds github.com/ipfs/kubo/plugin/plugins/pebbleds * peerlog github.com/ipfs/kubo/plugin/plugins/peerlog * fxtest github.com/ipfs/kubo/plugin/plugins/fxtest * -nopfs github.com/ipfs/kubo/plugin/plugins/nopfs * \ No newline at end of file +nopfs github.com/ipfs/kubo/plugin/plugins/nopfs * diff --git a/plugin/plugins/flatfs/flatfs.go b/plugin/plugins/flatfs/flatfs.go index 1a23dfcca9d..397c2656c76 100644 --- a/plugin/plugins/flatfs/flatfs.go +++ b/plugin/plugins/flatfs/flatfs.go @@ -42,7 +42,7 @@ type datastoreConfig struct { syncField bool } -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore +// BadgerdsDatastoreConfig returns a configuration stub for a flatfs datastore // from the given parameters. func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) { diff --git a/plugin/plugins/levelds/levelds.go b/plugin/plugins/levelds/levelds.go index b08872de63d..78331730ed3 100644 --- a/plugin/plugins/levelds/levelds.go +++ b/plugin/plugins/levelds/levelds.go @@ -42,7 +42,7 @@ type datastoreConfig struct { compression ldbopts.Compression } -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore +// DatastoreConfigParser returns a configuration stub for a badger datastore // from the given parameters. func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) { diff --git a/plugin/plugins/pebbleds/pebbleds.go b/plugin/plugins/pebbleds/pebbleds.go new file mode 100644 index 00000000000..d401ad75fe4 --- /dev/null +++ b/plugin/plugins/pebbleds/pebbleds.go @@ -0,0 +1,258 @@ +package pebbleds + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/cockroachdb/pebble" + pebbleds "github.com/ipfs/go-ds-pebble" + "github.com/ipfs/kubo/plugin" + "github.com/ipfs/kubo/repo" + "github.com/ipfs/kubo/repo/fsrepo" +) + +// Plugins is exported list of plugins that will be loaded. +var Plugins = []plugin.Plugin{ + &pebbledsPlugin{}, +} + +type pebbledsPlugin struct{} + +var _ plugin.PluginDatastore = (*pebbledsPlugin)(nil) + +func (*pebbledsPlugin) Name() string { + return "ds-pebble" +} + +func (*pebbledsPlugin) Version() string { + return "0.1.0" +} + +func (*pebbledsPlugin) Init(_ *plugin.Environment) error { + return nil +} + +func (*pebbledsPlugin) DatastoreTypeName() string { + return "pebbleds" +} + +type datastoreConfig struct { + path string + cacheSize int64 + + // Documentation of these values: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options + pebbleOpts *pebble.Options +} + +// PebbleDatastoreConfig returns a configuration stub for a pebble datastore +// from the given parameters. +func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { + return func(params map[string]any) (fsrepo.DatastoreConfig, error) { + var c datastoreConfig + var ok bool + + c.path, ok = params["path"].(string) + if !ok { + return nil, fmt.Errorf("'path' field is missing or not string") + } + + cacheSize, err := getConfigInt("cacheSize", params) + if err != nil { + return nil, err + } + c.cacheSize = int64(cacheSize) + + bytesPerSync, err := getConfigInt("bytesPerSync", params) + if err != nil { + return nil, err + } + disableWAL, err := getConfigBool("disableWAL", params) + if err != nil { + return nil, err + } + l0CompactionThreshold, err := getConfigInt("l0CompactionThreshold", params) + if err != nil { + return nil, err + } + l0StopWritesThreshold, err := getConfigInt("l0StopWritesThreshold", params) + if err != nil { + return nil, err + } + lBaseMaxBytes, err := getConfigInt("lBaseMaxBytes", params) + if err != nil { + return nil, err + } + maxConcurrentCompactions, err := getConfigInt("maxConcurrentCompactions", params) + if err != nil { + return nil, err + } + memTableSize, err := getConfigInt("memTableSize", params) + if err != nil { + return nil, err + } + memTableStopWritesThreshold, err := getConfigInt("memTableStopWritesThreshold", params) + if err != nil { + return nil, err + } + walBytesPerSync, err := getConfigInt("walBytesPerSync", params) + if err != nil { + return nil, err + } + walMinSyncSec, err := getConfigInt("walMinSyncIntervalSeconds", params) + if err != nil { + return nil, err + } + + if bytesPerSync != 0 || disableWAL || l0CompactionThreshold != 0 || l0StopWritesThreshold != 0 || lBaseMaxBytes != 0 || maxConcurrentCompactions != 0 || memTableSize != 0 || memTableStopWritesThreshold != 0 || walBytesPerSync != 0 || walMinSyncSec != 0 { + c.pebbleOpts = &pebble.Options{ + //BytesPerSync: bytesPerSyn, + DisableWAL: disableWAL, + L0CompactionThreshold: l0CompactionThreshold, + L0StopWritesThreshold: l0StopWritesThreshold, + LBaseMaxBytes: int64(lBaseMaxBytes), + MemTableSize: uint64(memTableSize), + MemTableStopWritesThreshold: memTableStopWritesThreshold, + WALBytesPerSync: walBytesPerSync, + } + if maxConcurrentCompactions != 0 { + c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions } + } + if walMinSyncSec != 0 { + c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second } + } + } + + return &c, nil + } +} + +func getConfigBool(name string, params map[string]any) (bool, error) { + val, ok := params[name] + if ok { + bval, ok := val.(bool) + if !ok { + return false, fmt.Errorf("%q field was not a bool", name) + } + return bval, nil + } + return false, nil +} + +func getConfigString(name string, params map[string]any) (string, error) { + val, ok := params[name] + if ok { + sval, ok := val.(string) + if !ok { + return "", fmt.Errorf("%q field was not a string", name) + } + return sval, nil + } + return "", nil +} + +func getConfigInt(name string, params map[string]any) (int, error) { + val, ok := params[name] + if ok { + // TODO: see why val may be an int or a float64. + ival, ok := val.(int) + if !ok { + fval, ok := val.(float64) + if !ok { + return 0, fmt.Errorf("%q field was not an integer or a float64", name) + } + return int(fval), nil + } + return ival, nil + } + return 0, nil +} + +func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec { + return map[string]interface{}{ + "type": "pebbleds", + "path": c.path, + } +} + +func (c *datastoreConfig) Create(path string) (repo.Datastore, error) { + p := c.path + if !filepath.IsAbs(p) { + p = filepath.Join(path, p) + } + + if err := dirWritable(p); err != nil { + return nil, err + } + + return pebbleds.NewDatastore(p, pebbleds.WithCacheSize(c.cacheSize), pebbleds.WithPebbleOpts(c.pebbleOpts)) +} + +// dirWritable checks if a directory is writable. If the directory does +// not exist it is created with writable permission. +func dirWritable(dir string) error { + if dir == "" { + return errors.New("directory not specified") + } + var err error + dir, err = expandHome(dir) + if err != nil { + return err + } + + fi, err := os.Stat(dir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // Directory does not exist, so create it. + err = os.Mkdir(dir, 0775) + if err == nil { + return nil + } + } + if errors.Is(err, fs.ErrPermission) { + err = fs.ErrPermission + } + return fmt.Errorf("directory not writable: %s: %w", dir, err) + } + if !fi.IsDir() { + return fmt.Errorf("not a directory: %s", dir) + } + + // Directory exists, check that a file can be written. + file, err := os.CreateTemp(dir, "writetest") + if err != nil { + if errors.Is(err, fs.ErrPermission) { + err = fs.ErrPermission + } + return fmt.Errorf("directory not writable: %s: %w", dir, err) + } + file.Close() + return os.Remove(file.Name()) +} + +// expandHome expands the path to include the home directory if the path is +// prefixed with `~`. If it isn't prefixed with `~`, the path is returned +// as-is. +func expandHome(path string) (string, error) { + if path == "" { + return path, nil + } + + if path[0] != '~' { + return path, nil + } + + if len(path) > 1 && path[1] != '/' && path[1] != '\\' { + return "", errors.New("cannot expand user-specific home dir") + } + + dir, err := os.UserHomeDir() + if err != nil { + return "", err + } + + return filepath.Join(dir, path[1:]), nil +}