From edd2dc883a5711076eb62396130811b2467b4ee3 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Tue, 1 Sep 2020 17:39:45 -0400 Subject: [PATCH] Use unsafe-free murmur3 in bloom filter (#2577) Use misuses of unsafe-free murmur3 in bloom filter, and converge on a single library for murmur3 hashes. --- go.mod | 9 ++++--- go.sum | 24 ++++++++++++------- src/aggregator/hash/hash.go | 2 +- src/aggregator/sharding/hash.go | 2 +- src/dbnode/client/connection_pool.go | 4 ++-- src/dbnode/persist/fs/bloom_filter.go | 2 +- src/dbnode/persist/fs/read_write_test.go | 2 +- src/dbnode/persist/fs/write.go | 2 +- src/dbnode/sharding/shardset.go | 4 ++-- .../commitlog/source_prop_test.go | 2 +- 10 files changed, 30 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 6f65fd5fa8..9b641c9786 100644 --- a/go.mod +++ b/go.mod @@ -48,11 +48,11 @@ require ( github.com/influxdata/influxdb v1.7.7 github.com/jhump/protoreflect v1.6.1 github.com/json-iterator/go v1.1.9 - github.com/leanovate/gopter v0.2.3-0.20181005062252-e2604588f4db + github.com/leanovate/gopter v0.2.8 github.com/lib/pq v1.6.0 // indirect github.com/lightstep/lightstep-tracer-go v0.18.1 github.com/m3db/bitset v2.0.0+incompatible - github.com/m3db/bloom v3.0.0+incompatible + github.com/m3db/bloom/v4 v4.0.0-20200901140942-52efb8544fe9 github.com/m3db/build-tools v0.0.0-20181013000606-edd1bdd1df8a github.com/m3db/m3x v0.0.0-20190408051622-ebf3c7b94afd // indirect github.com/m3db/prometheus_client_golang v0.8.1 @@ -60,7 +60,7 @@ require ( github.com/m3db/prometheus_common v0.0.0-20180517030744-25aaa3dff79b github.com/m3db/prometheus_procfs v0.8.1 github.com/m3db/stackadler32 v0.0.0-20180104200216-bfebcd73ef6f - github.com/m3db/stackmurmur3 v0.0.0-20171110233611-744c0229c12e // indirect + github.com/m3db/stackmurmur3/v2 v2.0.2 github.com/m3db/tools v0.0.0-20181008195521-c6ded3f34878 github.com/m3dbx/pilosa v0.0.0-20200602205121-7f389745e9ab github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de @@ -97,9 +97,8 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.6.1 github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.6.1 github.com/subosito/gotenv v1.2.1-0.20190917103637-de67a6614a4d // indirect - github.com/twmb/murmur3 v1.1.3 github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 github.com/uber-go/atomic v0.0.0-00010101000000-000000000000 // indirect github.com/uber-go/tally v3.3.13+incompatible diff --git a/go.sum b/go.sum index 9cb9f35f9b..b123b35cdb 100644 --- a/go.sum +++ b/go.sum @@ -435,8 +435,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/leanovate/gopter v0.2.3-0.20181005062252-e2604588f4db h1:OIgdqANJZZBFAwpmDekbBGzwhdFptLf56fuVpMwv0Qg= -github.com/leanovate/gopter v0.2.3-0.20181005062252-e2604588f4db/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= +github.com/leanovate/gopter v0.2.8 h1:eFPtJ3aa5zLfbxGROSNY75T9Dume60CWBAqoWQ3h/ig= +github.com/leanovate/gopter v0.2.8/go.mod h1:gNcbPWNEWRe4lm+bycKqxUYoH5uoVje5SkOJ3uoLer8= github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc= github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 h1:143Bb8f8DuGWck/xpNUOckBVYfFbBTnLevfRZ1aVVqo= @@ -446,8 +446,10 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/m3db/bitset v2.0.0+incompatible h1:wMgri1Z2QSwJ8K/7ZuV7vE4feLOT7EofVC8RakIOybI= github.com/m3db/bitset v2.0.0+incompatible/go.mod h1:X8CCqZmZxs2O6d4qHhiqtAKCin4G5mScPhiwX9rsc5c= -github.com/m3db/bloom v3.0.0+incompatible h1:cZzeDfFmgJ076MBv0zM8ZHS/soNoll7F+r5hudgxPCk= -github.com/m3db/bloom v3.0.0+incompatible/go.mod h1:W6XzpFw4t+CIYq+NGyp5c2394YsUc1P1+W/KAWty2lU= +github.com/m3db/bloom v3.0.1+incompatible h1:EILDlnoiPKJAGtg3RCm7Zf5VIpwh+hgcHokoZNzeJy8= +github.com/m3db/bloom v3.0.1+incompatible/go.mod h1:W6XzpFw4t+CIYq+NGyp5c2394YsUc1P1+W/KAWty2lU= +github.com/m3db/bloom/v4 v4.0.0-20200901140942-52efb8544fe9 h1:H5Iznc9FI44Sekos8STE+Hj2cPohzaYfWTUHf77q6RY= +github.com/m3db/bloom/v4 v4.0.0-20200901140942-52efb8544fe9/go.mod h1:JDmGHlO6ygyY1V9eOHtXiNl3+axznDTrBqwWEeWALlQ= github.com/m3db/build-tools v0.0.0-20181013000606-edd1bdd1df8a h1:CwsSHIJLeCESKdZ844jXg/3rQD3yA5azuVlJBp5w8U8= github.com/m3db/build-tools v0.0.0-20181013000606-edd1bdd1df8a/go.mod h1:Pk9AtZeKuCO2xcAth0gxwzRNFv4lV26GPSx4I6A7DQ8= github.com/m3db/m3x v0.0.0-20190408051622-ebf3c7b94afd h1:wzLBtXzxZM9b6IXwLSRE5crynocLTCuRDpGDaOJzyuI= @@ -462,8 +464,10 @@ github.com/m3db/prometheus_procfs v0.8.1 h1:LsxWzVELhDU9sLsZTaFLCeAwCn7bC7qecZcK github.com/m3db/prometheus_procfs v0.8.1/go.mod h1:N8lv8fLh3U3koZx1Bnisj60GYUMDpWb09x1R+dmMOJo= github.com/m3db/stackadler32 v0.0.0-20180104200216-bfebcd73ef6f h1:+FWCOZjB96lBc0L7lvz6/TW1gF5fEVd6JQNQG59CpZE= github.com/m3db/stackadler32 v0.0.0-20180104200216-bfebcd73ef6f/go.mod h1:VJQWii2dj/AE1cRKup4MXL4TKbbs5J4wZnNe5I1CZaw= -github.com/m3db/stackmurmur3 v0.0.0-20171110233611-744c0229c12e h1:B4xyv9ezWgbe30jxYzXxR45iE0Os7oZchmmyuQhGRVg= -github.com/m3db/stackmurmur3 v0.0.0-20171110233611-744c0229c12e/go.mod h1:hkR/F91aDUfPxDv5Jw55ifIXV6ZW1jzNHW0d8GP02rw= +github.com/m3db/stackmurmur3 v1.0.1 h1:ASTdJ6Bd2m34dgsnOBZHrwWdKGcpaavP3MokiKL/ERs= +github.com/m3db/stackmurmur3 v1.0.1/go.mod h1:hkR/F91aDUfPxDv5Jw55ifIXV6ZW1jzNHW0d8GP02rw= +github.com/m3db/stackmurmur3/v2 v2.0.2 h1:q/kOlC12PwOib9XziHUTM/jsCQULHnRxBz/H4hT/Aro= +github.com/m3db/stackmurmur3/v2 v2.0.2/go.mod h1:HnguA9wfEbTqYdTT/5klUlhxSDPRMGaWyvLzugxW+MA= github.com/m3db/thrift v0.0.0-20190820191926-05b5a2227fe4 h1:1x3mMuURd3wqKJ2qVjhRYOAmL9g4EA9JTagWB/y/3xo= github.com/m3db/thrift v0.0.0-20190820191926-05b5a2227fe4/go.mod h1:xVfRinGzD3cYDRvMjy6RkIwM+iNL2KHNLZjT0VpVZT8= github.com/m3db/tools v0.0.0-20181008195521-c6ded3f34878 h1:kww0LtVVfGrXR7Ofpbi/9bvc2EGYMQC0LCH/gQXoolE= @@ -474,6 +478,8 @@ github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de h1:C4DpCfTNzJf5RhJqxO github.com/m3dbx/vellum v0.0.0-20200826162549-f94c029903de/go.mod h1:DOTAUfV4bzK6Nrb0dboT/oCG0DnQuX+/n0jfZPh6xxI= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.2 h1:znVR8Q4g7/WlcvsxLBRWvo+vtFJUAbDn3w+Yak2xVMI= +github.com/magiconair/properties v1.8.2/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -689,6 +695,8 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -737,8 +745,8 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA= -github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.4 h1:NnlAxelwOgdQDmYuV0T/K+tpDQ/8wdsDVOGmvUqBOCw= +github.com/twmb/murmur3 v1.1.4/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 h1:DRC1ubdb3ZmyyIeCSTxjZIQAnpLPfKVgYrLETQuOPjo= github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553/go.mod h1:Rj7Csq/tZ/egz+Ltc2IVpsA5309AmSMEswjkTZmq2Xc= github.com/uber-go/tally v3.3.13+incompatible h1:5ic2UsDwjcWsw9jvEdWEE2XsmGCLMTt5Ukg4d74fed4= diff --git a/src/aggregator/hash/hash.go b/src/aggregator/hash/hash.go index cab04b71d6..6bd798a227 100644 --- a/src/aggregator/hash/hash.go +++ b/src/aggregator/hash/hash.go @@ -24,7 +24,7 @@ // were added. package hash -import "github.com/twmb/murmur3" +import "github.com/m3db/stackmurmur3/v2" // Hash128 is a 128-bit hash of an ID consisting of two unsigned 64-bit ints. type Hash128 [2]uint64 diff --git a/src/aggregator/sharding/hash.go b/src/aggregator/sharding/hash.go index 906f6b4496..6172e6a6a4 100644 --- a/src/aggregator/sharding/hash.go +++ b/src/aggregator/sharding/hash.go @@ -26,7 +26,7 @@ import ( "github.com/m3db/m3/src/metrics/metric/id" - "github.com/twmb/murmur3" + murmur3 "github.com/m3db/stackmurmur3/v2" ) const ( diff --git a/src/dbnode/client/connection_pool.go b/src/dbnode/client/connection_pool.go index 315886c175..bcfc04973c 100644 --- a/src/dbnode/client/connection_pool.go +++ b/src/dbnode/client/connection_pool.go @@ -32,8 +32,8 @@ import ( "github.com/m3db/m3/src/dbnode/generated/thrift/rpc" "github.com/m3db/m3/src/dbnode/topology" xclose "github.com/m3db/m3/src/x/close" + "github.com/m3db/stackmurmur3/v2" - "github.com/twmb/murmur3" "github.com/uber/tchannel-go/thrift" "go.uber.org/zap" ) @@ -80,7 +80,7 @@ type healthCheckFn func(client rpc.TChanNode, opts Options) error type sleepFn func(t time.Duration) func newConnectionPool(host topology.Host, opts Options) connectionPool { - seed := int64(murmur3.Sum32([]byte(host.Address()))) + seed := int64(murmur3.StringSum32(host.Address())) p := &connPool{ opts: opts, diff --git a/src/dbnode/persist/fs/bloom_filter.go b/src/dbnode/persist/fs/bloom_filter.go index 4a2f400d54..09f6e2612d 100644 --- a/src/dbnode/persist/fs/bloom_filter.go +++ b/src/dbnode/persist/fs/bloom_filter.go @@ -23,7 +23,7 @@ package fs import ( "os" - "github.com/m3db/bloom" + "github.com/m3db/bloom/v4" "github.com/m3db/m3/src/dbnode/digest" "github.com/m3db/m3/src/x/mmap" ) diff --git a/src/dbnode/persist/fs/read_write_test.go b/src/dbnode/persist/fs/read_write_test.go index 787c1febf6..c931946172 100644 --- a/src/dbnode/persist/fs/read_write_test.go +++ b/src/dbnode/persist/fs/read_write_test.go @@ -29,7 +29,7 @@ import ( "testing" "time" - "github.com/m3db/bloom" + "github.com/m3db/bloom/v4" "github.com/m3db/m3/src/dbnode/digest" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/x/checked" diff --git a/src/dbnode/persist/fs/write.go b/src/dbnode/persist/fs/write.go index 4e865aba7f..b7b6b3574a 100644 --- a/src/dbnode/persist/fs/write.go +++ b/src/dbnode/persist/fs/write.go @@ -29,7 +29,7 @@ import ( "sort" "time" - "github.com/m3db/bloom" + "github.com/m3db/bloom/v4" "github.com/m3db/m3/src/dbnode/digest" "github.com/m3db/m3/src/dbnode/persist" "github.com/m3db/m3/src/dbnode/persist/fs/msgpack" diff --git a/src/dbnode/sharding/shardset.go b/src/dbnode/sharding/shardset.go index d3b182b45e..5bf836a1d2 100644 --- a/src/dbnode/sharding/shardset.go +++ b/src/dbnode/sharding/shardset.go @@ -26,7 +26,7 @@ import ( "github.com/m3db/m3/src/cluster/shard" "github.com/m3db/m3/src/x/ident" - twmbmurmur3 "github.com/twmb/murmur3" + "github.com/m3db/stackmurmur3/v2" ) var ( @@ -160,6 +160,6 @@ func NewHashGenWithSeed(seed uint32) HashGen { // NewHashFn generates a HashFN based on murmur32 with a given seed func NewHashFn(length int, seed uint32) HashFn { return func(id ident.ID) uint32 { - return twmbmurmur3.SeedSum32(seed, id.Bytes()) % uint32(length) + return murmur3.SeedSum32(seed, id.Bytes()) % uint32(length) } } diff --git a/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go b/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go index 3b0d808eb8..feccae682b 100644 --- a/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go +++ b/src/dbnode/storage/bootstrap/bootstrapper/commitlog/source_prop_test.go @@ -53,7 +53,7 @@ import ( "github.com/leanovate/gopter" "github.com/leanovate/gopter/gen" "github.com/leanovate/gopter/prop" - "github.com/twmb/murmur3" + murmur3 "github.com/m3db/stackmurmur3/v2" "github.com/stretchr/testify/require" )