Merge pull request #55 from mailgun/thrawn/test-grpc-gateway

PIP-823: Refactor and Improvements for v1.0.0

CHANGELOG

@@ -4,6 +4,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.0-rc.1] - 2020-10-22
+### Change
+* Added `GUBER_DATA_CENTER` as a config option
+* Use `GUBER_PEER_DISCOVERY_TYPE` to pick a peer discovery type, removed
+'Enable' options from k8s, etcd, and member-list.
+* Added `GUBER_ADVERTISE_ADDRESS` to specify which address is published for
+discovery
+* Gubernator now attempts to detect the proper `GUBER_ADVERTISE_ADDRESS` if
+not specified
+* Gubernator now binds to `localhost` by default instead of binding to
+`0.0.0.0:80` to avoid allowing
+  access to a test version of gubernator from the network.
+* Fix inconsistent tests failing #57
+* Fix GRPC/HTTP Gateway #50
+* Renamed functions to ensure clarity of version
+* Removed deprecated `EtcdAdvertiseAddress` config option
+* Refactored configuration options
+* `member-list` metadata no longer assumes the member-list address is the same
+  as the gubernator advertise address.
+* Now MD5 sums the peer address key when using replicated hash. This ensures
+  better key distribution when using domain names or ip address that are very
+  similar. (gubernator-1, gubernator-2, etc...)
+* Now defaults to `replicated-hash` if `GUBER_PEER_PICKER` is unset
+* Added support for DataCenter fields when using etcd discovery
+* Now storing member-list metadata as JSON instead of glob
+
 ## [0.9.2] - 2020-10-23
 ### Change
 * ETCD discovery now sets the IsOwner property when updating the peers list.

Dockerfile

@@ -10,11 +10,12 @@ RUN go mod download
 
 # Copy the local package files to the container
 ADD . /go/src
-ENV VERSION=dev-build
 
-# Build the bot inside the container
+ARG VERSION
+
+# Build the server inside the container
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo \
-    -ldflags "-w -s -X main.Version=${VERSION}" -o /gubernator /go/src/cmd/gubernator/main.go /go/src/cmd/gubernator/config.go
+    -ldflags "-w -s -X main.Version=$VERSION" -o /gubernator /go/src/cmd/gubernator/main.go
 
 # Create our deploy image
 FROM scratch

Makefile

@@ -6,15 +6,15 @@ VERSION=$(shell cat version)
 LDFLAGS="-X main.Version=$(VERSION)"
 
 test:
-	go test ./... -v -race -count=1
+	go test ./... -v -race -p=1 -count=1
 
 docker:
 	docker build --build-arg VERSION=$(VERSION) -t thrawn01/gubernator:$(VERSION) .
 	docker tag thrawn01/gubernator:$(VERSION) thrawn01/gubernator:latest
 
 release:
-	GOOS=darwin GOARCH=amd64 go build -ldflags $(LDFLAGS) -o gubernator.darwin ./cmd/gubernator/main.go ./cmd/gubernator/config.go
-	GOOS=linux GOARCH=amd64 go build -ldflags $(LDFLAGS) -o gubernator.linux ./cmd/gubernator/main.go ./cmd/gubernator/config.go
+	GOOS=darwin GOARCH=amd64 go build -ldflags $(LDFLAGS) -o gubernator.darwin ./cmd/gubernator/main.go
+	GOOS=linux GOARCH=amd64 go build -ldflags $(LDFLAGS) -o gubernator.linux ./cmd/gubernator/main.go
 
 proto:
 	scripts/proto.sh

algorithms.go

@@ -17,7 +17,7 @@ limitations under the License.
 package gubernator
 
 import (
-	"time"
+	"github.com/mailgun/holster/v3/clock"
 )
 
 // Implements token bucket algorithm for rate limiting. https://en.wikipedia.org/wiki/Token_bucket
@@ -88,7 +88,7 @@ func tokenBucket(s Store, c Cache, r *RateLimitReq) (resp *RateLimitResp, err er
 		if t.Duration != r.Duration {
 			expire := t.CreatedAt + r.Duration
 			if HasBehavior(r.Behavior, Behavior_DURATION_IS_GREGORIAN) {
-				expire, err = GregorianExpiration(time.Now(), r.Duration)
+				expire, err = GregorianExpiration(clock.Now(), r.Duration)
 				if err != nil {
 					return nil, err
 				}
@@ -138,7 +138,7 @@ func tokenBucket(s Store, c Cache, r *RateLimitReq) (resp *RateLimitResp, err er
 	now := MillisecondNow()
 	expire := now + r.Duration
 	if HasBehavior(r.Behavior, Behavior_DURATION_IS_GREGORIAN) {
-		expire, err = GregorianExpiration(time.Now(), r.Duration)
+		expire, err = GregorianExpiration(clock.Now(), r.Duration)
 		if err != nil {
 			return nil, err
 		}
@@ -214,11 +214,11 @@ func leakyBucket(s Store, c Cache, r *RateLimitReq) (resp *RateLimitResp, err er
 		duration := r.Duration
 		rate := float64(duration) / float64(r.Limit)
 		if HasBehavior(r.Behavior, Behavior_DURATION_IS_GREGORIAN) {
-			d, err := GregorianDuration(time.Now(), r.Duration)
+			d, err := GregorianDuration(clock.Now(), r.Duration)
 			if err != nil {
 				return nil, err
 			}
-			n := time.Now()
+			n := clock.Now()
 			expire, err := GregorianExpiration(n, r.Duration)
 			if err != nil {
 				return nil, err
@@ -291,7 +291,7 @@ func leakyBucket(s Store, c Cache, r *RateLimitReq) (resp *RateLimitResp, err er
 
 	duration := r.Duration
 	if HasBehavior(r.Behavior, Behavior_DURATION_IS_GREGORIAN) {
-		n := time.Now()
+		n := clock.Now()
 		expire, err := GregorianExpiration(n, r.Duration)
 		if err != nil {
 			return nil, err

benchmark_test.go

@@ -51,7 +51,7 @@ func BenchmarkServer_GetPeerRateLimitNoBatching(b *testing.B) {
 }
 
 func BenchmarkServer_GetRateLimit(b *testing.B) {
-	client, err := guber.DialV1Server(cluster.GetRandomPeer().Address)
+	client, err := guber.DialV1Server(cluster.GetRandomPeer().GRPCAddress)
 	if err != nil {
 		b.Errorf("NewV1Client err: %s", err)
 	}
@@ -77,7 +77,7 @@ func BenchmarkServer_GetRateLimit(b *testing.B) {
 }
 
 func BenchmarkServer_Ping(b *testing.B) {
-	client, err := guber.DialV1Server(cluster.GetRandomPeer().Address)
+	client, err := guber.DialV1Server(cluster.GetRandomPeer().GRPCAddress)
 	if err != nil {
 		b.Errorf("NewV1Client err: %s", err)
 	}
@@ -105,7 +105,7 @@ func BenchmarkServer_Ping(b *testing.B) {
 }*/
 
 func BenchmarkServer_ThunderingHeard(b *testing.B) {
-	client, err := guber.DialV1Server(cluster.GetRandomPeer().Address)
+	client, err := guber.DialV1Server(cluster.GetRandomPeer().GRPCAddress)
 	if err != nil {
 		b.Errorf("NewV1Client err: %s", err)
 	}

cache.go

@@ -21,8 +21,8 @@ package gubernator
 import (
 	"container/list"
 	"sync"
-	"time"
 
+	"github.com/mailgun/holster/v3/clock"
 	"github.com/mailgun/holster/v3/setter"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -131,7 +131,7 @@ func (c *LRUCache) Add(record *CacheItem) bool {
 
 // Return unix epoch in milliseconds
 func MillisecondNow() int64 {
-	return time.Now().UnixNano() / 1000000
+	return clock.Now().UnixNano() / 1000000
 }
 
 // GetItem returns the item stored in the cache

client.go

@@ -18,8 +18,8 @@ package gubernator
 
 import (
 	"math/rand"
-	"time"
 
+	"github.com/mailgun/holster/v3/clock"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 )
@@ -48,23 +48,26 @@ func DialV1Server(server string) (V1Client, error) {
 	return NewV1Client(conn), nil
 }
 
-// Convert a time.Duration to a unix millisecond timestamp
-func ToTimeStamp(duration time.Duration) int64 {
-	return int64(duration / time.Millisecond)
+// Convert a clock.Duration to a unix millisecond timestamp
+func ToTimeStamp(duration clock.Duration) int64 {
+	return int64(duration / clock.Millisecond)
 }
 
 // Convert a unix millisecond timestamp to a time.Duration
-func FromTimeStamp(ts int64) time.Duration {
-	return time.Now().Sub(FromUnixMilliseconds(ts))
+func FromTimeStamp(ts int64) clock.Duration {
+	return clock.Now().Sub(FromUnixMilliseconds(ts))
 }
 
-func FromUnixMilliseconds(ts int64) time.Time {
-	return time.Unix(0, ts*int64(time.Millisecond))
+func FromUnixMilliseconds(ts int64) clock.Time {
+	return clock.Unix(0, ts*int64(clock.Millisecond))
 }
 
 // Given a list of peers, return a random peer
 func RandomPeer(peers []PeerInfo) PeerInfo {
-	return peers[rand.Intn(len(peers))]
+	rand.Shuffle(len(peers), func(i, j int) {
+		peers[i], peers[j] = peers[j], peers[i]
+	})
+	return peers[0]
 }
 
 // Return a random alpha string of 'n' length