Increase max message receive size.

This commit changes the grpc-proxy internal to dataplane so that it can
send / receive messages up to 50MB. Prior to this change, it would only
allow 4MB payloads, which was insufficient for some proxies that has a
very large number of upstreams. Whenever this limit was exceeded, the
gRPC connection to the server would be terminated, causing a reconnect
to occur, which would then repeat the issue all over again.

Because the underlying grpc-proxy library did not support configuring
this field, I have had to refer to a custom fork of my own. Eventually
if the corresponding change makes its way upstream, the go.mod file can
point back to the original codebase.

.changelog/357.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+Fix issue where the internal grpc-proxy would hit the max message size limit for xDS streams with a large amount of configuration.
+```

go.mod

@@ -7,8 +7,8 @@ replace github.com/hashicorp/consul/proto-public v0.4.1 => github.com/hashicorp/
 
 require (
 	dario.cat/mergo v1.0.0
-	github.com/adamthesax/grpc-proxy v0.0.0-20220525203857-13e92d14f87a
 	github.com/armon/go-metrics v0.4.1
+	github.com/hashi-derek/grpc-proxy v0.0.0-20231207191910-191266484d75
 	github.com/hashicorp/consul-server-connection-manager v0.1.6
 	github.com/hashicorp/consul/proto-public v0.4.1
 	github.com/hashicorp/go-hclog v1.2.2
@@ -45,7 +45,7 @@ require (
 	github.com/stretchr/objx v0.5.0 // indirect
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect

go.sum

@@ -37,8 +37,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/adamthesax/grpc-proxy v0.0.0-20220525203857-13e92d14f87a h1:8fjfNnk9RLn3F4R4XEljSOZARy1+h1f0KTh6xGFefjw=
-github.com/adamthesax/grpc-proxy v0.0.0-20220525203857-13e92d14f87a/go.mod h1:Aku9EjGILrB1V88F+yfJ8CaIVaKqDeWkW2vkCbY2WSA=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -145,6 +143,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashi-derek/grpc-proxy v0.0.0-20231207191910-191266484d75 h1:V5Uqf7VoWMd6UhNf/5EMA8LMPUm95GYvk2YF5SzT24o=
+github.com/hashi-derek/grpc-proxy v0.0.0-20231207191910-191266484d75/go.mod h1:5eEnHfK72jOkp4gC1dI/Q/E9MFNOM/ewE/vql5ijV3g=
 github.com/hashicorp/consul-server-connection-manager v0.1.6 h1:ktj8Fi+dRXn9hhM+FXsfEJayhzzgTqfH08Ne5M6Fmug=
 github.com/hashicorp/consul-server-connection-manager v0.1.6/go.mod h1:HngMIv57MT+pqCVeRQMa1eTB5dqnyMm8uxjyv+Hn8cs=
 github.com/hashicorp/consul/proto-public v0.1.2-0.20230929231147-632fd65c091c h1:d1ULTfDs6Hha01yfITC55MPGIsQpv0VqQfS45WZHJiY=
@@ -371,8 +371,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/consuldp/xds.go

@@ -9,8 +9,8 @@ import (
 	"net"
 	"strings"
 
-	"github.com/adamthesax/grpc-proxy/proxy"
 	"github.com/armon/go-metrics"
+	"github.com/hashi-derek/grpc-proxy/proxy"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
@@ -20,6 +20,7 @@ import (
 const (
 	metadataKeyToken   = "x-consul-token"
 	envoyADSMethodName = "envoy.service.discovery.v3.AggregatedDiscoveryService/DeltaAggregatedResources"
+	maxRecvSize        = 50 * 1024 * 1024
 )
 
 // director is the helper called by the unknown service gRPC handler. This helper is responsible for injecting the ACL token
@@ -73,7 +74,9 @@ func (cdp *ConsulDataplane) setupXDSServer() error {
 	// However, we needed this fix (https://github.com/mwitkow/grpc-proxy/pull/62) which was available on the fork we are using.
 	// TODO: Switch to the main library once the fix is merged to keep upto date.
 	newGRPCServer := grpc.NewServer(
-		grpc.UnknownServiceHandler(proxy.TransparentHandler(cdp.director)),
+		// Increase the maximum message size due to large proxies sometimes exceeding the default 4MB limit.
+		grpc.MaxRecvMsgSize(maxRecvSize),
+		grpc.UnknownServiceHandler(proxy.TransparentHandlerWithOpts(cdp.director, grpc.MaxCallRecvMsgSize(maxRecvSize))),
 		grpc.StreamInterceptor(cdp.streamInterceptor()),
 	)