From 6b92b3d4e5837e16ba9e11abc0269a8c6ce9d8fd Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Mon, 4 Dec 2023 14:24:31 -0800 Subject: [PATCH] updates --- protocol/app/app.go | 31 +++++++++++++++++++++++++------ protocol/app/flags/flags.go | 26 ++++++++++++++++++++++++++ protocol/go.mod | 6 ++++-- protocol/go.sum | 20 +++++++++++++------- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/protocol/app/app.go b/protocol/app/app.go index aa16a8313f..727b112938 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -85,6 +85,7 @@ import ( upgradekeeper "github.com/cosmos/cosmos-sdk/x/upgrade/keeper" upgradetypes "github.com/cosmos/cosmos-sdk/x/upgrade/types" "github.com/gorilla/mux" + mempoolpubsub "github.com/numiadata/tools/mempool" "github.com/rakyll/statik/fs" "github.com/spf13/cast" "google.golang.org/grpc" @@ -95,6 +96,7 @@ import ( "github.com/dydxprotocol/v4-chain/protocol/app/middleware" "github.com/dydxprotocol/v4-chain/protocol/app/prepare" "github.com/dydxprotocol/v4-chain/protocol/app/process" + // Lib "github.com/dydxprotocol/v4-chain/protocol/app/stoppable" "github.com/dydxprotocol/v4-chain/protocol/lib" @@ -182,10 +184,8 @@ import ( "github.com/dydxprotocol/v4-chain/protocol/indexer/msgsender" ) -var ( - // DefaultNodeHome default home directories for the application daemon - DefaultNodeHome string -) +// DefaultNodeHome default home directories for the application daemon +var DefaultNodeHome string var ( _ runtime.AppI = (*App)(nil) @@ -872,7 +872,7 @@ func New( // NOTE: we may consider parsing `appOpts` inside module constructors. For the moment // we prefer to be more strict in what arguments the modules expect. - var skipGenesisInvariants = cast.ToBool(appOpts.Get(crisis.FlagSkipGenesisInvariants)) + skipGenesisInvariants := cast.ToBool(appOpts.Get(crisis.FlagSkipGenesisInvariants)) // NOTE: Any module instantiated in the module manager that is later modified // must be passed by reference here. @@ -1102,7 +1102,26 @@ func New( // initialize BaseApp app.SetInitChainer(app.InitChainer) app.setAnteHandler(encodingConfig.TxConfig) - app.SetMempool(mempool.NewNoOpMempool()) + + // set custom mempool that emits Google Cloud Pubsub messages upon injestion + noOpMempool := mempool.NewNoOpMempool() + if appFlags.PubSubProjectID != "" && appFlags.PubSubTopic != "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") != "" { + // Note, operators must ensure the environment + // variable is set to the location of their creds file. + app.SetMempool(mempoolpubsub.NewPubSubMempool( + sdklog.NewLogger(os.Stderr), + noOpMempool, + txConfig.TxEncoder(), + cast.ToString(appOpts.Get(cosmosflags.FlagChainID)), + cast.ToString(appOpts.Get(cosmosflags.FlagNode)), // TODO/XXX: In order to get the moniker, we have to get/parse the CometBFT config file. + appFlags.PubSubProjectID, + appFlags.PubSubTopic, + false, + )) + } else { + app.SetMempool(noOpMempool) + } + app.SetBeginBlocker(app.BeginBlocker) app.SetEndBlocker(app.EndBlocker) app.SetCommiter(app.Commiter) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 21333d95f1..26ff7d2c74 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -11,6 +11,8 @@ import ( // A struct containing the values of all flags. type Flags struct { + PubSubProjectID string + PubSubTopic string DdAgentHost string DdTraceAgentPort uint16 NonValidatingFullNode bool @@ -22,6 +24,8 @@ type Flags struct { // List of CLI flags. const ( + PubSubProjectID = "pubsub-project-id" + PubSubTopic = "pubsub-topic" DdAgentHost = "dd-agent-host" DdTraceAgentPort = "dd-trace-agent-port" NonValidatingFullNodeFlag = "non-validating-full-node" @@ -42,6 +46,16 @@ const ( // These flags should be applied to the `start` command of the V4 Cosmos application. // E.g. `dydxprotocold start --non-validating-full-node true`. func AddFlagsToCmd(cmd *cobra.Command) { + cmd.Flags().String( + PubSubProjectID, + "", + "Set the Google Cloud PubSub project ID", + ) + cmd.Flags().String( + PubSubTopic, + "", + "Set the Google Cloud PubSub topic", + ) cmd.Flags().Bool( NonValidatingFullNodeFlag, DefaultNonValidatingFullNode, @@ -117,5 +131,17 @@ func GetFlagValuesFromOptions( } } + if option := appOpts.Get(PubSubProjectID); option != nil { + if v, err := cast.ToStringE(option); err == nil { + result.PubSubProjectID = v + } + } + + if option := appOpts.Get(PubSubTopic); option != nil { + if v, err := cast.ToStringE(option); err == nil { + result.PubSubTopic = v + } + } + return result } diff --git a/protocol/go.mod b/protocol/go.mod index b3e0b605d6..040f7e420f 100644 --- a/protocol/go.mod +++ b/protocol/go.mod @@ -44,10 +44,11 @@ require ( require ( cosmossdk.io/errors v1.0.0 - cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca + cosmossdk.io/log v1.2.1 github.com/burdiyan/kafkautil v0.0.0-20190131162249-eaf83ed22d5b github.com/deckarep/golang-set/v2 v2.3.0 github.com/ethereum/go-ethereum v1.12.0 + github.com/numiadata/tools/mempool v0.0.0-20231204222304-9f4924cd910f github.com/ory/dockertest/v3 v3.10.0 github.com/shopspring/decimal v1.3.1 google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 @@ -60,6 +61,7 @@ require ( cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.0 // indirect + cloud.google.com/go/pubsub v1.33.0 // indirect cloud.google.com/go/storage v1.30.1 // indirect cosmossdk.io/core v0.5.1 // indirect cosmossdk.io/depinject v1.0.0-alpha.3 // indirect @@ -286,7 +288,7 @@ require ( github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rs/cors v1.8.3 // indirect - github.com/rs/zerolog v1.29.1 // indirect + github.com/rs/zerolog v1.30.0 // indirect github.com/ryancurrah/gomodguard v1.3.0 // indirect github.com/ryanrolds/sqlclosecheck v0.4.0 // indirect github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect diff --git a/protocol/go.sum b/protocol/go.sum index 0890541aa1..ba35c7db0d 100644 --- a/protocol/go.sum +++ b/protocol/go.sum @@ -117,6 +117,8 @@ cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -145,6 +147,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= @@ -199,8 +203,8 @@ cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z cosmossdk.io/depinject v1.0.0-alpha.3/go.mod h1:eRbcdQ7MRpIPEM5YUJh8k97nxHpYbc3sMUnEtt8HPWU= cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= -cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca h1:msenprh2BLLRwNT7zN56TbBHOGk/7ARQckXHxXyvjoQ= -cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca/go.mod h1:PkIAKXZvaxrTRc++z53XMRvFk8AcGGWYHcMIPzVYX9c= +cosmossdk.io/log v1.2.1 h1:Xc1GgTCicniwmMiKwDxUjO4eLhPxoVdI9vtMW8Ti/uk= +cosmossdk.io/log v1.2.1/go.mod h1:GNSCc/6+DhFIj1aLn/j7Id7PaO8DzNylUZoOYBL9+I4= cosmossdk.io/math v1.0.1 h1:Qx3ifyOPaMLNH/89WeZFH268yCvU4xEcnPLu3sJqPPg= cosmossdk.io/math v1.0.1/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k= cosmossdk.io/tools/rosetta v0.2.1 h1:ddOMatOH+pbxWbrGJKRAawdBkPYLfKXutK9IETnjYxw= @@ -1119,6 +1123,8 @@ github.com/nishanths/exhaustive v0.11.0 h1:T3I8nUGhl/Cwu5Z2hfc92l0e04D2GEW6e0l8p github.com/nishanths/exhaustive v0.11.0/go.mod h1:RqwDsZ1xY0dNdqHho2z6X+bgzizwbLYOWnZbbl2wLB4= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/numiadata/tools/mempool v0.0.0-20231204222304-9f4924cd910f h1:QhannsIaCEjTRAFCze7qjl8VDx/J4Jfj2sMUcwRXHXQ= +github.com/numiadata/tools/mempool v0.0.0-20231204222304-9f4924cd910f/go.mod h1:1dojBb5jQO75Bps6gVCh81/rZs120xmwUEEu20/YSdY= github.com/nunnatsa/ginkgolinter v0.13.5 h1:fOsPB4CEZOPkyMqF4B9hoqOpooFWU7vWSVkCSscVpgU= github.com/nunnatsa/ginkgolinter v0.13.5/go.mod h1:OBHy4536xtuX3102NM63XRtOyxqZOO02chsaeDWXVO8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -1266,9 +1272,9 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1834,8 +1840,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=