From 60aab61f3823be8b9c913078e3c015419a57edac Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Thu, 8 Feb 2024 15:21:39 -0600 Subject: [PATCH] init commit --- protocol/app/app.go | 34 ++++++++++++++++++++++++++------ protocol/app/flags/flags.go | 39 +++++++++++++++++++++++++++++++++++++ protocol/go.mod | 6 ++++-- protocol/go.sum | 20 ++++++++++++------- 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/protocol/app/app.go b/protocol/app/app.go index cb3e38c93f..17e968958b 100644 --- a/protocol/app/app.go +++ b/protocol/app/app.go @@ -15,6 +15,7 @@ import ( autocliv1 "cosmossdk.io/api/cosmos/autocli/v1" reflectionv1 "cosmossdk.io/api/cosmos/reflection/v1" + sdklog "cosmossdk.io/log" dbm "github.com/cometbft/cometbft-db" abci "github.com/cometbft/cometbft/abci/types" tmjson "github.com/cometbft/cometbft/libs/json" @@ -103,6 +104,7 @@ import ( // Mempool "github.com/dydxprotocol/v4-chain/protocol/mempool" + mempoolpubsub "github.com/numiadata/tools/mempool" // Daemons bridgeclient "github.com/dydxprotocol/v4-chain/protocol/daemons/bridge/client" @@ -188,10 +190,8 @@ import ( "github.com/dydxprotocol/v4-chain/protocol/indexer/msgsender" ) -var ( - // DefaultNodeHome default home directories for the application daemon - DefaultNodeHome string -) +// DefaultNodeHome default home directories for the application daemon +var DefaultNodeHome string var ( _ runtime.AppI = (*App)(nil) @@ -939,7 +939,7 @@ func New( // NOTE: we may consider parsing `appOpts` inside module constructors. For the moment // we prefer to be more strict in what arguments the modules expect. - var skipGenesisInvariants = cast.ToBool(appOpts.Get(crisis.FlagSkipGenesisInvariants)) + skipGenesisInvariants := cast.ToBool(appOpts.Get(crisis.FlagSkipGenesisInvariants)) // NOTE: Any module instantiated in the module manager that is later modified // must be passed by reference here. @@ -1178,7 +1178,29 @@ func New( // initialize BaseApp app.SetInitChainer(app.InitChainer) app.setAnteHandler(encodingConfig.TxConfig) - app.SetMempool(mempool.NewNoOpMempool()) + + // setup mempool + // set custom mempool that emits Google Cloud Pubsub messages upon injestion + noOpMempool := mempool.NewNoOpMempool() + if appFlags.PubSubProjectID != "" && appFlags.PubSubTopic != "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") != "" { + logger.Info("Using Google Cloud Pubsub mempool") + // Note, operators must ensure the environment + // variable is set to the location of their creds file. + app.SetMempool(mempoolpubsub.NewPubSubMempool( + sdklog.NewLogger(os.Stderr), + noOpMempool, + txConfig.TxEncoder(), + cast.ToString(appOpts.Get(cosmosflags.FlagChainID)), + appFlags.PubSubMoniker, + appFlags.PubSubProjectID, + appFlags.PubSubTopic, + false, + )) + } else { + logger.Info("Using no-op mempool") + app.SetMempool(noOpMempool) + } + app.SetBeginBlocker(app.BeginBlocker) app.SetEndBlocker(app.EndBlocker) app.SetCommiter(app.Commiter) diff --git a/protocol/app/flags/flags.go b/protocol/app/flags/flags.go index 6f67611a22..137540edee 100644 --- a/protocol/app/flags/flags.go +++ b/protocol/app/flags/flags.go @@ -11,6 +11,9 @@ import ( // A struct containing the values of all flags. type Flags struct { + PubSubMoniker string + PubSubProjectID string + PubSubTopic string DdAgentHost string DdTraceAgentPort uint16 NonValidatingFullNode bool @@ -23,6 +26,9 @@ type Flags struct { // List of CLI flags. const ( + PubSubMoniker = "pubsub-moniker" + PubSubProjectID = "pubsub-project-id" + PubSubTopic = "pubsub-topic" DdAgentHost = "dd-agent-host" DdTraceAgentPort = "dd-trace-agent-port" NonValidatingFullNodeFlag = "non-validating-full-node" @@ -45,6 +51,21 @@ const ( // These flags should be applied to the `start` command of the V4 Cosmos application. // E.g. `dydxprotocold start --non-validating-full-node true`. func AddFlagsToCmd(cmd *cobra.Command) { + cmd.Flags().String( + PubSubMoniker, + "", + "Set the Google Cloud PubSub node moniker used for mempool events", + ) + cmd.Flags().String( + PubSubProjectID, + "", + "Set the Google Cloud PubSub project ID", + ) + cmd.Flags().String( + PubSubTopic, + "", + "Set the Google Cloud PubSub topic", + ) cmd.Flags().Bool( NonValidatingFullNodeFlag, DefaultNonValidatingFullNode, @@ -132,5 +153,23 @@ func GetFlagValuesFromOptions( } } + if option := appOpts.Get(PubSubMoniker); option != nil { + if v, err := cast.ToStringE(option); err == nil { + result.PubSubMoniker = v + } + } + + if option := appOpts.Get(PubSubProjectID); option != nil { + if v, err := cast.ToStringE(option); err == nil { + result.PubSubProjectID = v + } + } + + if option := appOpts.Get(PubSubTopic); option != nil { + if v, err := cast.ToStringE(option); err == nil { + result.PubSubTopic = v + } + } + return result } diff --git a/protocol/go.mod b/protocol/go.mod index f53735a4c1..28073b5272 100644 --- a/protocol/go.mod +++ b/protocol/go.mod @@ -25,6 +25,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/h2non/gock v1.2.0 + github.com/numiadata/tools/mempool v1.0.0 github.com/ory/dockertest v3.3.5+incompatible github.com/pkg/errors v0.9.1 github.com/rakyll/statik v0.1.7 @@ -43,14 +44,14 @@ require ( require ( cosmossdk.io/errors v1.0.0 - cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca + cosmossdk.io/log v1.2.1 github.com/burdiyan/kafkautil v0.0.0-20190131162249-eaf83ed22d5b github.com/cosmos/iavl v0.20.0 github.com/deckarep/golang-set/v2 v2.3.0 github.com/ethereum/go-ethereum v1.12.0 github.com/ory/dockertest/v3 v3.10.0 github.com/pelletier/go-toml v1.9.5 - github.com/rs/zerolog v1.29.1 + github.com/rs/zerolog v1.30.0 github.com/shopspring/decimal v1.3.1 google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 ) @@ -62,6 +63,7 @@ require ( cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.0 // indirect + cloud.google.com/go/pubsub v1.33.0 // indirect cloud.google.com/go/storage v1.30.1 // indirect cosmossdk.io/core v0.5.1 // indirect cosmossdk.io/depinject v1.0.0-alpha.3 // indirect diff --git a/protocol/go.sum b/protocol/go.sum index dd799915d9..57e68b3e9e 100644 --- a/protocol/go.sum +++ b/protocol/go.sum @@ -117,6 +117,8 @@ cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -145,6 +147,8 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= +cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= @@ -199,8 +203,8 @@ cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z cosmossdk.io/depinject v1.0.0-alpha.3/go.mod h1:eRbcdQ7MRpIPEM5YUJh8k97nxHpYbc3sMUnEtt8HPWU= cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= -cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca h1:msenprh2BLLRwNT7zN56TbBHOGk/7ARQckXHxXyvjoQ= -cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca/go.mod h1:PkIAKXZvaxrTRc++z53XMRvFk8AcGGWYHcMIPzVYX9c= +cosmossdk.io/log v1.2.1 h1:Xc1GgTCicniwmMiKwDxUjO4eLhPxoVdI9vtMW8Ti/uk= +cosmossdk.io/log v1.2.1/go.mod h1:GNSCc/6+DhFIj1aLn/j7Id7PaO8DzNylUZoOYBL9+I4= cosmossdk.io/math v1.0.1 h1:Qx3ifyOPaMLNH/89WeZFH268yCvU4xEcnPLu3sJqPPg= cosmossdk.io/math v1.0.1/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k= cosmossdk.io/tools/rosetta v0.2.1 h1:ddOMatOH+pbxWbrGJKRAawdBkPYLfKXutK9IETnjYxw= @@ -1119,6 +1123,8 @@ github.com/nishanths/exhaustive v0.11.0 h1:T3I8nUGhl/Cwu5Z2hfc92l0e04D2GEW6e0l8p github.com/nishanths/exhaustive v0.11.0/go.mod h1:RqwDsZ1xY0dNdqHho2z6X+bgzizwbLYOWnZbbl2wLB4= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/numiadata/tools/mempool v1.0.0 h1:JRWkytEANfP89xgvJFX3ryT4znb4NbKSXOLmLkIpPz4= +github.com/numiadata/tools/mempool v1.0.0/go.mod h1:1dojBb5jQO75Bps6gVCh81/rZs120xmwUEEu20/YSdY= github.com/nunnatsa/ginkgolinter v0.13.5 h1:fOsPB4CEZOPkyMqF4B9hoqOpooFWU7vWSVkCSscVpgU= github.com/nunnatsa/ginkgolinter v0.13.5/go.mod h1:OBHy4536xtuX3102NM63XRtOyxqZOO02chsaeDWXVO8= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -1266,9 +1272,9 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1834,8 +1840,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=