Merge PR #41: refactor testnet for better fit in CI

* wip

* first pass go based test net

* cleanup

* accounts genesis

* generating keys, accounts, and ethereum genesis

* generating and signing gentx

* go mod files

* trying to bring up containers

* bringing up containers

* validators failing to connect

* disable state sync

* all validators up, miscommunicating

* nodes signing, contracts deploying

* orchestators failing on delegate keys

* no need to give orchestrator consensus key

* adding debuging rpc

* delegate keys dump

* failing keygen test

* failing test for mnemonic mismatch

* unused code

* use fundraiser paths in key gen

* happy path 2

* auto remove containers

* cleanup

* desperate attempt to keep containers clean

* deterministic test

* happy path passes

* use go test for integration test

* debug proto check breaking

* build binary before test

* check out put

* short circuit on failed contract deploy

* proto breaking debugging

* fix integration test action

* checkout project

* maybe they intended to use an env

* build test in step

* cmd typo

* attempting to read logs from tet runner

* only pull stderr

.github/workflows/integration-tests.yml

@@ -2,17 +2,21 @@ name: Integration tests
 
 on:
   push:
-    branches: [master, main]
+    branches: [ master, main ]
   pull_request:
-    branches: [master, main]
+    branches: [ master, main ]
 
 jobs:
   happy-path:
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
       - uses: actions/checkout@v2
       - name: Run all up happy-path test
-        run: make e2e_happy_path
+        run: cd testnet && go test -c && cd .. && ./testnet/testnet.test -test.run BasicChainDynamicKeys -test.failfast
   validator-out:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -6,6 +6,7 @@ tests/dockerfile/gravity.tar.gz
 solidity/dist
 testdata/
 orchestrator/bins
+out/
 
 # IDE files
 .idea/
@@ -14,3 +15,5 @@ orchestrator/bins
 cache
 artifacts
 *.test
+
+.DS_Store

ethereum/Dockerfile

@@ -1,4 +1,4 @@
-FROM ethereum/client-go:stable
+FROM ethereum/client-go:v1.10.3
 
 RUN apk add --no-cache curl
 
@@ -18,5 +18,6 @@ ENTRYPOINT geth --identity "GravityTestnet" --nodiscover \
            --http.vhosts "*" \
            --miner.threads=1 \
            --nousb \
-           --verbosity=5 \
-           --miner.etherbase=0xBf660843528035a5A4921534E156a27e64B231fE
+           --verbosity=3 \
+           --miner.etherbase=0xBf660843528035a5A4921534E156a27e64B231fE \
+           --rpc.allow-unprotected-txs

module/Makefile

@@ -92,7 +92,8 @@ proto-lint:
 	@$(DOCKER_BUF) lint --error-format=json
 
 proto-check-breaking:
-	@$(DOCKER_BUF) breaking --against $(HTTPS_GIT)#branch=main
+	@echo $(DOCKER_BUF) breaking --against "$(env.HTTPS_GIT)#branch=main"
+	@$(DOCKER_BUF) breaking --against "$(env.HTTPS_GIT)#branch=main"
 
 TM_URL           = https://raw.githubusercontent.com/tendermint/tendermint/v0.34.0-rc3/proto/tendermint
 GOGO_PROTO_URL   = https://raw.githubusercontent.com/regen-network/protobuf/cosmos

module/cmd/gravity/cmd/root_test.go

@@ -0,0 +1,60 @@
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"github.com/cosmos/cosmos-sdk/client/keys"
+	"github.com/cosmos/cosmos-sdk/crypto/hd"
+	"github.com/cosmos/cosmos-sdk/crypto/keyring"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/libs/cli"
+	"strings"
+	"testing"
+)
+
+type KeyOutput struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+	Address string `json:"address"`
+	PubKey string `json:"pubkey"`
+}
+
+func TestKeyGen(t *testing.T) {
+	mnemonic := "weasel lunch attack blossom tone drum unfair worry risk level negative height sight nation inside task oyster client shiver aware neck mansion gun dune"
+
+	// generate key from binary
+	keyCmd := keys.AddKeyCommand()
+	keyCmd.Flags().String(cli.OutputFlag, "json", "output flag")
+	keyCmd.SetArgs([]string{"--dry-run=true", "--output=json", "--recover=true", "orch"})
+	keyCmd.SetIn(strings.NewReader(mnemonic + "\n"))
+
+	buf := bytes.NewBuffer(nil)
+	keyCmd.SetOut(buf)
+	keyCmd.SetErr(buf)
+
+	err := Execute(keyCmd)
+	require.NoError(t, err)
+
+	var key KeyOutput
+	output := buf.Bytes()
+	t.Log("outputs: ", string(output))
+	err = json.Unmarshal(output, &key)
+	require.NoError(t, err)
+
+	// generate a memory key directly
+	kb, err := keyring.New("testnet", keyring.BackendMemory, "", nil)
+	if err != nil {
+		return
+	}
+
+	keyringAlgos, _ := kb.SupportedAlgorithms()
+	algo, err := keyring.NewSigningAlgoFromString(string(hd.Secp256k1Type), keyringAlgos)
+	if err != nil {
+		return
+	}
+
+	account, err := kb.NewAccount("", mnemonic, "", "m/44'/118'/0'/0/0", algo)
+	require.NoError(t, err)
+
+	require.Equal(t, account.GetAddress().String(), key.Address)
+}

module/proto/gravity/v1/query.proto

@@ -23,16 +23,17 @@ service Query {
   rpc SignerSetTx(SignerSetTxRequest) returns (SignerSetTxResponse) {
     // option (google.api.http).get = "/gravity/v1/signer_set";
   }
-  rpc LatestSignerSetTx(LatestSignerSetTxRequest) returns (SignerSetTxResponse) {
+  rpc LatestSignerSetTx(LatestSignerSetTxRequest)
+      returns (SignerSetTxResponse) {
     // option (google.api.http).get = "/gravity/v1/signer_set/latest";
   }
   rpc BatchTx(BatchTxRequest) returns (BatchTxResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/batch_txs/{token_contract}/{nonce}";
+    // "/gravity/v1/batch_txs/{token_contract}/{nonce}";
   }
   rpc ContractCallTx(ContractCallTxRequest) returns (ContractCallTxResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/contract_call_txs/{invalidation_id}/{invalidation_nonce}";
+    // "/gravity/v1/contract_call_txs/{invalidation_id}/{invalidation_nonce}";
   }
 
   // get collections of outgoing traffic from the bridge
@@ -54,42 +55,43 @@ service Query {
   rpc SignerSetTxConfirmations(SignerSetTxConfirmationsRequest)
       returns (SignerSetTxConfirmationsResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/signer_sets/ethereum_signatures";
+    // "/gravity/v1/signer_sets/ethereum_signatures";
   }
   rpc BatchTxConfirmations(BatchTxConfirmationsRequest)
       returns (BatchTxConfirmationsResponse) {
-    // option (google.api.http).get = "/gravity/v1/batch_txs/ethereum_signatures";
+    // option (google.api.http).get =
+    // "/gravity/v1/batch_txs/ethereum_signatures";
   }
   rpc ContractCallTxConfirmations(ContractCallTxConfirmationsRequest)
       returns (ContractCallTxConfirmationsResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/logic_calls/ethereum_signatures";
+    // "/gravity/v1/logic_calls/ethereum_signatures";
   }
 
   // ^^^^^^^^^^^^ seem okay for now ^^^^^^
 
   // pending ethereum signature queries for orchestrators to figure out which
   // signatures they are missing
   // TODO: can/should we group this into one endpoint?
-  rpc UnsignedSignerSetTxs(
-      UnsignedSignerSetTxsRequest)
+  rpc UnsignedSignerSetTxs(UnsignedSignerSetTxsRequest)
       returns (UnsignedSignerSetTxsResponse) {
-    // option (google.api.http).get = "/gravity/v1/SignerSetTxs/{address}/pending";
+    // option (google.api.http).get =
+    // "/gravity/v1/SignerSetTxs/{address}/pending";
   }
   rpc UnsignedBatchTxs(UnsignedBatchTxsRequest)
       returns (UnsignedBatchTxsResponse) {
     // option (google.api.http).get = "/gravity/v1/batches/{address}/pending";
   }
-  rpc UnsignedContractCallTxs(
-      UnsignedContractCallTxsRequest)
+  rpc UnsignedContractCallTxs(UnsignedContractCallTxsRequest)
       returns (UnsignedContractCallTxsResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/ContractCallTxs/{address}/pending";
+    // "/gravity/v1/ContractCallTxs/{address}/pending";
   }
 
   rpc LastSubmittedEthereumEvent(LastSubmittedEthereumEventRequest)
       returns (LastSubmittedEthereumEventResponse) {
-    // option (google.api.http).get = "/gravity/v1/oracle/event_nonce/{address}";
+    // option (google.api.http).get =
+    // "/gravity/v1/oracle/event_nonce/{address}";
   }
 
   // Queries the fees for all pending batches, results are returned in sdk.Coin
@@ -101,12 +103,12 @@ service Query {
   // Query for info about denoms tracked by gravity
   rpc ERC20ToDenom(ERC20ToDenomRequest) returns (ERC20ToDenomResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/cosmos_originated/erc20_to_denom";
+    // "/gravity/v1/cosmos_originated/erc20_to_denom";
   }
   // Query for info about denoms tracked by gravity
   rpc DenomToERC20(DenomToERC20Request) returns (DenomToERC20Response) {
     // option (google.api.http).get =
-        // "/gravity/v1/cosmos_originated/denom_to_erc20";
+    // "/gravity/v1/cosmos_originated/denom_to_erc20";
   }
   // Query for batch send to ethereums
   rpc BatchedSendToEthereums(BatchedSendToEthereumsRequest)
@@ -123,17 +125,22 @@ service Query {
   rpc DelegateKeysByValidator(DelegateKeysByValidatorRequest)
       returns (DelegateKeysByValidatorResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/delegate_keys/validator/{validator}";
+    // "/gravity/v1/delegate_keys/validator/{validator}";
   }
   rpc DelegateKeysByEthereumSigner(DelegateKeysByEthereumSignerRequest)
       returns (DelegateKeysByEthereumSignerResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/delegate_keys/ethereum/{ethereum_signer}";
+    // "/gravity/v1/delegate_keys/ethereum/{ethereum_signer}";
   }
   rpc DelegateKeysByOrchestrator(DelegateKeysByOrchestratorRequest)
       returns (DelegateKeysByOrchestratorResponse) {
     // option (google.api.http).get =
-        // "/gravity/v1/delegate_keys/orchestrator/{orchestrator}";
+    // "/gravity/v1/delegate_keys/orchestrator/{orchestrator}";
+  }
+
+  rpc DelegateKeys(DelegateKeysRequest) returns (DelegateKeysResponse) {
+    // option (google.api.http).get =
+    // "/gravity/v1/delegate_keys";
   }
 }
 
@@ -142,9 +149,7 @@ message ParamsRequest {}
 message ParamsResponse { Params params = 1 [ (gogoproto.nullable) = false ]; }
 
 //  rpc SignerSetTx
-message SignerSetTxRequest {
-  uint64 signer_set_nonce = 1;
-}
+message SignerSetTxRequest { uint64 signer_set_nonce = 1; }
 message LatestSignerSetTxRequest {}
 message SignerSetTxResponse { SignerSetTx signer_set = 1; }
 
@@ -163,9 +168,7 @@ message ContractCallTxRequest {
 message ContractCallTxResponse { ContractCallTx logic_call = 1; }
 
 // rpc SignerSetTxConfirmations
-message SignerSetTxConfirmationsRequest {
-  uint64 signer_set_nonce = 1;
-}
+message SignerSetTxConfirmationsRequest { uint64 signer_set_nonce = 1; }
 message SignerSetTxConfirmationsResponse {
   repeated SignerSetTxConfirmation signatures = 1;
 }
@@ -207,9 +210,7 @@ message UnsignedSignerSetTxsRequest {
   // orchestartor address or the cooresponding validator address
   string address = 1;
 }
-message UnsignedSignerSetTxsResponse {
-  repeated SignerSetTx signer_sets = 1;
-}
+message UnsignedSignerSetTxsResponse { repeated SignerSetTx signer_sets = 1; }
 
 message UnsignedBatchTxsRequest {
   // NOTE: this is an sdk.AccAddress and can represent either the
@@ -223,9 +224,7 @@ message UnsignedBatchTxsResponse {
 
 //  rpc UnsignedContractCallTxs
 message UnsignedContractCallTxsRequest { string address = 1; }
-message UnsignedContractCallTxsResponse {
-  repeated ContractCallTx calls = 1;
-}
+message UnsignedContractCallTxsResponse { repeated ContractCallTx calls = 1; }
 
 message BatchTxFeesRequest {}
 message BatchTxFeesResponse {
@@ -284,6 +283,9 @@ message DelegateKeysByOrchestratorResponse {
   string ethereum_signer = 2;
 }
 
+message DelegateKeysRequest {}
+message DelegateKeysResponse { repeated MsgDelegateKeys delegate_keys = 1; }
+
 // NOTE: if there is no sender address, return all
 message BatchedSendToEthereumsRequest {
   string sender_address = 1;

module/x/gravity/client/cli/query.go

@@ -41,6 +41,7 @@ func GetQueryCmd() *cobra.Command {
 		CmdDelegateKeysByValidator(),
 		CmdDelegateKeysByEthereumSigner(),
 		CmdDelegateKeysByOrchestrator(),
+		CmdDelegateKeys(),
 	)
 
 	return gravityQueryCmd
@@ -643,9 +644,7 @@ func CmdDelegateKeysByValidator() *cobra.Command {
 				return err
 			}
 
-			var ( // args
-				validatorAddress string // TODO(levi) init and validate from args[0]
-			)
+			validatorAddress := args[0]
 
 			req := types.DelegateKeysByValidatorRequest{
 				ValidatorAddress: validatorAddress,
@@ -675,9 +674,7 @@ func CmdDelegateKeysByEthereumSigner() *cobra.Command {
 				return err
 			}
 
-			var ( // args
-				ethereumSigner string // TODO(levi) init and validate from args[0]
-			)
+			ethereumSigner := args[0] // TODO(levi) init and validate from args[0]
 
 			req := types.DelegateKeysByEthereumSignerRequest{
 				EthereumSigner: ethereumSigner,
@@ -707,9 +704,7 @@ func CmdDelegateKeysByOrchestrator() *cobra.Command {
 				return err
 			}
 
-			var ( // args
-				orcAddr string // TODO(levi) init and validate from args[0]
-			)
+			orcAddr := args[0]
 
 			req := types.DelegateKeysByOrchestratorRequest{
 				OrchestratorAddress: orcAddr,
@@ -728,6 +723,34 @@ func CmdDelegateKeysByOrchestrator() *cobra.Command {
 	return cmd
 }
 
+
+func CmdDelegateKeys() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "all-delegate-keys",
+		Args:  cobra.NoArgs,
+		Short: "", // TODO(levi) provide short description
+		RunE: func(cmd *cobra.Command, args []string) error {
+			clientCtx, queryClient, err := newContextAndQueryClient(cmd)
+			if err != nil {
+				return err
+			}
+
+			req := types.DelegateKeysRequest{}
+
+			res, err := queryClient.DelegateKeys(cmd.Context(), &req)
+			if err != nil {
+				return err
+			}
+
+			return clientCtx.PrintProto(res)
+		},
+	}
+
+	flags.AddQueryFlagsToCmd(cmd)
+	return cmd
+}
+
+
 func newContextAndQueryClient(cmd *cobra.Command) (client.Context, types.QueryClient, error) {
 	clientCtx, err := client.GetClientQueryContext(cmd)
 	if err != nil {

module/x/gravity/keeper/grpc_query.go

@@ -401,3 +401,14 @@ func (k Keeper) DelegateKeysByOrchestrator(c context.Context, req *types.Delegat
 	}
 	return res, nil
 }
+
+func (k Keeper) DelegateKeys(c context.Context, req *types.DelegateKeysRequest) (*types.DelegateKeysResponse, error) {
+	ctx := sdk.UnwrapSDKContext(c)
+	delegateKeys := k.getDelegateKeys(ctx)
+
+	res := &types.DelegateKeysResponse{
+		DelegateKeys: delegateKeys,
+	}
+	return res, nil
+}
+