test: add end-to-end testing framework (#5435)

Partial fix for #5291. For details, see [README.md](https://github.com/tendermint/tendermint/blob/erik/e2e-tests/test/e2e/README.md) and [RFC-001](https://github.com/tendermint/tendermint/blob/master/docs/rfc/rfc-001-end-to-end-testing.md).

This only includes a single test case under `test/e2e/tests/`, as a proof of concept - additional test cases will be submitted separately. A randomized testnet generator will also be submitted separately, there a currently just a handful of static testnets under `test/e2e/networks/`. This will eventually replace the current P2P tests and run in CI.

.dockerignore

@@ -0,0 +1,5 @@
+build
+test/e2e/build
+test/e2e/networks
+test/logs
+test/p2p/data

.gitignore

@@ -10,6 +10,8 @@ remote_dump
 .revision
 vendor
 .vagrant
+test/e2e/build
+test/e2e/networks/*/
 test/p2p/data/
 test/logs
 coverage.txt

go.mod

@@ -3,6 +3,7 @@ module github.com/tendermint/tendermint
 go 1.14
 
 require (
+	github.com/BurntSushi/toml v0.3.1
 	github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d
 	github.com/Workiva/go-datastructures v1.0.52
 	github.com/btcsuite/btcd v0.21.0-beta

privval/file.go

@@ -152,11 +152,8 @@ type FilePV struct {
 	LastSignState FilePVLastSignState
 }
 
-// GenFilePV generates a new validator with randomly generated private key
-// and sets the filePaths, but does not call Save().
-func GenFilePV(keyFilePath, stateFilePath string) *FilePV {
-	privKey := ed25519.GenPrivKey()
-
+// NewFilePV generates a new validator from the given key and paths.
+func NewFilePV(privKey crypto.PrivKey, keyFilePath, stateFilePath string) *FilePV {
 	return &FilePV{
 		Key: FilePVKey{
 			Address:  privKey.PubKey().Address(),
@@ -171,6 +168,12 @@ func GenFilePV(keyFilePath, stateFilePath string) *FilePV {
 	}
 }
 
+// GenFilePV generates a new validator with randomly generated private key
+// and sets the filePaths, but does not call Save().
+func GenFilePV(keyFilePath, stateFilePath string) *FilePV {
+	return NewFilePV(ed25519.GenPrivKey(), keyFilePath, stateFilePath)
+}
+
 // LoadFilePV loads a FilePV from the filePaths.  The FilePV handles double
 // signing prevention by persisting data to the stateFilePath.  If either file path
 // does not exist, the program will exit.

test/e2e/Makefile

@@ -0,0 +1,16 @@
+docker:
+	docker build --tag tendermint/e2e-node -f docker/Dockerfile ../..
+
+ci: runner
+	./build/runner -f networks/ci.toml
+
+# We need to build support for database backends into the app in
+# order to build a binary with a Tendermint node in it (for built-in
+# ABCI testing).
+app:
+	go build -o build/app -tags badgerdb,boltdb,cleveldb,rocksdb ./app
+
+runner:
+	go build -o build/runner ./runner
+
+.PHONY: app ci docker runner

test/e2e/README.md

@@ -0,0 +1,78 @@
+# End-to-End Tests
+
+Spins up and tests Tendermint networks in Docker Compose based on a testnet manifest. To run the CI testnet:
+
+```sh
+make docker
+make runner
+./build/runner -f networks/ci.toml
+```
+
+This creates and runs a testnet named `ci` under `networks/ci/` (determined by the manifest filename).
+
+## Testnet Manifests
+
+Testnets are specified as TOML manifests. For an example see [`networks/ci.toml`](networks/ci.toml), and for documentation see [`pkg/manifest.go`](pkg/manifest.go).
+
+## Test Stages
+
+The test runner has the following stages, which can also be executed explicitly by running `./build/runner -f <manifest> <stage>`:
+
+* `setup`: generates configuration files.
+
+* `start`: starts Docker containers.
+
+* `load`: generates a transaction load against the testnet nodes.
+
+* `perturb`: runs any requested perturbations (e.g. node restarts or network disconnects).
+
+* `wait`: waits for a few blocks to be produced, and for all nodes to catch up to it.
+
+* `test`: runs test cases in `tests/` against all nodes in a running testnet.
+
+* `stop`: stops Docker containers.
+
+* `cleanup`: removes configuration files and Docker containers/networks.
+
+* `logs`: outputs all node logs.
+
+## Tests
+
+Test cases are written as normal Go tests in `tests/`. They use a `testNode()` helper which executes each test as a parallel subtest for each node in the network.
+
+### Running Manual Tests
+
+To run tests manually, set the `E2E_MANIFEST` environment variable to the path of the testnet manifest (e.g. `networks/ci.toml`) and run them as normal, e.g.:
+
+```sh
+./build/runner -f networks/ci.toml start
+E2E_MANIFEST=networks/ci.toml go test -v ./tests/...
+```
+
+Optionally, `E2E_NODE` specifies the name of a single testnet node to test.
+
+These environment variables can also be specified in `tests/e2e_test.go` to run tests from an editor or IDE:
+
+```go
+func init() {
+	// This can be used to manually specify a testnet manifest and/or node to
+	// run tests against. The testnet must have been started by the runner first.
+	os.Setenv("E2E_MANIFEST", "networks/ci.toml")
+	os.Setenv("E2E_NODE", "validator01")
+}
+```
+
+### Debugging Failures
+
+If a command or test fails, the runner simply exits with an error message and non-zero status code. The testnet is left running with data in the testnet directory, and can be inspected with e.g. `docker ps`, `docker logs`, or `./build/runner -f <manifest> logs` or `tail`. To shut down and remove the testnet, run `./build/runner -f <manifest> cleanup`.
+
+## Enabling IPv6
+
+Docker does not enable IPv6 by default. To do so, enter the following in `daemon.json` (or in the Docker for Mac UI under Preferences → Docker Engine):
+
+```json
+{
+  "ipv6": true,
+  "fixed-cidr-v6": "2001:db8:1::/64"
+}
+```

test/e2e/app/app.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/tendermint/tendermint/abci/example/code"
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/version"
+)
+
+// Application is an ABCI application for use by end-to-end tests. It is a
+// simple key/value store for strings, storing data in memory and persisting
+// to disk as JSON, taking state sync snapshots if requested.
+type Application struct {
+	abci.BaseApplication
+	logger          log.Logger
+	state           *State
+	snapshots       *SnapshotStore
+	cfg             *Config
+	restoreSnapshot *abci.Snapshot
+	restoreChunks   [][]byte
+}
+
+// NewApplication creates the application.
+func NewApplication(cfg *Config) (*Application, error) {
+	state, err := NewState(filepath.Join(cfg.Dir, "state.json"), cfg.PersistInterval)
+	if err != nil {
+		return nil, err
+	}
+	snapshots, err := NewSnapshotStore(filepath.Join(cfg.Dir, "snapshots"))
+	if err != nil {
+		return nil, err
+	}
+	return &Application{
+		logger:    log.NewTMLogger(log.NewSyncWriter(os.Stdout)),
+		state:     state,
+		snapshots: snapshots,
+		cfg:       cfg,
+	}, nil
+}
+
+// Info implements ABCI.
+func (app *Application) Info(req abci.RequestInfo) abci.ResponseInfo {
+	return abci.ResponseInfo{
+		Version:          version.ABCIVersion,
+		AppVersion:       1,
+		LastBlockHeight:  int64(app.state.Height),
+		LastBlockAppHash: app.state.Hash,
+	}
+}
+
+// Info implements ABCI.
+func (app *Application) InitChain(req abci.RequestInitChain) abci.ResponseInitChain {
+	var err error
+	app.state.initialHeight = uint64(req.InitialHeight)
+	if len(req.AppStateBytes) > 0 {
+		err = app.state.Import(0, req.AppStateBytes)
+		if err != nil {
+			panic(err)
+		}
+	}
+	resp := abci.ResponseInitChain{
+		AppHash: app.state.Hash,
+	}
+	if resp.Validators, err = app.validatorUpdates(0); err != nil {
+		panic(err)
+	}
+	return resp
+}
+
+// CheckTx implements ABCI.
+func (app *Application) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx {
+	_, _, err := parseTx(req.Tx)
+	if err != nil {
+		return abci.ResponseCheckTx{
+			Code: code.CodeTypeEncodingError,
+			Log:  err.Error(),
+		}
+	}
+	return abci.ResponseCheckTx{Code: code.CodeTypeOK, GasWanted: 1}
+}
+
+// DeliverTx implements ABCI.
+func (app *Application) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
+	key, value, err := parseTx(req.Tx)
+	if err != nil {
+		panic(err) // shouldn't happen since we verified it in CheckTx
+	}
+	app.state.Set(key, value)
+	return abci.ResponseDeliverTx{Code: code.CodeTypeOK}
+}
+
+// EndBlock implements ABCI.
+func (app *Application) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
+	var err error
+	resp := abci.ResponseEndBlock{}
+	if resp.ValidatorUpdates, err = app.validatorUpdates(uint64(req.Height)); err != nil {
+		panic(err)
+	}
+	return resp
+}
+
+// Commit implements ABCI.
+func (app *Application) Commit() abci.ResponseCommit {
+	height, hash, err := app.state.Commit()
+	if err != nil {
+		panic(err)
+	}
+	if app.cfg.SnapshotInterval > 0 && height%app.cfg.SnapshotInterval == 0 {
+		snapshot, err := app.snapshots.Create(app.state)
+		if err != nil {
+			panic(err)
+		}
+		logger.Info("Created state sync snapshot", "height", snapshot.Height)
+	}
+	retainHeight := int64(0)
+	if app.cfg.RetainBlocks > 0 {
+		retainHeight = int64(height - app.cfg.RetainBlocks + 1)
+	}
+	return abci.ResponseCommit{
+		Data:         hash,
+		RetainHeight: retainHeight,
+	}
+}
+
+// Query implements ABCI.
+func (app *Application) Query(req abci.RequestQuery) abci.ResponseQuery {
+	return abci.ResponseQuery{
+		Height: int64(app.state.Height),
+		Key:    req.Data,
+		Value:  []byte(app.state.Get(string(req.Data))),
+	}
+}
+
+// ListSnapshots implements ABCI.
+func (app *Application) ListSnapshots(req abci.RequestListSnapshots) abci.ResponseListSnapshots {
+	snapshots, err := app.snapshots.List()
+	if err != nil {
+		panic(err)
+	}
+	return abci.ResponseListSnapshots{Snapshots: snapshots}
+}
+
+// LoadSnapshotChunk implements ABCI.
+func (app *Application) LoadSnapshotChunk(req abci.RequestLoadSnapshotChunk) abci.ResponseLoadSnapshotChunk {
+	chunk, err := app.snapshots.LoadChunk(req.Height, req.Format, req.Chunk)
+	if err != nil {
+		panic(err)
+	}
+	return abci.ResponseLoadSnapshotChunk{Chunk: chunk}
+}
+
+// OfferSnapshot implements ABCI.
+func (app *Application) OfferSnapshot(req abci.RequestOfferSnapshot) abci.ResponseOfferSnapshot {
+	if app.restoreSnapshot != nil {
+		panic("A snapshot is already being restored")
+	}
+	app.restoreSnapshot = req.Snapshot
+	app.restoreChunks = [][]byte{}
+	return abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}
+}
+
+// ApplySnapshotChunk implements ABCI.
+func (app *Application) ApplySnapshotChunk(req abci.RequestApplySnapshotChunk) abci.ResponseApplySnapshotChunk {
+	if app.restoreSnapshot == nil {
+		panic("No restore in progress")
+	}
+	app.restoreChunks = append(app.restoreChunks, req.Chunk)
+	if len(app.restoreChunks) == int(app.restoreSnapshot.Chunks) {
+		bz := []byte{}
+		for _, chunk := range app.restoreChunks {
+			bz = append(bz, chunk...)
+		}
+		err := app.state.Import(app.restoreSnapshot.Height, bz)
+		if err != nil {
+			panic(err)
+		}
+		app.restoreSnapshot = nil
+		app.restoreChunks = nil
+	}
+	return abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}
+}
+
+// validatorUpdates generates a validator set update.
+func (app *Application) validatorUpdates(height uint64) (abci.ValidatorUpdates, error) {
+	updates := app.cfg.ValidatorUpdates[fmt.Sprintf("%v", height)]
+	if len(updates) == 0 {
+		return nil, nil
+	}
+	valUpdates := abci.ValidatorUpdates{}
+	for keyString, power := range updates {
+		keyBytes, err := base64.StdEncoding.DecodeString(keyString)
+		if err != nil {
+			return nil, fmt.Errorf("invalid base64 pubkey value %q: %w", keyString, err)
+		}
+		valUpdates = append(valUpdates, abci.Ed25519ValidatorUpdate(keyBytes, int64(power)))
+	}
+	return valUpdates, nil
+}
+
+// parseTx parses a tx in 'key=value' format into a key and value.
+func parseTx(tx []byte) (string, string, error) {
+	parts := bytes.Split(tx, []byte("="))
+	if len(parts) != 2 {
+		return "", "", fmt.Errorf("invalid tx format: %q", string(tx))
+	}
+	if len(parts[0]) == 0 {
+		return "", "", errors.New("key cannot be empty")
+	}
+	return string(parts[0]), string(parts[1]), nil
+}