From 2cc95e5708fb2214478e08473cd2cde8510cee6b Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Wed, 15 Jan 2025 11:38:31 +0100 Subject: [PATCH 1/7] feat(docker/network): harden docker networking This PR exposes unexposed ports (e.g. if the image doesn't specify a port). Also starts using explicit protocol in the PortSet (TCP). --- core/provider/docker/network.go | 6 ++++-- core/provider/docker/provider.go | 37 +++++++++++++++++++++----------- core/provider/docker/task.go | 8 +------ core/provider/docker/util.go | 3 ++- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/core/provider/docker/network.go b/core/provider/docker/network.go index 70a876e..f18d42c 100644 --- a/core/provider/docker/network.go +++ b/core/provider/docker/network.go @@ -95,8 +95,6 @@ func (p *Provider) openListenerOnFreePort() (*net.TCPListener, error) { return nil, err } - p.networkMu.Lock() - defer p.networkMu.Unlock() l, err := net.ListenTCP("tcp", addr) if err != nil { return nil, err @@ -110,6 +108,10 @@ func (p *Provider) openListenerOnFreePort() (*net.TCPListener, error) { // This allows multiple nextAvailablePort calls to find multiple available ports // before closing them so they are available for the PortBinding. func (p *Provider) nextAvailablePort() (nat.PortBinding, *net.TCPListener, error) { + // TODO: add listeners to state + p.networkMu.Lock() + defer p.networkMu.Unlock() + l, err := p.openListenerOnFreePort() if err != nil { if l != nil { diff --git a/core/provider/docker/provider.go b/core/provider/docker/provider.go index f4c0969..0cbd43b 100644 --- a/core/provider/docker/provider.go +++ b/core/provider/docker/provider.go @@ -25,10 +25,10 @@ type ProviderState struct { Name string `json:"name"` - NetworkID string `json:"network_id"` - NetworkName string `json:"network_name"` - NetworkCIDR string `json:"network_cidr"` - AllocatedIPs []string `json:"allocated_ips"` + NetworkID string `json:"network_id"` + NetworkName string `json:"network_name"` + NetworkCIDR string `json:"network_cidr"` + NetworkGateway string `json:"network_gateway"` BuilderImageName string `json:"builder_image_name"` } @@ -86,9 +86,14 @@ func CreateProvider(ctx context.Context, logger *zap.Logger, providerName string } dockerProvider.state.NetworkCIDR = cidrMask.String() + dockerProvider.state.NetworkGateway = network.IPAM.Config[0].Gateway dockerProvider.dockerNetworkAllocator, err = ipallocator.NewCIDRRange(cidrMask) + if err := dockerProvider.dockerNetworkAllocator.Allocate(net.ParseIP(network.IPAM.Config[0].Gateway)); err != nil { + return nil, fmt.Errorf("failed to allocate gateway ip: %w", err) + } + if err != nil { return nil, err } @@ -96,7 +101,7 @@ func CreateProvider(ctx context.Context, logger *zap.Logger, providerName string return dockerProvider, nil } -func RestoreProvider(ctx context.Context, state []byte) (*Provider, error) { +func RestoreProvider(ctx context.Context, logger *zap.Logger, state []byte) (*Provider, error) { var providerState ProviderState err := json.Unmarshal(state, &providerState) @@ -106,7 +111,8 @@ func RestoreProvider(ctx context.Context, state []byte) (*Provider, error) { } dockerProvider := &Provider{ - state: &providerState, + state: &providerState, + logger: logger, } dockerClient, err := client.NewClientWithOpts() @@ -133,8 +139,12 @@ func RestoreProvider(ctx context.Context, state []byte) (*Provider, error) { return nil, fmt.Errorf("failed to create ip allocator from state: %w", err) } - for _, ip := range providerState.AllocatedIPs { - if err := dockerProvider.dockerNetworkAllocator.Allocate(net.ParseIP(ip)); err != nil { + if err := dockerProvider.dockerNetworkAllocator.Allocate(net.ParseIP(providerState.NetworkGateway)); err != nil { + return nil, fmt.Errorf("failed to allocate gateway ip: %w", err) + } + + for _, task := range providerState.TaskStates { + if err := dockerProvider.dockerNetworkAllocator.Allocate(net.ParseIP(task.IpAddress)); err != nil { return nil, fmt.Errorf("failed to restore ip allocator state: %w", err) } } @@ -223,12 +233,12 @@ func (p *Provider) CreateTask(ctx context.Context, definition provider.TaskDefin Labels: map[string]string{ providerLabelName: p.state.Name, }, - Env: convertEnvMapToList(definition.Environment), + Env: convertEnvMapToList(definition.Environment), + ExposedPorts: portSet, }, &container.HostConfig{ - Mounts: mounts, - PortBindings: portBindings, - PublishAllPorts: true, - NetworkMode: container.NetworkMode(p.state.NetworkName), + Mounts: mounts, + PortBindings: portBindings, + NetworkMode: container.NetworkMode(p.state.NetworkName), }, &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ p.state.NetworkName: { @@ -245,6 +255,7 @@ func (p *Provider) CreateTask(ctx context.Context, definition provider.TaskDefin taskState.Id = createdContainer.ID taskState.Status = provider.TASK_STOPPED + taskState.IpAddress = ip p.stateMu.Lock() defer p.stateMu.Unlock() diff --git a/core/provider/docker/task.go b/core/provider/docker/task.go index 29b9682..a6f6647 100644 --- a/core/provider/docker/task.go +++ b/core/provider/docker/task.go @@ -106,19 +106,13 @@ func (t *Task) GetExternalAddress(ctx context.Context, port string) (string, err return "", fmt.Errorf("failed to inspect container: %w", err) } - ip, err := t.GetIP(ctx) - - if err != nil { - return "", fmt.Errorf("failed to get IP: %w", err) - } - portBindings, ok := dockerContainer.NetworkSettings.Ports[nat.Port(fmt.Sprintf("%s/tcp", port))] if !ok || len(portBindings) == 0 { return "", fmt.Errorf("port %s not found", port) } - return fmt.Sprintf("%s:%s", ip, portBindings[0].HostPort), nil + return fmt.Sprintf("0.0.0.0:%s", portBindings[0].HostPort), nil } func (t *Task) GetIP(ctx context.Context) (string, error) { diff --git a/core/provider/docker/util.go b/core/provider/docker/util.go index 353c2cf..3877e25 100644 --- a/core/provider/docker/util.go +++ b/core/provider/docker/util.go @@ -1,6 +1,7 @@ package docker import ( + "fmt" "github.com/docker/go-connections/nat" "github.com/skip-mev/petri/core/v2/provider" @@ -10,7 +11,7 @@ func convertTaskDefinitionPortsToPortSet(definition provider.TaskDefinition) nat bindings := nat.PortSet{} for _, port := range definition.Ports { - bindings[nat.Port(port)] = struct{}{} + bindings[nat.Port(fmt.Sprintf("%s/tcp", port))] = struct{}{} } return bindings From f1faed99b2a879c32604b0d71af679dbc19e9322 Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Wed, 15 Jan 2025 11:42:31 +0100 Subject: [PATCH 2/7] fix: lint nit --- core/provider/docker/provider_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/provider/docker/provider_test.go b/core/provider/docker/provider_test.go index 2adbd49..fe92245 100644 --- a/core/provider/docker/provider_test.go +++ b/core/provider/docker/provider_test.go @@ -92,7 +92,7 @@ func TestRestoreProvider(t *testing.T) { serialized, err := p1.SerializeProvider(ctx) require.NoError(t, err) - p2, err := docker.RestoreProvider(ctx, serialized) + p2, err := docker.RestoreProvider(ctx, logger, serialized) require.NoError(t, err) state2 := p2.GetState() @@ -265,7 +265,7 @@ func TestProviderSerialization(t *testing.T) { serialized, err := p1.SerializeProvider(ctx) require.NoError(t, err) - p2, err := docker.RestoreProvider(ctx, serialized) + p2, err := docker.RestoreProvider(ctx, logger, serialized) require.NoError(t, err) state2 := p2.GetState() From 8515f8d05d99e2941b408b33b3021c86f16bf56a Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Sat, 4 Jan 2025 01:06:02 +0200 Subject: [PATCH 3/7] feat: rewrite the docker provider --- core/provider/docker/task.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/provider/docker/task.go b/core/provider/docker/task.go index a6f6647..657ff3f 100644 --- a/core/provider/docker/task.go +++ b/core/provider/docker/task.go @@ -20,7 +20,6 @@ type TaskState struct { Volume *VolumeState `json:"volumes"` Definition provider.TaskDefinition `json:"definition"` Status provider.TaskStatus `json:"status"` - IpAddress string `json:"ip_address"` } type VolumeState struct { From e65ee94711e013cfadbd2f113323fb33a146d5c2 Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Fri, 10 Jan 2025 11:34:51 +0100 Subject: [PATCH 4/7] feat: add tests and conform to new interfaces --- core/provider/docker/task.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/provider/docker/task.go b/core/provider/docker/task.go index 657ff3f..9f7ca0f 100644 --- a/core/provider/docker/task.go +++ b/core/provider/docker/task.go @@ -20,6 +20,7 @@ type TaskState struct { Volume *VolumeState `json:"volumes"` Definition provider.TaskDefinition `json:"definition"` Status provider.TaskStatus `json:"status"` + IpAddress string `json:"ip_address"` } type VolumeState struct { From bd08ed68325bb795ae61b4ab95fff32a6c40782e Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Wed, 15 Jan 2025 11:41:35 +0100 Subject: [PATCH 5/7] feat(chain): add chain serialization --- core/types/chain.go | 22 ++--- core/types/node.go | 13 ++- cosmos/chain/chain.go | 172 ++++++++++++++++++++++++++--------- cosmos/chain/chain_test.go | 119 +++++++++++++++++++++--- cosmos/chain/chain_wallet.go | 10 +- cosmos/node/config.go | 2 +- cosmos/node/genesis.go | 8 +- cosmos/node/init.go | 2 +- cosmos/node/keys.go | 6 +- cosmos/node/node.go | 83 ++++++++++++++--- cosmos/node/node_test.go | 45 ++++++++- 11 files changed, 384 insertions(+), 98 deletions(-) diff --git a/core/types/chain.go b/core/types/chain.go index 1b76b3c..649d4d8 100644 --- a/core/types/chain.go +++ b/core/types/chain.go @@ -16,7 +16,7 @@ type GenesisModifier func([]byte) ([]byte, error) // ChainI is an interface for a logical chain type ChainI interface { - Init(context.Context) error + Init(context.Context, ChainOptions) error Teardown(context.Context) error GetConfig() ChainConfig @@ -32,6 +32,16 @@ type ChainI interface { Height(context.Context) (uint64, error) WaitForBlocks(ctx context.Context, delta uint64) error WaitForHeight(ctx context.Context, desiredHeight uint64) error + + Serialize(ctx context.Context, p provider.ProviderI) ([]byte, error) +} + +type ChainOptions struct { + ModifyGenesis GenesisModifier // ModifyGenesis is a function that modifies the genesis bytes of the chain + NodeOptions NodeOptions // NodeOptions is the options for creating a node + NodeCreator NodeCreator // NodeCreator is a function that creates a node + + WalletConfig WalletConfig // WalletConfig is the default configuration of a chain's wallet } // ChainConfig is the configuration structure for a logical chain. @@ -55,14 +65,8 @@ type ChainConfig struct { CoinType string // CoinType is the coin type of the chain (e.g. 118) ChainId string // ChainId is the chain ID of the chain - ModifyGenesis GenesisModifier // ModifyGenesis is a function that modifies the genesis bytes of the chain - - WalletConfig WalletConfig // WalletConfig is the default configuration of a chain's wallet - UseGenesisSubCommand bool // UseGenesisSubCommand is a flag that indicates whether to use the 'genesis' subcommand to initialize the chain. Set to true if Cosmos SDK >v0.50 - NodeCreator NodeCreator // NodeCreator is a function that creates a node - NodeDefinitionModifier NodeDefinitionModifier // NodeDefinitionModifier is a function that modifies a node's definition // number of tokens to allocate per account in the genesis state (unscaled). This value defaults to 10_000_000 if not set. // if not set. GenesisDelegation *big.Int @@ -121,9 +125,5 @@ func (c *ChainConfig) ValidateBasic() error { return fmt.Errorf("chain ID cannot be empty") } - if c.NodeCreator == nil { - return fmt.Errorf("node creator cannot be nil") - } - return nil } diff --git a/core/types/node.go b/core/types/node.go index b18403d..5d4e50a 100644 --- a/core/types/node.go +++ b/core/types/node.go @@ -12,6 +12,11 @@ import ( "github.com/skip-mev/petri/core/v2/provider" ) +// NodeOptions is a struct that contains the options for creating a node +type NodeOptions struct { + NodeDefinitionModifier NodeDefinitionModifier // NodeDefinitionModifier is a function that modifies a node's definition +} + // NodeConfig is the configuration structure for a logical node. type NodeConfig struct { Name string // Name is the name of the node @@ -40,7 +45,10 @@ func (c NodeConfig) ValidateBasic() error { type NodeDefinitionModifier func(provider.TaskDefinition, NodeConfig) provider.TaskDefinition // NodeCreator is a type of function that given a NodeConfig creates a new logical node -type NodeCreator func(context.Context, *zap.Logger, provider.ProviderI, NodeConfig) (NodeI, error) +type NodeCreator func(context.Context, *zap.Logger, provider.ProviderI, NodeConfig, NodeOptions) (NodeI, error) + +// NodeRestorer is a type of function that given a NodeState restores a logical node +type NodeRestorer func(context.Context, *zap.Logger, []byte, provider.ProviderI) (NodeI, error) // NodeI represents an interface for a logical node that is running on a chain type NodeI interface { @@ -95,4 +103,7 @@ type NodeI interface { // GetIP returns the IP address of the node GetIP(context.Context) (string, error) + + // Serialize serializes the node + Serialize(context.Context, provider.ProviderI) ([]byte, error) } diff --git a/cosmos/chain/chain.go b/cosmos/chain/chain.go index 58d04bb..f9bb0de 100644 --- a/cosmos/chain/chain.go +++ b/cosmos/chain/chain.go @@ -2,6 +2,7 @@ package chain import ( "context" + "encoding/json" "fmt" "math" "strings" @@ -20,9 +21,19 @@ import ( petritypes "github.com/skip-mev/petri/core/v2/types" ) +type PackagedState struct { + State + ValidatorStates [][]byte + NodeStates [][]byte +} + +type State struct { + Config petritypes.ChainConfig +} + // Chain is a logical representation of a Cosmos-based blockchain type Chain struct { - Config petritypes.ChainConfig + State State logger *zap.Logger @@ -39,7 +50,7 @@ type Chain struct { var _ petritypes.ChainI = &Chain{} // CreateChain creates the Chain object and initializes the node tasks, their backing compute and the validator wallets -func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider.ProviderI, config petritypes.ChainConfig) (*Chain, error) { +func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider.ProviderI, config petritypes.ChainConfig, opts petritypes.ChainOptions) (*Chain, error) { if err := config.ValidateBasic(); err != nil { return nil, fmt.Errorf("failed to validate chain config: %w", err) } @@ -47,7 +58,10 @@ func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider var chain Chain chain.mu = sync.RWMutex{} - chain.Config = config + chain.State = State{ + Config: config, + } + chain.logger = logger.Named("chain").With(zap.String("chain_id", config.ChainId)) chain.logger.Info("creating chain") @@ -66,12 +80,12 @@ func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider logger.Info("creating validator", zap.String("name", validatorName)) - validator, err := config.NodeCreator(ctx, logger, infraProvider, petritypes.NodeConfig{ + validator, err := opts.NodeCreator(ctx, logger, infraProvider, petritypes.NodeConfig{ Index: i, Name: validatorName, IsValidator: true, ChainConfig: config, - }) + }, opts.NodeOptions) if err != nil { return err } @@ -94,12 +108,12 @@ func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider logger.Info("creating node", zap.String("name", nodeName)) - node, err := config.NodeCreator(ctx, logger, infraProvider, petritypes.NodeConfig{ + node, err := opts.NodeCreator(ctx, logger, infraProvider, petritypes.NodeConfig{ Index: i, Name: nodeName, IsValidator: true, ChainConfig: config, - }) + }, opts.NodeOptions) if err != nil { return err } @@ -118,15 +132,42 @@ func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider chain.Nodes = nodes chain.Validators = validators - chain.Config = config chain.ValidatorWallets = make([]petritypes.WalletI, config.NumValidators) return &chain, nil } -// GetConfig returns the Chain's configuration -func (c *Chain) GetConfig() petritypes.ChainConfig { - return c.Config +// RestoreChain restores a Chain object from a serialized state +func RestoreChain(ctx context.Context, logger *zap.Logger, infraProvider provider.ProviderI, state []byte, nodeRestore petritypes.NodeRestorer) (*Chain, error) { + var packagedState PackagedState + + if err := json.Unmarshal(state, &packagedState); err != nil { + return nil, err + } + + chain := Chain{ + State: packagedState.State, + } + + for _, vs := range packagedState.ValidatorStates { + v, err := nodeRestore(ctx, logger, vs, infraProvider) + if err != nil { + return nil, err + } + + chain.Validators = append(chain.Validators, v) + } + + for _, ns := range packagedState.NodeStates { + n, err := nodeRestore(ctx, logger, ns, infraProvider) + if err != nil { + return nil, err + } + + chain.Nodes = append(chain.Nodes, n) + } + + return &chain, nil } // Height returns the chain's height from the first available full node in the network @@ -135,12 +176,12 @@ func (c *Chain) Height(ctx context.Context) (uint64, error) { client, err := node.GetTMClient(ctx) - c.logger.Debug("fetching height from", zap.String("node", node.GetDefinition().Name), zap.String("ip", client.Remote())) - if err != nil { return 0, err } + c.logger.Debug("fetching height from", zap.String("node", node.GetDefinition().Name), zap.String("ip", client.Remote())) + status, err := client.Status(context.Background()) if err != nil { return 0, err @@ -151,18 +192,18 @@ func (c *Chain) Height(ctx context.Context) (uint64, error) { // Init initializes the chain. That consists of generating the genesis transactions, genesis file, wallets, // the distribution of configuration files and starting the network nodes up -func (c *Chain) Init(ctx context.Context) error { - decimalPow := int64(math.Pow10(int(c.Config.Decimals))) +func (c *Chain) Init(ctx context.Context, opts petritypes.ChainOptions) error { + decimalPow := int64(math.Pow10(int(c.GetConfig().Decimals))) genesisCoin := types.Coin{ Amount: sdkmath.NewIntFromBigInt(c.GetConfig().GetGenesisBalance()).MulRaw(decimalPow), - Denom: c.Config.Denom, + Denom: c.GetConfig().Denom, } c.logger.Info("creating genesis accounts", zap.String("coin", genesisCoin.String())) genesisSelfDelegation := types.Coin{ Amount: sdkmath.NewIntFromBigInt(c.GetConfig().GetGenesisDelegation()).MulRaw(decimalPow), - Denom: c.Config.Denom, + Denom: c.GetConfig().Denom, } c.logger.Info("creating genesis self-delegations", zap.String("coin", genesisSelfDelegation.String())) @@ -179,7 +220,7 @@ func (c *Chain) Init(ctx context.Context) error { return fmt.Errorf("error initializing home dir: %v", err) } - validatorWallet, err := v.CreateWallet(ctx, petritypes.ValidatorKeyName, c.Config.WalletConfig) + validatorWallet, err := v.CreateWallet(ctx, petritypes.ValidatorKeyName, opts.WalletConfig) if err != nil { return err } @@ -221,7 +262,7 @@ func (c *Chain) Init(ctx context.Context) error { } c.logger.Info("adding faucet genesis") - faucetWallet, err := c.BuildWallet(ctx, petritypes.FaucetAccountKeyName, "", c.Config.WalletConfig) + faucetWallet, err := c.BuildWallet(ctx, petritypes.FaucetAccountKeyName, "", opts.WalletConfig) if err != nil { return err } @@ -230,38 +271,28 @@ func (c *Chain) Init(ctx context.Context) error { firstValidator := c.Validators[0] + c.logger.Info("first validator name", zap.String("validator", firstValidator.GetDefinition().Name)) + if err := firstValidator.AddGenesisAccount(ctx, faucetWallet.FormattedAddress(), genesisAmounts); err != nil { return err } for i := 1; i < len(c.Validators); i++ { validatorN := c.Validators[i] - validatorWalletAddress := c.ValidatorWallets[i].FormattedAddress() - eg.Go(func() error { - bech32, err := validatorN.KeyBech32(ctx, petritypes.ValidatorKeyName, "acc") - if err != nil { - return err - } - - c.logger.Info("setting up validator keys", zap.String("validator", validatorN.GetDefinition().Name), zap.String("address", bech32)) - if err := firstValidator.AddGenesisAccount(ctx, bech32, genesisAmounts); err != nil { - return err - } - - if err := firstValidator.AddGenesisAccount(ctx, validatorWalletAddress, genesisAmounts); err != nil { - return err - } - if err := validatorN.CopyGenTx(ctx, firstValidator); err != nil { - return err - } + bech32, err := validatorN.KeyBech32(ctx, petritypes.ValidatorKeyName, "acc") + if err != nil { + return err + } - return nil - }) - } + c.logger.Info("setting up validator keys", zap.String("validator", validatorN.GetDefinition().Name), zap.String("address", bech32)) + if err := firstValidator.AddGenesisAccount(ctx, bech32, genesisAmounts); err != nil { + return fmt.Errorf("failed to add validator %s genesis account: %w", validatorN.GetDefinition().Name, err) + } - if err := eg.Wait(); err != nil { - return err + if err := validatorN.CopyGenTx(ctx, firstValidator); err != nil { + return fmt.Errorf("failed to copy gentx from %s: %w", validatorN.GetDefinition().Name, err) + } } if err := firstValidator.CollectGenTxs(ctx); err != nil { @@ -273,8 +304,9 @@ func (c *Chain) Init(ctx context.Context) error { return err } - if c.Config.ModifyGenesis != nil { - genbz, err = c.Config.ModifyGenesis(genbz) + if opts.ModifyGenesis != nil { + c.logger.Info("modifying genesis") + genbz, err = opts.ModifyGenesis(genbz) if err != nil { return err } @@ -354,7 +386,7 @@ func (c *Chain) Init(ctx context.Context) error { // Teardown destroys all resources related to a chain and its' nodes func (c *Chain) Teardown(ctx context.Context) error { - c.logger.Info("tearing down chain", zap.String("name", c.Config.ChainId)) + c.logger.Info("tearing down chain", zap.String("name", c.GetConfig().ChainId)) for _, v := range c.Validators { if err := v.Destroy(ctx); err != nil { @@ -413,6 +445,25 @@ func (c *Chain) GetFullNode() petritypes.NodeI { return c.Validators[0] } +func (c *Chain) WaitForStartup(ctx context.Context) error { + ticker := time.NewTicker(1 * time.Second) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + err := c.WaitForHeight(ctx, 1) + if err != nil { + c.logger.Error("error waiting for height", zap.Error(err)) + continue + } + ticker.Stop() + return nil + } + } +} + // WaitForBlocks blocks until the chain increases in block height by delta func (c *Chain) WaitForBlocks(ctx context.Context, delta uint64) error { c.logger.Info("waiting for blocks", zap.Uint64("delta", delta)) @@ -475,3 +526,34 @@ func (c *Chain) GetValidatorWallets() []petritypes.WalletI { func (c *Chain) GetFaucetWallet() petritypes.WalletI { return c.FaucetWallet } + +// GetConfig is the configuration structure for a logical chain. +func (c *Chain) GetConfig() petritypes.ChainConfig { + return c.State.Config +} + +// Serialize returns the serialized representation of the chain +func (c *Chain) Serialize(ctx context.Context, p provider.ProviderI) ([]byte, error) { + state := PackagedState{ + State: c.State, + } + + for _, v := range c.Validators { + vs, err := v.Serialize(ctx, p) + if err != nil { + return nil, err + } + state.ValidatorStates = append(state.ValidatorStates, vs) + } + + for _, n := range c.Nodes { + ns, err := n.Serialize(ctx, p) + if err != nil { + return nil, err + } + + state.NodeStates = append(state.NodeStates, ns) + } + + return json.Marshal(state) +} diff --git a/cosmos/chain/chain_test.go b/cosmos/chain/chain_test.go index 7ffa053..a5e827e 100644 --- a/cosmos/chain/chain_test.go +++ b/cosmos/chain/chain_test.go @@ -2,6 +2,7 @@ package chain_test import ( "context" + "fmt" "github.com/cosmos/cosmos-sdk/crypto/hd" gonanoid "github.com/matoous/go-nanoid/v2" "github.com/skip-mev/petri/core/v2/provider" @@ -11,6 +12,8 @@ import ( "github.com/skip-mev/petri/cosmos/v2/node" "github.com/stretchr/testify/require" "go.uber.org/zap" + "io" + "net/http" "testing" "time" ) @@ -20,7 +23,7 @@ const idAlphabet = "abcdefghijklqmnoqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ12345678 var defaultChainConfig = types.ChainConfig{ Denom: "stake", Decimals: 6, - NumValidators: 1, + NumValidators: 4, NumNodes: 0, BinaryName: "/simd/simd", Image: provider.ImageDefinition{ @@ -28,11 +31,16 @@ var defaultChainConfig = types.ChainConfig{ UID: "1000", GID: "1000", }, - GasPrices: "0.0005stake", - Bech32Prefix: "cosmos", - HomeDir: "/gaia", - CoinType: "118", - ChainId: "stake-1", + GasPrices: "0.0005stake", + Bech32Prefix: "cosmos", + HomeDir: "/gaia", + CoinType: "118", + ChainId: "stake-1", + UseGenesisSubCommand: true, +} + +var defaultChainOptions = types.ChainOptions{ + NodeCreator: node.CreateNode, WalletConfig: types.WalletConfig{ SigningAlgorithm: string(hd.Secp256k1.Name()), Bech32Prefix: "cosmos", @@ -40,8 +48,6 @@ var defaultChainConfig = types.ChainConfig{ DerivationFn: hd.Secp256k1.Derive(), GenerationFn: hd.Secp256k1.Generate(), }, - UseGenesisSubCommand: true, - NodeCreator: node.CreateNode, } func TestChainLifecycle(t *testing.T) { @@ -55,11 +61,11 @@ func TestChainLifecycle(t *testing.T) { require.NoError(t, p.Teardown(ctx)) }(p, ctx) - c, err := chain.CreateChain(ctx, logger, p, defaultChainConfig) + c, err := chain.CreateChain(ctx, logger, p, defaultChainConfig, defaultChainOptions) require.NoError(t, err) - require.NoError(t, c.Init(ctx)) - require.Len(t, c.GetValidators(), 1) + require.NoError(t, c.Init(ctx, defaultChainOptions)) + require.Len(t, c.GetValidators(), 4) require.Len(t, c.GetNodes(), 0) time.Sleep(1 * time.Second) @@ -68,3 +74,94 @@ func TestChainLifecycle(t *testing.T) { require.NoError(t, c.Teardown(ctx)) } + +func TestChainSerialization(t *testing.T) { + ctx := context.Background() + logger, _ := zap.NewDevelopment() + providerName := gonanoid.MustGenerate(idAlphabet, 10) + + p, err := docker.CreateProvider(ctx, logger, providerName) + require.NoError(t, err) + + pState, err := p.SerializeProvider(ctx) + require.NoError(t, err) + + p2, err := docker.RestoreProvider(ctx, logger, pState) + require.NoError(t, err) + defer func(p provider.ProviderI, ctx context.Context) { + if !t.Failed() { + require.NoError(t, p.Teardown(ctx)) + } + }(p2, ctx) + + c, err := chain.CreateChain(ctx, logger, p2, defaultChainConfig, defaultChainOptions) + require.NoError(t, err) + + require.NoError(t, c.Init(ctx, defaultChainOptions)) + require.Len(t, c.GetValidators(), 4) + require.Len(t, c.GetNodes(), 0) + + require.NoError(t, c.WaitForStartup(ctx)) + + state, err := c.Serialize(ctx, p) + require.NoError(t, err) + + require.NotEmpty(t, state) + + c2, err := chain.RestoreChain(ctx, logger, p2, state, node.RestoreNode) + require.NoError(t, err) + + require.Equal(t, c.GetConfig(), c2.GetConfig()) + require.Equal(t, len(c.GetValidators()), len(c2.GetValidators())) + + if !t.Failed() { + require.NoError(t, c.Teardown(ctx)) + } +} + +func TestGenesisModifier(t *testing.T) { + ctx := context.Background() + logger, _ := zap.NewDevelopment() + providerName := gonanoid.MustGenerate(idAlphabet, 10) + chainName := gonanoid.MustGenerate(idAlphabet, 64) + + p, err := docker.CreateProvider(ctx, logger, providerName) + require.NoError(t, err) + defer func(p provider.ProviderI, ctx context.Context) { + require.NoError(t, p.Teardown(ctx)) + }(p, ctx) + + chainOpts := defaultChainOptions + chainOpts.ModifyGenesis = chain.ModifyGenesis([]chain.GenesisKV{ + { + Key: "app_state.gov.params.min_deposit.0.denom", + Value: chainName, + }, + }) + + c, err := chain.CreateChain(ctx, logger, p, defaultChainConfig, chainOpts) + require.NoError(t, err) + + require.NoError(t, c.Init(ctx, chainOpts)) + require.Len(t, c.GetValidators(), 4) + require.Len(t, c.GetNodes(), 0) + + time.Sleep(1 * time.Second) + + require.NoError(t, c.WaitForBlocks(ctx, 1)) + + cometIp, err := c.GetValidators()[0].GetExternalAddress(ctx, "26657") + require.NoError(t, err) + + resp, err := http.Get(fmt.Sprintf("http://%s/genesis", cometIp)) + require.NoError(t, err) + defer func(resp *http.Response) { + require.NoError(t, resp.Body.Close()) + }(resp) + + bz, err := io.ReadAll(resp.Body) + fmt.Println(string(bz)) + require.NoError(t, err) + + require.Contains(t, string(bz), chainName) +} diff --git a/cosmos/chain/chain_wallet.go b/cosmos/chain/chain_wallet.go index a2c6f27..3335224 100644 --- a/cosmos/chain/chain_wallet.go +++ b/cosmos/chain/chain_wallet.go @@ -15,11 +15,11 @@ import ( func (c *Chain) BuildWallet(ctx context.Context, keyName, mnemonic string, walletConfig types.WalletConfig) (types.WalletI, error) { // if mnemonic is empty, we just generate a wallet if mnemonic == "" { - return c.CreateWallet(ctx, keyName) + return c.CreateWallet(ctx, keyName, walletConfig) } if err := c.RecoverKey(ctx, keyName, mnemonic); err != nil { - return nil, fmt.Errorf("failed to recover key with name %q on chain %s: %w", keyName, c.Config.ChainId, err) + return nil, fmt.Errorf("failed to recover key with name %q on chain %s: %w", keyName, c.GetConfig().ChainId, err) } return wallet.NewWallet(keyName, mnemonic, walletConfig) @@ -31,8 +31,8 @@ func (c *Chain) RecoverKey(ctx context.Context, keyName, mnemonic string) error } // CreateWallet creates a wallet in the first available full node's keystore using a randomly generated mnemonic -func (c *Chain) CreateWallet(ctx context.Context, keyName string) (types.WalletI, error) { - return c.GetFullNode().CreateWallet(ctx, keyName, c.Config.WalletConfig) +func (c *Chain) CreateWallet(ctx context.Context, keyName string, config types.WalletConfig) (types.WalletI, error) { + return c.GetFullNode().CreateWallet(ctx, keyName, config) } // GetAddress returns a Bech32 formatted address for a given key in the first available full node's keystore @@ -42,5 +42,5 @@ func (c *Chain) GetAddress(ctx context.Context, keyName string) ([]byte, error) return nil, err } - return sdk.GetFromBech32(b32Addr, c.Config.Bech32Prefix) + return sdk.GetFromBech32(b32Addr, c.GetConfig().Bech32Prefix) } diff --git a/cosmos/node/config.go b/cosmos/node/config.go index 129e5cc..a14bf96 100644 --- a/cosmos/node/config.go +++ b/cosmos/node/config.go @@ -139,7 +139,7 @@ func (n *Node) ModifyTomlConfigFile( // SetDefaultConfigs will generate the default configs for CometBFT and the app, and write them to disk func (n *Node) SetDefaultConfigs(ctx context.Context) error { - appConfig := GenerateDefaultAppConfig(n.chainConfig) + appConfig := GenerateDefaultAppConfig(n.GetChainConfig()) consensusConfig := GenerateDefaultConsensusConfig() if err := n.ModifyTomlConfigFile( diff --git a/cosmos/node/genesis.go b/cosmos/node/genesis.go index 19ea5ed..f3fd31a 100644 --- a/cosmos/node/genesis.go +++ b/cosmos/node/genesis.go @@ -64,7 +64,7 @@ func (n *Node) AddGenesisAccount(ctx context.Context, address string, genesisAmo var command []string - if n.chainConfig.UseGenesisSubCommand { + if n.GetChainConfig().UseGenesisSubCommand { command = append(command, "genesis") } @@ -91,13 +91,13 @@ func (n *Node) GenerateGenTx(ctx context.Context, genesisSelfDelegation types.Co var command []string - if n.chainConfig.UseGenesisSubCommand { + if n.GetChainConfig().UseGenesisSubCommand { command = append(command, "genesis") } command = append(command, "gentx", petritypes.ValidatorKeyName, fmt.Sprintf("%s%s", genesisSelfDelegation.Amount.String(), genesisSelfDelegation.Denom), "--keyring-backend", keyring.BackendTest, - "--chain-id", n.chainConfig.ChainId) + "--chain-id", n.GetChainConfig().ChainId) command = n.BinCommand(command...) @@ -121,7 +121,7 @@ func (n *Node) CollectGenTxs(ctx context.Context) error { command := []string{} - if n.chainConfig.UseGenesisSubCommand { + if n.GetChainConfig().UseGenesisSubCommand { command = append(command, "genesis") } diff --git a/cosmos/node/init.go b/cosmos/node/init.go index 24857ac..cd5d997 100644 --- a/cosmos/node/init.go +++ b/cosmos/node/init.go @@ -11,7 +11,7 @@ import ( func (n *Node) InitHome(ctx context.Context) error { n.logger.Info("initializing home", zap.String("name", n.GetDefinition().Name)) - stdout, stderr, exitCode, err := n.RunCommand(ctx, n.BinCommand([]string{"init", n.GetDefinition().Name, "--chain-id", n.chainConfig.ChainId}...)) + stdout, stderr, exitCode, err := n.RunCommand(ctx, n.BinCommand([]string{"init", n.GetDefinition().Name, "--chain-id", n.GetChainConfig().ChainId}...)) n.logger.Debug("init home", zap.String("stdout", stdout), zap.String("stderr", stderr), zap.Int("exitCode", exitCode)) if err != nil { diff --git a/cosmos/node/keys.go b/cosmos/node/keys.go index d5a3032..3f47975 100644 --- a/cosmos/node/keys.go +++ b/cosmos/node/keys.go @@ -36,7 +36,7 @@ func (n *Node) RecoverKey(ctx context.Context, name, mnemonic string) error { command := []string{ "sh", "-c", - fmt.Sprintf(`echo %q | %s keys add %s --recover --keyring-backend %s --coin-type %s --home %s --output json`, mnemonic, n.chainConfig.BinaryName, name, keyring.BackendTest, n.chainConfig.CoinType, n.chainConfig.HomeDir), + fmt.Sprintf(`echo %q | %s keys add %s --recover --keyring-backend %s --coin-type %s --home %s --output json`, mnemonic, n.GetChainConfig().BinaryName, name, keyring.BackendTest, n.GetChainConfig().CoinType, n.GetChainConfig().HomeDir), } _, _, _, err := n.RunCommand(ctx, command) @@ -47,8 +47,8 @@ func (n *Node) RecoverKey(ctx context.Context, name, mnemonic string) error { // KeyBech32 returns the bech32 address of a key on the node using the app's binary func (n *Node) KeyBech32(ctx context.Context, name, bech string) (string, error) { command := []string{ - n.chainConfig.BinaryName, - "keys", "show", name, "-a", "--keyring-backend", keyring.BackendTest, "--home", n.chainConfig.HomeDir, + n.GetChainConfig().BinaryName, + "keys", "show", name, "-a", "--keyring-backend", keyring.BackendTest, "--home", n.GetChainConfig().HomeDir, } if bech != "" { diff --git a/cosmos/node/node.go b/cosmos/node/node.go index 72eaa8d..f3a6940 100644 --- a/cosmos/node/node.go +++ b/cosmos/node/node.go @@ -2,6 +2,7 @@ package node import ( "context" + "encoding/json" "fmt" "time" @@ -18,18 +19,28 @@ import ( petritypes "github.com/skip-mev/petri/core/v2/types" ) +type PackagedState struct { + State + TaskState []byte +} + +type State struct { + Config petritypes.NodeConfig + ChainConfig petritypes.ChainConfig +} + type Node struct { provider.TaskI - logger *zap.Logger - config petritypes.NodeConfig - chainConfig petritypes.ChainConfig + state State + logger *zap.Logger } var _ petritypes.NodeCreator = CreateNode +var _ petritypes.NodeRestorer = RestoreNode // CreateNode creates a new logical node and creates the underlying workload for it -func CreateNode(ctx context.Context, logger *zap.Logger, infraProvider provider.ProviderI, nodeConfig petritypes.NodeConfig) (petritypes.NodeI, error) { +func CreateNode(ctx context.Context, logger *zap.Logger, infraProvider provider.ProviderI, nodeConfig petritypes.NodeConfig, opts petritypes.NodeOptions) (petritypes.NodeI, error) { if err := nodeConfig.ValidateBasic(); err != nil { return nil, fmt.Errorf("failed to validate node config: %w", err) } @@ -38,8 +49,13 @@ func CreateNode(ctx context.Context, logger *zap.Logger, infraProvider provider. node.logger = logger.Named("node") chainConfig := nodeConfig.ChainConfig - node.chainConfig = nodeConfig.ChainConfig - node.config = nodeConfig + + nodeState := State{ + Config: nodeConfig, + ChainConfig: chainConfig, + } + + node.state = nodeState node.logger.Info("creating node", zap.String("name", nodeConfig.Name)) @@ -47,13 +63,13 @@ func CreateNode(ctx context.Context, logger *zap.Logger, infraProvider provider. Name: nodeConfig.Name, ContainerName: nodeConfig.Name, Image: chainConfig.Image, - Ports: []string{"9090", "26656", "26657", "26660", "80"}, + Ports: []string{"9090", "26656", "26657", "26660", "1317"}, Entrypoint: []string{chainConfig.BinaryName, "--home", chainConfig.HomeDir, "start"}, DataDir: chainConfig.HomeDir, } - if chainConfig.NodeDefinitionModifier != nil { - def = chainConfig.NodeDefinitionModifier(def, nodeConfig) + if opts.NodeDefinitionModifier != nil { + def = opts.NodeDefinitionModifier(def, nodeConfig) } task, err := infraProvider.CreateTask(ctx, def) @@ -66,11 +82,33 @@ func CreateNode(ctx context.Context, logger *zap.Logger, infraProvider provider. return &node, nil } +func RestoreNode(ctx context.Context, logger *zap.Logger, state []byte, p provider.ProviderI) (petritypes.NodeI, error) { + var packagedState PackagedState + if err := json.Unmarshal(state, &packagedState); err != nil { + return nil, fmt.Errorf("unmarshaling state: %w", err) + } + + node := Node{ + state: packagedState.State, + logger: logger.Named("node"), + } + + task, err := p.DeserializeTask(ctx, packagedState.TaskState) + + if err != nil { + return nil, err + } + + node.TaskI = task + + return &node, err +} + // GetTMClient returns a CometBFT HTTP client for the node func (n *Node) GetTMClient(ctx context.Context) (*rpchttp.HTTP, error) { addr, err := n.GetExternalAddress(ctx, "26657") if err != nil { - panic(err) + return nil, err } httpAddr := fmt.Sprintf("http://%s", addr) @@ -138,13 +176,32 @@ func (n *Node) NodeId(ctx context.Context) (string, error) { // BinCommand returns a command that can be used to run a binary on the node func (n *Node) BinCommand(command ...string) []string { - command = append([]string{n.chainConfig.BinaryName}, command...) + command = append([]string{n.GetChainConfig().BinaryName}, command...) return append(command, - "--home", n.chainConfig.HomeDir, + "--home", n.state.ChainConfig.HomeDir, ) } // GetConfig returns the node's config func (n *Node) GetConfig() petritypes.NodeConfig { - return n.config + return n.state.Config +} + +func (n *Node) GetChainConfig() petritypes.ChainConfig { + return n.state.ChainConfig +} + +func (n *Node) Serialize(ctx context.Context, p provider.ProviderI) ([]byte, error) { + taskState, err := p.SerializeTask(ctx, n.TaskI) + + if err != nil { + return nil, err + } + + state := PackagedState{ + State: n.state, + TaskState: taskState, + } + + return json.Marshal(state) } diff --git a/cosmos/node/node_test.go b/cosmos/node/node_test.go index 19e3e11..fa9f646 100644 --- a/cosmos/node/node_test.go +++ b/cosmos/node/node_test.go @@ -30,9 +30,7 @@ var defaultChainConfig = types.ChainConfig{ HomeDir: "/gaia", CoinType: "118", ChainId: "stake-1", - WalletConfig: types.WalletConfig{}, UseGenesisSubCommand: true, - NodeCreator: node.CreateNode, } func TestNodeLifecycle(t *testing.T) { @@ -50,7 +48,37 @@ func TestNodeLifecycle(t *testing.T) { Name: "test", Index: 0, ChainConfig: defaultChainConfig, - }) + }, types.NodeOptions{}) + require.NoError(t, err) + + defer func(n types.NodeI, ctx context.Context) { + require.NoError(t, n.Stop(ctx)) + }(n, ctx) + + status, err := n.GetStatus(ctx) + require.NoError(t, err) + require.Equal(t, provider.TASK_STOPPED, status) + + err = n.Start(ctx) + require.NoError(t, err) +} + +func TestNodeSerialization(t *testing.T) { + ctx := context.Background() + logger, _ := zap.NewDevelopment() + providerName := gonanoid.MustGenerate(idAlphabet, 10) + + p, err := docker.CreateProvider(ctx, logger, providerName) + require.NoError(t, err) + defer func(p provider.ProviderI, ctx context.Context) { + require.NoError(t, p.Teardown(ctx)) + }(p, ctx) + + n, err := node.CreateNode(ctx, logger, p, types.NodeConfig{ + Name: "test", + Index: 0, + ChainConfig: defaultChainConfig, + }, types.NodeOptions{}) require.NoError(t, err) defer func(n types.NodeI, ctx context.Context) { require.NoError(t, n.Stop(ctx)) @@ -62,4 +90,15 @@ func TestNodeLifecycle(t *testing.T) { err = n.Start(ctx) require.NoError(t, err) + + state, err := n.Serialize(ctx, p) + require.NoError(t, err) + require.NotEmpty(t, state) + + n2, err := node.RestoreNode(ctx, logger, state, p) + require.NoError(t, err) + + status, err = n2.GetStatus(ctx) + require.NoError(t, err) + require.Equal(t, provider.TASK_RUNNING, status) } From 77ff697aa14d576a4713e95ddd935696df357129 Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Fri, 17 Jan 2025 16:56:16 +0100 Subject: [PATCH 6/7] feat: validate chainopts --- core/types/chain.go | 12 ++++++++++++ core/types/wallet.go | 29 ++++++++++++++++++++++++++++- cosmos/chain/chain.go | 8 ++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/core/types/chain.go b/core/types/chain.go index 649d4d8..49e3b86 100644 --- a/core/types/chain.go +++ b/core/types/chain.go @@ -44,6 +44,18 @@ type ChainOptions struct { WalletConfig WalletConfig // WalletConfig is the default configuration of a chain's wallet } +func (o ChainOptions) ValidateBasic() error { + if err := o.WalletConfig.ValidateBasic(); err != nil { + return fmt.Errorf("wallet config is invalid: %w", err) + } + + if o.NodeCreator == nil { + return fmt.Errorf("node creator cannot be nil") + } + + return nil +} + // ChainConfig is the configuration structure for a logical chain. // It contains all the relevant details needed to create a Cosmos chain type ChainConfig struct { diff --git a/core/types/wallet.go b/core/types/wallet.go index 4de989e..062385e 100644 --- a/core/types/wallet.go +++ b/core/types/wallet.go @@ -1,6 +1,9 @@ package types -import "github.com/cosmos/cosmos-sdk/crypto/hd" +import ( + "fmt" + "github.com/cosmos/cosmos-sdk/crypto/hd" +) // WalletConfig is a configuration for a Cosmos SDK type wallet type WalletConfig struct { @@ -10,3 +13,27 @@ type WalletConfig struct { HDPath *hd.BIP44Params // HDPath is the default HD path to use for deriving keys SigningAlgorithm string // SigningAlgorithm is the default signing algorithm to use } + +func (c WalletConfig) ValidateBasic() error { + if c.DerivationFn == nil { + return fmt.Errorf("derivation function cannot be nil") + } + + if c.GenerationFn == nil { + return fmt.Errorf("generation function cannot be nil") + } + + if c.Bech32Prefix == "" { + return fmt.Errorf("bech32 prefix cannot be empty") + } + + if c.HDPath == nil { + return fmt.Errorf("HD path cannot be nil") + } + + if c.SigningAlgorithm == "" { + return fmt.Errorf("signing algorithm cannot be empty") + } + + return nil +} diff --git a/cosmos/chain/chain.go b/cosmos/chain/chain.go index f9bb0de..8b7ce5d 100644 --- a/cosmos/chain/chain.go +++ b/cosmos/chain/chain.go @@ -55,6 +55,10 @@ func CreateChain(ctx context.Context, logger *zap.Logger, infraProvider provider return nil, fmt.Errorf("failed to validate chain config: %w", err) } + if err := opts.ValidateBasic(); err != nil { + return nil, fmt.Errorf("failed to validate chain options: %w", err) + } + var chain Chain chain.mu = sync.RWMutex{} @@ -193,6 +197,10 @@ func (c *Chain) Height(ctx context.Context) (uint64, error) { // Init initializes the chain. That consists of generating the genesis transactions, genesis file, wallets, // the distribution of configuration files and starting the network nodes up func (c *Chain) Init(ctx context.Context, opts petritypes.ChainOptions) error { + if err := opts.ValidateBasic(); err != nil { + return fmt.Errorf("failed to validate chain options: %w", err) + } + decimalPow := int64(math.Pow10(int(c.GetConfig().Decimals))) genesisCoin := types.Coin{ From 6cafe422c33a94e79fed2a13eeaaf944ec063318 Mon Sep 17 00:00:00 2001 From: Zygimantas Date: Mon, 20 Jan 2025 12:27:48 +0100 Subject: [PATCH 7/7] fix: validate ipam config --- core/provider/docker/provider.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/provider/docker/provider.go b/core/provider/docker/provider.go index 0cbd43b..ffedc8b 100644 --- a/core/provider/docker/provider.go +++ b/core/provider/docker/provider.go @@ -80,6 +80,10 @@ func CreateProvider(ctx context.Context, logger *zap.Logger, providerName string dockerProvider.state.NetworkID = network.ID + if len(network.IPAM.Config) == 0 { + return nil, fmt.Errorf("network does not have an IPAM config") + } + _, cidrMask, err := net.ParseCIDR(network.IPAM.Config[0].Subnet) if err != nil { return nil, err