Merge branch 'integration/v1' into quasystaty1/bridge-cli

* integration/v1:
  Feature: store private key in keyring or keyfile (#64)
  use astria-dusk-5 for all networks
  Update go.mod
  Chore: Update to Dusk-5 (#67)
  Fix: Wait for Sequencer gRPC server to start before starting other components (#65)

README.md

@@ -101,10 +101,10 @@ The following binaries are downloaded:
 
 | App              | Version |
 |------------------|---------|
-| Cometbft         | v0.37.4 |
-| Astria-Sequencer | v0.10.1 |
-| Astria-Conductor | v0.13.1 |
-| Astria-Composer  | v0.5.0  |
+| Cometbft         | v0.38.6 |
+| Astria-Sequencer | v0.11.0 |
+| Astria-Conductor | v0.14.0 |
+| Astria-Composer  | v0.6.0  |
 
 The `init` command will also run the initialization steps required by CometBFT,
 using the `genesis.json` and `priv_validator_key.json` files in the

cmd/devtools/binaries_config_darwin_amd64.go

@@ -2,15 +2,14 @@
 
 package devtools
 
-// Your code here
 type Binary struct {
 	Name string
 	Url  string
 }
 
 var Binaries = []Binary{
-	{"cometbft", "https://github.com/cometbft/cometbft/releases/download/v0.37.4/cometbft_0.37.4_darwin_amd64.tar.gz"},
-	{"astria-sequencer", "https://github.com/astriaorg/astria/releases/download/sequencer-v0.10.1/astria-sequencer-x86_64-apple-darwin.tar.gz"},
-	{"astria-composer", "https://github.com/astriaorg/astria/releases/download/composer-v0.5.0/astria-composer-x86_64-apple-darwin.tar.gz"},
-	{"astria-conductor", "https://github.com/astriaorg/astria/releases/download/conductor-v0.13.1/astria-conductor-x86_64-apple-darwin.tar.gz"},
+	{"cometbft", "https://github.com/cometbft/cometbft/releases/download/v0.38.6/cometbft_0.38.6_darwin_amd64.tar.gz"},
+	{"astria-sequencer", "https://github.com/astriaorg/astria/releases/download/sequencer-v0.11.0/astria-sequencer-x86_64-apple-darwin.tar.gz"},
+	{"astria-composer", "https://github.com/astriaorg/astria/releases/download/composer-v0.6.0/astria-composer-x86_64-apple-darwin.tar.gz"},
+	{"astria-conductor", "https://github.com/astriaorg/astria/releases/download/conductor-v0.14.0/astria-conductor-x86_64-apple-darwin.tar.gz"},
 }

cmd/devtools/binaries_config_darwin_arm64.go

@@ -2,15 +2,14 @@
 
 package devtools
 
-// Your code here
 type Binary struct {
 	Name string
 	Url  string
 }
 
 var Binaries = []Binary{
-	{"cometbft", "https://github.com/cometbft/cometbft/releases/download/v0.37.4/cometbft_0.37.4_darwin_arm64.tar.gz"},
-	{"astria-sequencer", "https://github.com/astriaorg/astria/releases/download/sequencer-v0.10.1/astria-sequencer-aarch64-apple-darwin.tar.gz"},
-	{"astria-composer", "https://github.com/astriaorg/astria/releases/download/composer-v0.5.0/astria-composer-aarch64-apple-darwin.tar.gz"},
-	{"astria-conductor", "https://github.com/astriaorg/astria/releases/download/conductor-v0.13.1/astria-conductor-aarch64-apple-darwin.tar.gz"},
+	{"cometbft", "https://github.com/cometbft/cometbft/releases/download/v0.38.6/cometbft_0.38.6_darwin_arm64.tar.gz"},
+	{"astria-sequencer", "https://github.com/astriaorg/astria/releases/download/sequencer-v0.11.0/astria-sequencer-aarch64-apple-darwin.tar.gz"},
+	{"astria-composer", "https://github.com/astriaorg/astria/releases/download/composer-v0.6.0/astria-composer-aarch64-apple-darwin.tar.gz"},
+	{"astria-conductor", "https://github.com/astriaorg/astria/releases/download/conductor-v0.14.0/astria-conductor-aarch64-apple-darwin.tar.gz"},
 }

cmd/devtools/binaries_config_linux_amd64.go

@@ -2,15 +2,14 @@
 
 package devtools
 
-// Your code here
 type Binary struct {
 	Name string
 	Url  string
 }
 
 var Binaries = []Binary{
-	{"cometbft", "https://github.com/cometbft/cometbft/releases/download/v0.37.4/cometbft_0.37.4_linux_amd64.tar.gz"},
-	{"astria-sequencer", "https://github.com/astriaorg/astria/releases/download/sequencer-v0.10.1/astria-sequencer-x86_64-unknown-linux-gnu.tar.gz"},
-	{"astria-composer", "https://github.com/astriaorg/astria/releases/download/composer-v0.5.0/astria-composer-x86_64-unknown-linux-gnu.tar.gz"},
-	{"astria-conductor", "https://github.com/astriaorg/astria/releases/download/conductor-v0.13.1/astria-conductor-x86_64-unknown-linux-gnu.tar.gz"},
+	{"cometbft", "https://github.com/cometbft/cometbft/releases/download/v0.38.6/cometbft_0.38.6_linux_amd64.tar.gz"},
+	{"astria-sequencer", "https://github.com/astriaorg/astria/releases/download/sequencer-v0.11.0/astria-sequencer-x86_64-unknown-linux-gnu.tar.gz"},
+	{"astria-composer", "https://github.com/astriaorg/astria/releases/download/composer-v0.6.0/astria-composer-x86_64-unknown-linux-gnu.tar.gz"},
+	{"astria-conductor", "https://github.com/astriaorg/astria/releases/download/conductor-v0.14.0/astria-conductor-x86_64-unknown-linux-gnu.tar.gz"},
 }

cmd/devtools/config/genesis.json

@@ -1,6 +1,6 @@
 {
   "genesis_time": "2024-02-21T22:23:22.87689Z",
-  "chain_id": "test-chain-Lc3lcg",
+  "chain_id": "astria-dusk-5",
   "initial_height": "0",
   "consensus_params": {
     "block": {

cmd/devtools/config/local.env.example

@@ -1,6 +1,11 @@
 ###########################
 ######## conductor ########
 ###########################
+
+# The Celestia block time in milliseconds. Used for periodically polling Celestia for its latest network height.
+# 12000 milliseconds is the default Celestia block time.
+ASTRIA_CONDUCTOR_CELESTIA_BLOCK_TIME_MS=12000
+
 # The bearer token to retrieve sequencer blocks as blobs from Celestia.
 # The token is obtained by running `celestia bridge auth <permissions>`
 # on the host running the celestia node.
@@ -10,10 +15,6 @@ ASTRIA_CONDUCTOR_CELESTIA_BEARER_TOKEN="<JWT Bearer token>"
 # either http or https as scheme.
 ASTRIA_CONDUCTOR_CELESTIA_NODE_HTTP_URL="http://127.0.0.1:26658"
 
-# The URL of the celestia node to subscribe to new network headers. This URL must contain
-# either ws or wss as scheme.
-ASTRIA_CONDUCTOR_CELESTIA_NODE_WEBSOCKET_URL="ws://127.0.0.1:26658"
-
 # Execution RPC URL
 ASTRIA_CONDUCTOR_EXECUTION_RPC_URL="http://127.0.0.1:50051"
 
@@ -144,6 +145,9 @@ ASTRIA_COMPOSER_API_LISTEN_ADDR="0.0.0.0:0"
 # Address of the RPC server for the sequencer chain
 ASTRIA_COMPOSER_SEQUENCER_URL="http://127.0.0.1:26657"
 
+# Chain ID of the sequencer chain which transactions are submitted to.
+ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID="astria-dusk-5"
+
 # A list of execution `<rollup_name_1>::<rpc_server_1>,<rollup_name_2>::<rpc_server_2>`.
 # Rollup names are not case sensitive. If a name is repeated, the last list item is used.
 # names are sha256 hashed and used as the `rollup_id` in `SequenceAction`s
@@ -164,6 +168,13 @@ ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS=2000
 # key and nonce bytes
 ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE=200000
 
+# Max amount of finished bundles that can be in the submission queue.
+# ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY * ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE (e.g.
+# 40000 * 200KB=8GB) is the limit on how much memory the finished bundle queue can consume.
+# This should be lower than the resource limit enforced by Kubernetes on the pod, defined here:
+# https://github.com/astriaorg/astria/blob/622d4cb8695e4fbcd86456bd16149420b8acda79/charts/evm-rollup/values.yaml#L276
+ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY=40000
+
 # Set to true to enable prometheus metrics.
 ASTRIA_COMPOSER_NO_METRICS=true
 

cmd/devtools/config/remote.env.example

@@ -1,6 +1,10 @@
 ###########################
 ######## conductor ########
 ###########################
+# The Celestia block time in milliseconds. Used for periodically polling Celestia for its latest network height.
+# 12000 milliseconds is the default Celestia block time.
+ASTRIA_CONDUCTOR_CELESTIA_BLOCK_TIME_MS=12000
+
 # The bearer token to retrieve sequencer blocks as blobs from Celestia.
 # The token is obtained by running `celestia bridge auth <permissions>`
 # on the host running the celestia node.
@@ -10,10 +14,6 @@ ASTRIA_CONDUCTOR_CELESTIA_BEARER_TOKEN="<JWT Bearer token>"
 # either http or https as scheme.
 ASTRIA_CONDUCTOR_CELESTIA_NODE_HTTP_URL="http://127.0.0.1:26658"
 
-# The URL of the celestia node to subscribe to new network headers. This URL must contain
-# either ws or wss as scheme.
-ASTRIA_CONDUCTOR_CELESTIA_NODE_WEBSOCKET_URL="ws://127.0.0.1:26658"
-
 # Execution RPC URL
 ASTRIA_CONDUCTOR_EXECUTION_RPC_URL="http://127.0.0.1:50051"
 
@@ -47,13 +47,13 @@ NO_COLOR=
 
 # The URL of the Sequencer gRPC service for requesting new blocks.
 # 127.0.0.1:8080 is the default socket address for its gRPC server.
-ASTRIA_CONDUCTOR_SEQUENCER_GRPC_URL="https://grpc.sequencer.dusk-4.devnet.astria.org/"
+ASTRIA_CONDUCTOR_SEQUENCER_GRPC_URL="https://grpc.sequencer.dusk-5.devnet.astria.org/"
 
 # The URL of a fully trusted CometBFT/Sequencer node to get the latest height of
 # the Sequencer network, and to get the validators for a given Sequencer height
 # to verify Sequencer data read off Celestia.
 # 127.0.0.1:26657 is the default socket address in comebft's `rpc.laddr` setting.
-ASTRIA_CONDUCTOR_SEQUENCER_COMETBFT_URL="https://rpc.sequencer.dusk-4.devnet.astria.org"
+ASTRIA_CONDUCTOR_SEQUENCER_COMETBFT_URL="https://rpc.sequencer.dusk-5.devnet.astria.org/"
 
 # The duration in milliseconds that conductor waits between requests for the latest
 # block height from sequencer.
@@ -97,7 +97,10 @@ NO_COLOR=
 ASTRIA_COMPOSER_API_LISTEN_ADDR="0.0.0.0:0"
 
 # Address of the RPC server for the sequencer chain
-ASTRIA_COMPOSER_SEQUENCER_URL="https://rpc.sequencer.dusk-4.devnet.astria.org"
+ASTRIA_COMPOSER_SEQUENCER_URL="https://rpc.sequencer.dusk-5.devnet.astria.org"
+
+# Chain ID of the sequencer chain which transactions are submitted to.
+ASTRIA_COMPOSER_SEQUENCER_CHAIN_ID="astria-dusk-5"
 
 # A list of execution `<rollup_name_1>::<rpc_server_1>,<rollup_name_2>::<rpc_server_2>`.
 # Rollup names are not case sensitive. If a name is repeated, the last list item is used.
@@ -119,6 +122,13 @@ ASTRIA_COMPOSER_MAX_SUBMIT_INTERVAL_MS=2000
 # key and nonce bytes
 ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE=200000
 
+# Max amount of finished bundles that can be in the submission queue.
+# ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY * ASTRIA_COMPOSER_MAX_BYTES_PER_BUNDLE (e.g.
+# 40000 * 200KB=8GB) is the limit on how much memory the finished bundle queue can consume.
+# This should be lower than the resource limit enforced by Kubernetes on the pod, defined here:
+# https://github.com/astriaorg/astria/blob/622d4cb8695e4fbcd86456bd16149420b8acda79/charts/evm-rollup/values.yaml#L276
+ASTRIA_COMPOSER_BUNDLE_QUEUE_CAPACITY=40000
+
 # Set to true to enable prometheus metrics.
 ASTRIA_COMPOSER_NO_METRICS=true
 

cmd/devtools/run.go

@@ -2,8 +2,11 @@ package devtools
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/astria/astria-cli-go/cmd"
 	"github.com/astria/astria-cli-go/internal/processrunner"
@@ -80,39 +83,51 @@ func runCmdHandler(c *cobra.Command, args []string) {
 		log.Debugf("Using binaries from %s", binDir)
 
 		// sequencer
+		seqRCOpts := processrunner.ReadyCheckerOpts{
+			CallBackName:  "Sequencer gRPC server is OK",
+			Callback:      getSequencerOKCallback(envPath),
+			RetryCount:    10,
+			RetryInterval: 100 * time.Millisecond,
+			HaltIfFailed:  false,
+		}
+		seqReadinessCheck := processrunner.NewReadyChecker(seqRCOpts)
 		seqOpts := processrunner.NewProcessRunnerOpts{
-			Title:   "Sequencer",
-			BinPath: sequencerPath,
-			EnvPath: envPath,
-			Args:    nil,
+			Title:      "Sequencer",
+			BinPath:    sequencerPath,
+			EnvPath:    envPath,
+			Args:       nil,
+			ReadyCheck: &seqReadinessCheck,
 		}
 		seqRunner := processrunner.NewProcessRunner(ctx, seqOpts)
 
 		// cometbft
 		cometDataPath := filepath.Join(dataDir, ".cometbft")
 		cometOpts := processrunner.NewProcessRunnerOpts{
-			Title:   "Comet BFT",
-			BinPath: cometbftPath,
-			EnvPath: envPath,
-			Args:    []string{"node", "--home", cometDataPath},
+			Title:      "Comet BFT",
+			BinPath:    cometbftPath,
+			EnvPath:    envPath,
+			Args:       []string{"node", "--home", cometDataPath},
+			ReadyCheck: nil,
 		}
 		cometRunner := processrunner.NewProcessRunner(ctx, cometOpts)
 
 		// composer
 		composerOpts := processrunner.NewProcessRunnerOpts{
-			Title:   "Composer",
-			BinPath: composerPath,
-			EnvPath: envPath,
-			Args:    nil,
+			Title:      "Composer",
+			BinPath:    composerPath,
+			EnvPath:    envPath,
+			Args:       nil,
+			ReadyCheck: nil,
 		}
 		compRunner := processrunner.NewProcessRunner(ctx, composerOpts)
 
 		// conductor
 		conductorOpts := processrunner.NewProcessRunnerOpts{
-			Title:   "Conductor",
-			BinPath: conductorPath,
-			EnvPath: envPath,
-			Args:    nil,
+			Title:      "Conductor",
+			BinPath:    conductorPath,
+			EnvPath:    envPath,
+			Args:       nil,
+			ReadyCheck: nil,
 		}
 		condRunner := processrunner.NewProcessRunner(ctx, conductorOpts)
 
@@ -150,19 +165,21 @@ func runCmdHandler(c *cobra.Command, args []string) {
 
 		// composer
 		composerOpts := processrunner.NewProcessRunnerOpts{
-			Title:   "Composer",
-			BinPath: composerPath,
-			EnvPath: envPath,
-			Args:    nil,
+			Title:      "Composer",
+			BinPath:    composerPath,
+			EnvPath:    envPath,
+			Args:       nil,
+			ReadyCheck: nil,
 		}
 		compRunner := processrunner.NewProcessRunner(ctx, composerOpts)
 
 		// conductor
 		conductorOpts := processrunner.NewProcessRunnerOpts{
-			Title:   "Conductor",
-			BinPath: conductorPath,
-			EnvPath: envPath,
-			Args:    nil,
+			Title:      "Conductor",
+			BinPath:    conductorPath,
+			EnvPath:    envPath,
+			Args:       nil,
+			ReadyCheck: nil,
 		}
 		condRunner := processrunner.NewProcessRunner(ctx, conductorOpts)
 
@@ -221,3 +238,38 @@ func getFlagPathOrPanic(c *cobra.Command, flagName string, defaultValue string)
 		return defaultValue
 	}
 }
+
+// getSequencerOKCallback builds an anonymous function for use in a ProcessRunner
+// ReadyChecker callback. The anonymous function checks if the gRPC server that
+// is started by the sequencer is OK by making an HTTP request to the health
+// endpoint. Being able to connect to the gRPC server is a requirement for both
+// the Conductor and Composer services.
+func getSequencerOKCallback(envPath string) func() bool {
+	return func() bool {
+		// Get the sequencer gRPC address from the environment
+		seqEnv := processrunner.GetEnvironment(envPath)
+		var seqGRPCAddr string
+		for _, envVar := range seqEnv {
+			if strings.HasPrefix(envVar, "ASTRIA_SEQUENCER_GRPC_ADDR") {
+				seqGRPCAddr = strings.Split(envVar, "=")[1]
+				break
+			}
+		}
+
+		// Make the HTTP request
+		resp, err := http.Get("http://" + seqGRPCAddr + "/health")
+		if err != nil {
+			log.WithError(err).Debug("Startup callback check to sequencer gRPC /health did not succeed")
+			return false
+		}
+		defer resp.Body.Close()
+
+		// Check status code
+		if resp.StatusCode == 200 {
+			log.Debug("Sequencer gRPC server started")
+			return true
+		}
+
+		return false
+	}
+}

cmd/logging.go

@@ -53,5 +53,5 @@ func CreateUILog(destDir string) {
 		DisableColors: true, // Disable ANSI color codes
 		FullTimestamp: true,
 	})
-	log.Info("New log file created successfully:", logPath)
+	log.Debug("New log file created successfully:", logPath)
 }

cmd/sequencer/transfer.go

@@ -26,6 +26,7 @@ func init() {
 	transferCmd.Flags().String("keyring-address", "", "The address of the sender. Requires private key be stored in keyring.")
 	transferCmd.Flags().String("privkey", "", "The private key of the sender.")
 	transferCmd.MarkFlagsOneRequired("keyfile", "keyring-address", "privkey")
+	transferCmd.MarkFlagsMutuallyExclusive("keyfile", "keyring-address", "privkey")
 }
 
 func transferCmdHandler(cmd *cobra.Command, args []string) {