feat: add support for alt-da with da-server

fix: type bugs in da_server related configs

chore: refactor da-server launcher to not hardcode image

feat: add optimism repo as submodule to build da-server image

revert: changes to network_params.yaml

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "optimism"]
+	path = optimism
+	url = [email protected]:ethereum-optimism/optimism.git

README.md

@@ -251,6 +251,14 @@ optimism_package:
       # - blockscout
       additional_services: []
 
+      # Configuration place for da-server - https://github.com/ethereum-optimism/optimism/tree/develop/op-alt-da
+      da_server_params:
+        image: us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:dev
+        build_image: true
+        # A list of optional extra params that will be passed to the da-server container for modifying its behaviour
+        da_server_extra_args: []
+        generic_commitment: false
+
   # L2 contract deployer configuration - used for all L2 networks
   # The docker image that should be used for the L2 contract deployer
   op_contract_deployer_params:

main.star

@@ -27,6 +27,7 @@ def run(plan, args):
     global_node_selectors = optimism_args_with_right_defaults.global_node_selectors
     global_log_level = optimism_args_with_right_defaults.global_log_level
     persistent = optimism_args_with_right_defaults.persistent
+    da_server_params = args_with_right_defaults.da_server_params
 
     # Deploy the L1
     plan.print("Deploying a local L1")
@@ -55,6 +56,7 @@ def run(plan, args):
         l1_priv_key,
         l1_config_env_vars,
         optimism_args_with_right_defaults,
+        da_server_params,
     )
 
     for chain in optimism_args_with_right_defaults.chains:

src/alt-da/da-server/da_server_launcher.star

@@ -0,0 +1,98 @@
+shared_utils = import_module(
+    "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star"
+)
+constants = import_module(
+    "github.com/ethpandaops/ethereum-package/src/package_io/constants.star"
+)
+
+# The dirpath of the data directory on the da-server container
+# note that we use /home which is available but not persistent
+# because we aren't mounting an external kurtosis file
+# this means that the data is lost when the container is deleted
+DATA_DIRPATH_ON_DA_SERVER_CONTAINER = "/home"
+
+# Port IDs
+DA_SERVER_HTTP_PORT_ID = "http"
+
+# Port nums
+DA_SERVER_HTTP_PORT_NUM = 3100
+
+
+def get_used_ports():
+    used_ports = {
+        DA_SERVER_HTTP_PORT_ID: shared_utils.new_port_spec(
+            DA_SERVER_HTTP_PORT_NUM,
+            shared_utils.TCP_PROTOCOL,
+            shared_utils.HTTP_APPLICATION_PROTOCOL,
+        ),
+    }
+    return used_ports
+
+def launch(
+    plan,
+    service_name,
+    image,
+    da_server_extra_args,
+    generic_commitment,
+):
+
+    config = get_da_server_config(
+        plan,
+        service_name,
+        image,
+        da_server_extra_args,
+        generic_commitment,
+    )
+
+    da_server_service = plan.add_service(service_name, config)
+
+    http_url = "http://{0}:{1}".format(da_server_service.ip_address, DA_SERVER_HTTP_PORT_NUM)
+    return new_da_server_context(
+        http_url=http_url,
+        generic_commitment=generic_commitment,
+    )
+
+
+def get_da_server_config(
+    plan,
+    service_name,
+    image,
+    da_server_extra_args,
+    generic_commitment,
+):
+    ports = get_used_ports()
+
+    cmd = [
+        "da-server",
+        "--file.path=" + DATA_DIRPATH_ON_DA_SERVER_CONTAINER,
+        "--addr=0.0.0.0",
+        "--port=3100",
+        "--log.level=debug",
+        "--generic-commitment=" + str(generic_commitment),
+    ]
+
+    if len(da_server_extra_args) > 0:
+        cmd.extend([param for param in da_server_extra_args])
+
+    return ServiceConfig(
+        image=image,
+        ports=ports,
+        cmd=cmd,
+        private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
+    )
+
+def disabled_da_server_context():
+    return new_da_server_context(
+        http_url="",
+        generic_commitment=True,
+    )
+
+def new_da_server_context(
+    http_url,
+    generic_commitment
+):
+    return struct(
+        enabled=http_url != "",
+        http_url=http_url,
+        generic_commitment=generic_commitment,
+    )

src/batcher/op-batcher/op_batcher_launcher.star

@@ -41,6 +41,7 @@ def launch(
     l1_config_env_vars,
     gs_batcher_private_key,
     batcher_params,
+    da_server_context,
 ):
     batcher_service_name = "{0}".format(service_name)
 
@@ -53,6 +54,7 @@ def launch(
         l1_config_env_vars,
         gs_batcher_private_key,
         batcher_params,
+        da_server_context,
     )
 
     batcher_service = plan.add_service(service_name, config)
@@ -74,6 +76,7 @@ def get_batcher_config(
     l1_config_env_vars,
     gs_batcher_private_key,
     batcher_params,
+    da_server_context,
 ):
     cmd = [
         "op-batcher",
@@ -90,7 +93,10 @@ def get_batcher_config(
         "--max-channel-duration=1",
         "--l1-eth-rpc=" + l1_config_env_vars["L1_RPC_URL"],
         "--private-key=" + gs_batcher_private_key,
-        "--data-availability-type=blobs",
+        "--altda.enabled=" + str(da_server_context.enabled),
+        "--altda.da-server=" + da_server_context.http_url,
+        "--altda.da-service=" + str(da_server_context.generic_commitment),
+        "--data-availability-type=" + "calldata" if da_server_context.enabled else "blobs",
     ]
 
     cmd += batcher_params.extra_params

src/cl/hildr/hildr_launcher.star

@@ -73,6 +73,7 @@ def launch(
     existing_cl_clients,
     l1_config_env_vars,
     sequencer_enabled,
+    da_server_context,
 ):
     # beacon_node_identity_recipe = PostHttpRequestRecipe(
     #     endpoint="/",
@@ -103,6 +104,7 @@ def launch(
         existing_cl_clients,
         l1_config_env_vars,
         sequencer_enabled,
+        da_server_context,
     )
 
     beacon_service = plan.add_service(service_name, config)
@@ -143,6 +145,7 @@ def get_beacon_config(
     existing_cl_clients,
     l1_config_env_vars,
     sequencer_enabled,
+    da_server_context,
 ):
     EXECUTION_ENGINE_ENDPOINT = "http://{0}:{1}".format(
         el_context.ip_addr,
@@ -169,6 +172,9 @@ def get_beacon_config(
         "--network="
         + ethereum_package_constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS
         + "/rollup-{0}.json".format(launcher.network_params.network_id),
+        # TODO: support altda flags once they are implemented.
+        # See https://github.com/optimism-java/hildr/issues/134
+        # eg: "--altda.enabled=" + str(da_server_context.enabled),
     ]
 
     sequencer_private_key = util.read_network_config_value(

src/cl/op-node/op_node_launcher.star

@@ -73,6 +73,7 @@ def launch(
     existing_cl_clients,
     l1_config_env_vars,
     sequencer_enabled,
+    da_server_context,
 ):
     beacon_node_identity_recipe = PostHttpRequestRecipe(
         endpoint="/",
@@ -104,6 +105,7 @@ def launch(
         l1_config_env_vars,
         beacon_node_identity_recipe,
         sequencer_enabled,
+        da_server_context,
     )
 
     beacon_service = plan.add_service(service_name, config)
@@ -148,6 +150,7 @@ def get_beacon_config(
     l1_config_env_vars,
     beacon_node_identity_recipe,
     sequencer_enabled,
+    da_server_context,
 ):
     EXECUTION_ENGINE_ENDPOINT = "http://{0}:{1}".format(
         el_context.ip_addr,
@@ -178,7 +181,11 @@ def get_beacon_config(
         "--p2p.listen.ip=0.0.0.0",
         "--p2p.listen.tcp={0}".format(BEACON_DISCOVERY_PORT_NUM),
         "--p2p.listen.udp={0}".format(BEACON_DISCOVERY_PORT_NUM),
+        "--altda.enabled=" + str(da_server_context.enabled),
+        "--altda.da-service=" + str(da_server_context.generic_commitment),
+        "--altda.da-server=" + da_server_context.http_url,
     ]
+    plan.print(da_server_context.generic_commitment)
 
     sequencer_private_key = util.read_network_config_value(
         plan,

src/contracts/contract_deployer.star

@@ -14,6 +14,7 @@ def deploy_contracts(
     priv_key,
     l1_config_env_vars,
     optimism_args,
+    da_server_params, # for alt-da
 ):
     l2_chain_ids = ",".join(
         [str(chain.network_params.network_id) for chain in optimism_args.chains]
@@ -23,7 +24,11 @@ def deploy_contracts(
         name="op-deployer-init",
         description="Initialize L2 contract deployments",
         image=optimism_args.op_contract_deployer_params.image,
-        env_vars=l1_config_env_vars,
+        env_vars=l1_config_env_vars
+        | {
+            "USE_ALTDA": "true" if da_server_params.enabled else "false",
+            "DA_COMMITMENT_TYPE": "GenericCommitment" if da_server_params.generic_commitment else "KeccakCommitment"
+        },
         store=[
             StoreSpec(
                 src="/network-data",

src/el_cl_launcher.star

@@ -30,6 +30,7 @@ def launch(
     global_node_selectors,
     global_tolerations,
     persistent,
+    da_server_context,
 ):
     el_launchers = {
         "op-geth": {
@@ -177,6 +178,7 @@ def launch(
             all_cl_contexts,
             l1_config_env_vars,
             sequencer_enabled,
+            da_server_context,
         )
 
         sequencer_enabled = False

src/l2.star

@@ -1,5 +1,6 @@
 participant_network = import_module("./participant_network.star")
 blockscout = import_module("./blockscout/blockscout_launcher.star")
+da_server_launcher = import_module("./alt-da/da-server/da_server_launcher.star")
 contract_deployer = import_module("./contracts/contract_deployer.star")
 input_parser = import_module("./package_io/input_parser.star")
 ethereum_package_static_files = import_module(
@@ -30,6 +31,32 @@ def launch_l2(
         name="op_jwt_file{0}".format(l2_services_suffix),
     )
 
+    # we need to launch da-server before launching the participant network
+    # because op-node and op-batcher need to know the da-server url, if present
+    da_server_context = da_server_launcher.disabled_da_server_context()
+    if "da_server" in args_with_right_defaults.additional_services:
+        da_server_image = args_with_right_defaults.da_server_params.image
+        if args_with_right_defaults.da_server_params.build_image:
+            plan.print("Building da-server image")
+            da_server_image = ImageBuildSpec(
+                image_name=args_with_right_defaults.da_server_params.image,
+                # TODO: this doesn't work... because can't point to a dir outside of the kurtosis package
+                # also can't install optimism monorepo as a submodule because that makes the kurtosis package > 100MB, which is not allowed.
+                # Not sure how to fix this... detailed problem in https://github.com/ethpandaops/optimism-package/issues/72
+                build_context_dir="/optimism/ops/docker/op-stack-go",
+                target_stage="da-server-target",
+            )
+            plan.print("Successfully built da-server image")
+        plan.print("Launching da-server")
+        da_server_context = da_server_launcher.launch(
+            plan,
+            "da-server{0}".format(l2_services_suffix),
+            da_server_image,
+            args_with_right_defaults.da_server_params.da_server_extra_args,
+            args_with_right_defaults.da_server_params.generic_commitment,
+        )
+        plan.print("Successfully launched da-server")
+
     all_l2_participants = participant_network.launch_participant_network(
         plan,
         l2_args.participants,
@@ -43,6 +70,7 @@ def launch_l2(
         global_node_selectors,
         global_tolerations,
         persistent,
+        da_server_context,
     )
 
     all_el_contexts = []

src/package_io/input_parser.star

@@ -25,6 +25,13 @@ DEFAULT_PROPOSER_IMAGES = {
     "op-proposer": "us-docker.pkg.dev/oplabs-tools-artifacts/images/op-proposer:develop",
 }
 
+DEFAULT_DA_SERVER_IMAGES = {
+    # latest tag is super outdated, and doesn't have the --generic-commitment flag
+    # so we use the dev tag as default, which requires building locally (see default_da_server_params)
+    "da-server": "us-docker.pkg.dev/oplabs-tools-artifacts/images/da-server:dev",
+}
+
+
 DEFAULT_ADDITIONAL_SERVICES = []
 
 
@@ -35,6 +42,14 @@ def input_parser(plan, input_args):
     results["global_node_selectors"] = {}
     results["global_tolerations"] = []
     results["persistent"] = False
+
+    results["da_server_params"] = default_da_server_params()
+    for attr, value in input_args.items():
+        if attr == "da_server_params":
+            results["da_server_params"]["enabled"] = "true"
+            for sub_attr, sub_value in value.items():
+                results["da_server_params"][sub_attr] = sub_value
+
     return struct(
         chains=[
             struct(
@@ -99,6 +114,13 @@ def input_parser(plan, input_args):
         global_node_selectors=results["global_node_selectors"],
         global_tolerations=results["global_tolerations"],
         persistent=results["persistent"],
+        da_server_params=struct(
+            enabled=result["da_server_params"]["enabled"],
+            image=result["da_server_params"]["image"],
+            build_image=result["da_server_params"]["build_image"],
+            da_server_extra_args=result["da_server_params"]["da_server_extra_args"],
+            generic_commitment=result["da_server_params"]["generic_commitment"],
+        ),
     )
 
 
@@ -275,3 +297,12 @@ def default_ethereum_package_network_params():
             ),
         }
     }
+
+def default_da_server_params():
+    return {
+        "enabled": False,
+        "image": DEFAULT_DA_SERVER_IMAGES["da-server"],
+        "build_image": True,
+        "da_server_extra_args": [],
+        "generic_commitment": False,
+    }