From 0b5494bccabd494d78f23ae1d3eb182ef606d2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Vincent?= <28714795+leovct@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:26:22 +0100 Subject: [PATCH] refactor: break down the deployment process into various stages (#28) * feat: introduce stages * chore: group config and service * chore: nit * chore: move code to its own package * refactor: clean up stage 4 * chore: nit * chore: clean up stage 2 * chore: nit * chore: lint * chore: nit * chore: clean up * chore: remove updates to bridge, agglayer and dac configs in run-contract-setup.sh * fix: typo * refactor: introduce `get_key_from_config` * chore: fetch genesis and keystores + simplify get key from config * chore: use `plan.store_service_files` * fix: a few bugs * fix: permissionless-node genesis issue * fix: genesis artifact issue * chore: nit * doc: document how to leverage stages in README --------- Co-authored-by: John Hilliard --- README.org | 41 ++ lib/service.star | 13 + main.star | 737 +++++++++++++++++++------------- params.yml | 5 + templates/agglayer-config.toml | 2 +- templates/bridge-config.toml | 12 +- templates/dac-config.toml | 4 +- templates/run-contract-setup.sh | 23 - 8 files changed, 509 insertions(+), 328 deletions(-) create mode 100644 lib/service.star diff --git a/README.org b/README.org index 78eaee72..1f2338c8 100644 --- a/README.org +++ b/README.org @@ -122,6 +122,47 @@ node to the ~cdk-v1~ enclave: kurtosis run --enclave cdk-v1 --args-file params.yml --main-file zkevm_permissionless_node.star . #+end_src +** For Developers + +Rather than executing the deployment process as a monolithic operation, you can break it down into stages and run each stage separately. The `stages` parameter indicates the specific stages you wish the deployment to proceed through. By default, it will execute all the stages. + +Currently, the deployment process includes the following stages: + + 1. Deploy L1 + 2. Configure L1 (which involves funding accounts, deploying CDK contracts, and creating configuration files) + 3. Deploy Trusted / Central Environment + 4. Deploy Bridge, AggLayer, and DAC + 5. Deploy Permissionless Node + +Here's an example of how you can specify the stages to run through. + +#+begin_src bash +# Execute the first stage (deploy L1). +yq e '.stages = [1]' --inplace params.yml +kurtosis run --enclave cdk-v1 --args-file params.yml . +# Perform additional tasks... + +# Execute the second stage (configure L1). +yq e '.stages = [2]' --inplace params.yml +kurtosis run --enclave cdk-v1 --args-file params.yml . +# Perform additional tasks... + +# Execute the thirs stage (deploy trusted/central environment). +yq e '.stages = [3]' --inplace params.yml +kurtosis run --enclave cdk-v1 --args-file params.yml . +# Perform additional tasks... + +# Execute the fourth stage (deploy bridge/agglayer/dac). +yq e '.stages = [4]' --inplace params.yml +kurtosis run --enclave cdk-v1 --args-file params.yml . +# Perform additional tasks... + +# Execute the fifth stage (deploy permissionless node). +yq e '.stages = [5]' --inplace params.yml +kurtosis run --enclave cdk-v1 --args-file params.yml . +# Perform additional tasks... +#+end_src + ** License Copyright (c) 2024 PT Services DMCC diff --git a/lib/service.star b/lib/service.star new file mode 100644 index 00000000..9a53571b --- /dev/null +++ b/lib/service.star @@ -0,0 +1,13 @@ +# Extract a specific key from a JSON file, within a service, using jq. +def extract_json_key_from_service(plan, service_name, filename, key): + plan.print("Extracting contract addresses and ports...") + exec_recipe = ExecRecipe( + command=[ + "/bin/sh", + "-c", + "cat {}".format(filename, key), + ], + extract={"extracted_value": "fromjson | .{}".format(key)}, + ) + result = plan.exec(service_name=service_name, recipe=exec_recipe) + return result["extract.extracted_value"] diff --git a/main.star b/main.star index eb9d5846..ed3f36d7 100644 --- a/main.star +++ b/main.star @@ -1,15 +1,29 @@ ethereum_package = import_module( "github.com/kurtosis-tech/ethereum-package/main.star@2.0.0" ) +service_package = import_module("./lib/service.star") zkevm_databases_package = import_module("./lib/zkevm_databases.star") zkevm_node_package = import_module("./lib/zkevm_node.star") zkevm_prover_package = import_module("./lib/zkevm_prover.star") zkevm_permissionless_node_package = import_module("./zkevm_permissionless_node.star") +DEPLOYMENT_STAGE = struct( + deploy_l1=1, + configure_l1=2, + deploy_central_environment=3, + deploy_cdk_bridge_infra=4, + deploy_permissionless_node=5, +) + + def run(plan, args): + plan.print("Deploying CDK environment for stages: " + str(args["stages"])) + # Determine system architecture - cpu_arch_result = plan.run_sh(run="uname -m | tr -d '\n'") + cpu_arch_result = plan.run_sh( + run="uname -m | tr -d '\n'", description="Determining CPU system architecture" + ) cpu_arch = cpu_arch_result.output plan.print("Running on {} architecture".format(cpu_arch)) if not "cpu_arch" in args: @@ -19,245 +33,314 @@ def run(plan, args): if args["zkevm_rollup_consensus"] == "PolygonValidiumEtrog": args["is_cdk"] = True - # Deploy L1 chain + ## STAGE 1: Deploy L1 # For now we'll stick with most of the defaults - ethereum_package.run( - plan, - { - "additional_services": [], - "network_params": { - # The ethereum package requires the network id to be a string. - "network_id": str(args["l1_network_id"]), - "preregistered_validator_keys_mnemonic": args[ - "l1_preallocated_mnemonic" - ], + if DEPLOYMENT_STAGE.deploy_l1 in args["stages"]: + plan.print("Executing stage " + str(DEPLOYMENT_STAGE.deploy_l1)) + ethereum_package.run( + plan, + { + "additional_services": [], + "network_params": { + # The ethereum package requires the network id to be a string. + "network_id": str(args["l1_network_id"]), + "preregistered_validator_keys_mnemonic": args[ + "l1_preallocated_mnemonic" + ], + }, }, - }, - ) + ) + else: + plan.print("Skipping stage " + str(DEPLOYMENT_STAGE.deploy_l1)) - # Create deploy parameters - deploy_parameters_template = read_file(src="./templates/deploy_parameters.json") - deploy_parameters_artifact = plan.render_templates( - config={ - "deploy_parameters.json": struct( - template=deploy_parameters_template, data=args - ) - }, - name="deploy-parameters-artifact", - ) - # Create rollup paramaters - create_rollup_parameters_template = read_file( - src="./templates/create_rollup_parameters.json" - ) - create_rollup_parameters_artifact = plan.render_templates( - config={ - "create_rollup_parameters.json": struct( - template=create_rollup_parameters_template, data=args - ) - }, - name="create-rollup-parameters-artifact", - ) - # Create contract deployment script - contract_deployment_script_template = read_file( - src="./templates/run-contract-setup.sh" - ) - contract_deployment_script_artifact = plan.render_templates( - config={ - "run-contract-setup.sh": struct( - template=contract_deployment_script_template, data=args - ) - }, - name="contract-deployment-script-artifact", - ) + ## STAGE 2: Configure L1 + # Ffund accounts, deploy cdk contracts and create config files. + if DEPLOYMENT_STAGE.configure_l1 in args["stages"]: + plan.print("Executing stage " + str(DEPLOYMENT_STAGE.configure_l1)) - # Create bridge configuration - bridge_config_template = read_file(src="./templates/bridge-config.toml") - bridge_config_artifact = plan.render_templates( - config={ - "bridge-config.toml": struct(template=bridge_config_template, data=args) - }, - name="bridge-config-artifact", - ) - # Create AggLayer configuration - agglayer_config_template = read_file(src="./templates/agglayer-config.toml") - agglayer_config_artifact = plan.render_templates( - config={ - "agglayer-config.toml": struct(template=agglayer_config_template, data=args) - }, - name="agglayer-config-artifact", - ) - # Create DAC configuration - dac_config_template = read_file(src="./templates/dac-config.toml") - dac_config_artifact = plan.render_templates( - config={"dac-config.toml": struct(template=dac_config_template, data=args)}, - name="dac-config-artifact", - ) - # Create prover configuration - prover_config_template = read_file( - src="./templates/trusted-node/prover-config.json" - ) - prover_config_artifact = plan.render_templates( - config={ - "prover-config.json": struct(template=prover_config_template, data=args) - }, - name="prover-config-artifact", - ) + # Create deploy parameters + deploy_parameters_template = read_file(src="./templates/deploy_parameters.json") + deploy_parameters_artifact = plan.render_templates( + name="deploy-parameters-artifact", + config={ + "deploy_parameters.json": struct( + template=deploy_parameters_template, data=args + ) + }, + ) - # Create helper service to deploy contracts - zkevm_etc_directory = Directory(persistent_key="zkevm-artifacts") - plan.add_service( - name="contracts" + args["deployment_suffix"], - config=ServiceConfig( - image="node:20-bookworm", - files={ - "/opt/zkevm": zkevm_etc_directory, - "/opt/contract-deploy/": Directory( - artifact_names=[ - deploy_parameters_artifact, - create_rollup_parameters_artifact, - contract_deployment_script_artifact, - prover_config_artifact, - bridge_config_artifact, - agglayer_config_artifact, - dac_config_artifact, - ] - ), + # Create rollup paramaters + create_rollup_parameters_template = read_file( + src="./templates/create_rollup_parameters.json" + ) + create_rollup_parameters_artifact = plan.render_templates( + name="create-rollup-parameters-artifact", + config={ + "create_rollup_parameters.json": struct( + template=create_rollup_parameters_template, data=args + ) }, - ), - ) + ) - # TODO: Check if the contracts were already initialized.. I'm leaving this here for now, but it's not useful!! - contract_init_stat = plan.exec( - service_name="contracts" + args["deployment_suffix"], - acceptable_codes=[0, 1], - recipe=ExecRecipe(command=["stat", "/opt/zkevm/.init-complete.lock"]), - ) + # Create contract deployment script + contract_deployment_script_template = read_file( + src="./templates/run-contract-setup.sh" + ) + contract_deployment_script_artifact = plan.render_templates( + name="contract-deployment-script-artifact", + config={ + "run-contract-setup.sh": struct( + template=contract_deployment_script_template, data=args + ) + }, + ) - # Deploy contracts - plan.exec( - service_name="contracts" + args["deployment_suffix"], - recipe=ExecRecipe( - command=[ - "git", - "clone", - "--depth", - "1", - "-b", - args["zkevm_contracts_branch"], - args["zkevm_contracts_repo"], - "/opt/zkevm-contracts", - ] - ), - ) - plan.exec( - service_name="contracts" + args["deployment_suffix"], - recipe=ExecRecipe( - command=["chmod", "a+x", "/opt/contract-deploy/run-contract-setup.sh"] - ), - ) - plan.print("Running zkEVM contract deployment. This might take some time...") - plan.exec( - service_name="contracts" + args["deployment_suffix"], - recipe=ExecRecipe(command=["/opt/contract-deploy/run-contract-setup.sh"]), - ) - zkevm_configs = plan.store_service_files( - service_name="contracts" + args["deployment_suffix"], - src="/opt/zkevm", - name="zkevm", - description="These are the files needed to start various node services", - ) + # Create helper service to deploy contracts + plan.add_service( + name="contracts" + args["deployment_suffix"], + config=ServiceConfig( + image="node:20-bookworm", + files={ + "/opt/zkevm": Directory(persistent_key="zkevm-artifacts"), + "/opt/contract-deploy/": Directory( + artifact_names=[ + deploy_parameters_artifact, + create_rollup_parameters_artifact, + contract_deployment_script_artifact, + ] + ), + }, + ), + ) + + # TODO: Check if the contracts were already initialized.. I'm leaving this here for now, but it's not useful!! + contract_init_stat = plan.exec( + service_name="contracts" + args["deployment_suffix"], + acceptable_codes=[0, 1], + recipe=ExecRecipe(command=["stat", "/opt/zkevm/.init-complete.lock"]), + ) + + # Deploy contracts + plan.exec( + service_name="contracts" + args["deployment_suffix"], + recipe=ExecRecipe( + command=[ + "git", + "clone", + "--depth", + "1", + "-b", + args["zkevm_contracts_branch"], + args["zkevm_contracts_repo"], + "/opt/zkevm-contracts", + ] + ), + ) + plan.exec( + service_name="contracts" + args["deployment_suffix"], + recipe=ExecRecipe( + command=["chmod", "a+x", "/opt/contract-deploy/run-contract-setup.sh"] + ), + ) + plan.print("Running zkEVM contract deployment. This might take some time...") + plan.exec( + service_name="contracts" + args["deployment_suffix"], + recipe=ExecRecipe(command=["/opt/contract-deploy/run-contract-setup.sh"]), + ) + else: + plan.print("Skipping stage " + str(DEPLOYMENT_STAGE.configure_l1)) + + # Get the genesis file. genesis_artifact = plan.store_service_files( - service_name="contracts" + args["deployment_suffix"], - src="/opt/zkevm/genesis.json", name="genesis", - ) - sequencer_keystore_artifact = plan.store_service_files( service_name="contracts" + args["deployment_suffix"], - src="/opt/zkevm/sequencer.keystore", - name="sequencer-keystore", - ) - aggregator_keystore_artifact = plan.store_service_files( - service_name="contracts" + args["deployment_suffix"], - src="/opt/zkevm/aggregator.keystore", - name="aggregator-keystore", + src="/opt/zkevm/genesis.json", ) - # Start databases - event_db_init_script = plan.upload_files( - src="./templates/databases/event-db-init.sql", - name="event-db-init.sql" + args["deployment_suffix"], - ) - prover_db_init_script = plan.upload_files( - src="./templates/databases/prover-db-init.sql", - name="prover-db-init.sql" + args["deployment_suffix"], - ) - zkevm_databases_package.start_node_databases( - plan, args, event_db_init_script, prover_db_init_script - ) - zkevm_databases_package.start_peripheral_databases(plan, args) + ## STAGE 3: Deploy trusted / central environment + if DEPLOYMENT_STAGE.deploy_central_environment in args["stages"]: + plan.print( + "Executing stage " + str(DEPLOYMENT_STAGE.deploy_central_environment) + ) - # Start prover - zkevm_prover_package.start_prover(plan, args, prover_config_artifact) + # Start databases + event_db_init_script = plan.upload_files( + src="./templates/databases/event-db-init.sql", + name="event-db-init.sql" + args["deployment_suffix"], + ) + prover_db_init_script = plan.upload_files( + src="./templates/databases/prover-db-init.sql", + name="prover-db-init.sql" + args["deployment_suffix"], + ) + zkevm_databases_package.start_node_databases( + plan, args, event_db_init_script, prover_db_init_script + ) + zkevm_databases_package.start_peripheral_databases(plan, args) - # Start AggLayer - plan.add_service( - name="zkevm-agglayer" + args["deployment_suffix"], - config=ServiceConfig( - image=args["zkevm_agglayer_image"], - ports={ - "agglayer": PortSpec( - args["zkevm_agglayer_port"], application_protocol="http" - ), - "prometheus": PortSpec( - args["zkevm_prometheus_port"], application_protocol="http" - ), - }, - files={ - "/etc/": zkevm_configs, + # Start prover + prover_config_template = read_file( + src="./templates/trusted-node/prover-config.json" + ) + prover_config_artifact = plan.render_templates( + config={ + "prover-config.json": struct(template=prover_config_template, data=args) }, - entrypoint=[ - "/app/agglayer", - ], - cmd=["run", "--cfg", "/etc/zkevm/agglayer-config.toml"], - ), + name="prover-config-artifact", + ) + zkevm_prover_package.start_prover(plan, args, prover_config_artifact) + + # Start the zkevm node components + sequencer_keystore_artifact = plan.store_service_files( + name="sequencer-keystore", + service_name="contracts" + args["deployment_suffix"], + src="/opt/zkevm/sequencer.keystore", + ) + aggregator_keystore_artifact = plan.store_service_files( + name="aggregator-keystore", + service_name="contracts" + args["deployment_suffix"], + src="/opt/zkevm/aggregator.keystore", + ) + start_node_components( + plan, + args, + genesis_artifact, + sequencer_keystore_artifact, + aggregator_keystore_artifact, + ) + else: + plan.print("Skipping stage " + str(DEPLOYMENT_STAGE.deploy_central_environment)) + + ## STAGE 4: Deploy CDK/Bridge infra + if DEPLOYMENT_STAGE.deploy_cdk_bridge_infra in args["stages"]: + plan.print("Executing stage " + str(DEPLOYMENT_STAGE.deploy_cdk_bridge_infra)) + zkevm_bridge_service = start_bridge_service(plan, args) + start_bridge_ui(plan, args, zkevm_bridge_service) + start_agglayer(plan, args) + start_dac(plan, args) + else: + plan.print("Skipping stage " + str(DEPLOYMENT_STAGE.deploy_cdk_bridge_infra)) + + ## STAGE 5: Deploy permissionless node + if DEPLOYMENT_STAGE.deploy_permissionless_node in args["stages"]: + plan.print( + "Executing stage " + str(DEPLOYMENT_STAGE.deploy_permissionless_node) + ) + + # Note that an additional suffix will be added to the permissionless services. + permissionless_args = dict(args) # Create a shallow copy of args. + permissionless_args["deployment_suffix"] = "-pless" + args["deployment_suffix"] + permissionless_args["genesis_artifact"] = genesis_artifact + zkevm_permissionless_node_package.run(plan, permissionless_args) + else: + plan.print("Skipping stage " + str(DEPLOYMENT_STAGE.deploy_permissionless_node)) + + +def start_node_components( + plan, + args, + genesis_artifact, + sequencer_keystore_artifact, + aggregator_keystore_artifact, +): + # Create node configuration file. + config_template = read_file(src="./templates/trusted-node/node-config.toml") + config_artifact = plan.render_templates( + config={"node-config.toml": struct(template=config_template, data=args)}, + name="trusted-node-config", ) - # Start DAC - plan.add_service( - name="zkevm-dac" + args["deployment_suffix"], - config=ServiceConfig( - image=args["zkevm_dac_image"], - ports={ - "dac": PortSpec(args["zkevm_dac_port"], application_protocol="http"), - # Does the DAC have prometheus?! - # "prometheus": PortSpec( - # args["zkevm_prometheus_port"], application_protocol="http" - # ), - }, - files={ - "/etc/": zkevm_configs, - }, - entrypoint=[ - "/app/cdk-data-availability", - ], - cmd=["run", "--cfg", "/etc/zkevm/dac-config.toml"], - ), + # Deploy components. + service_map = {} + service_map["synchronizer"] = zkevm_node_package.start_synchronizer( + plan, args, config_artifact, genesis_artifact + ) + service_map["sequencer"] = zkevm_node_package.start_sequencer( + plan, args, config_artifact, genesis_artifact + ) + service_map["sequence_sender"] = zkevm_node_package.start_sequence_sender( + plan, args, config_artifact, genesis_artifact, sequencer_keystore_artifact + ) + service_map["start_aggregator"] = zkevm_node_package.start_aggregator( + plan, + args, + config_artifact, + genesis_artifact, + sequencer_keystore_artifact, + aggregator_keystore_artifact, + ) + service_map["rpc"] = zkevm_node_package.start_rpc( + plan, args, config_artifact, genesis_artifact ) - # Start the zkevm node components - service_map = start_node_components( + service_map["eth_tx_manager"] = zkevm_node_package.start_eth_tx_manager( plan, args, + config_artifact, genesis_artifact, sequencer_keystore_artifact, aggregator_keystore_artifact, ) - # Start bridge - zkevm_bridge_service = plan.add_service( + service_map["l2_gas_pricer"] = zkevm_node_package.start_l2_gas_pricer( + plan, args, config_artifact, genesis_artifact + ) + return service_map + + +def start_bridge_service(plan, args): + # Create bridge config. + bridge_config_template = read_file(src="./templates/bridge-config.toml") + rollup_manager_block_number = get_key_from_config( + plan, args, "deploymentRollupManagerBlockNumber" + ) + zkevm_global_exit_root_address = get_key_from_config( + plan, args, "polygonZkEVMGlobalExitRootAddress" + ) + zkevm_bridge_address = get_key_from_config(plan, args, "polygonZkEVMBridgeAddress") + zkevm_rollup_manager_address = get_key_from_config( + plan, args, "polygonRollupManagerAddress" + ) + claimtx_keystore_artifact = plan.store_service_files( + name="claimtxmanager-keystore", + service_name="contracts" + args["deployment_suffix"], + src="/opt/zkevm/claimtxmanager.keystore", + ) + zkevm_rollup_address = get_key_from_config(plan, args, "rollupAddress") + bridge_config_artifact = plan.render_templates( + name="bridge-config-artifact", + config={ + "bridge-config.toml": struct( + template=bridge_config_template, + data={ + "deployment_suffix": args["deployment_suffix"], + "l1_rpc_url": args["l1_rpc_url"], + "zkevm_l2_keystore_password": args["zkevm_l2_keystore_password"], + # addresses + "rollup_manager_block_number": rollup_manager_block_number, + "zkevm_bridge_address": zkevm_bridge_address, + "zkevm_global_exit_root_address": zkevm_global_exit_root_address, + "zkevm_rollup_manager_address": zkevm_rollup_manager_address, + "zkevm_rollup_address": zkevm_rollup_address, + # bridge db + "zkevm_db_bridge_hostname": args["zkevm_db_bridge_hostname"], + "zkevm_db_bridge_name": args["zkevm_db_bridge_name"], + "zkevm_db_bridge_user": args["zkevm_db_bridge_user"], + "zkevm_db_bridge_password": args["zkevm_db_bridge_password"], + # ports + "zkevm_db_postgres_port": args["zkevm_db_postgres_port"], + "zkevm_bridge_grpc_port": args["zkevm_bridge_grpc_port"], + "zkevm_bridge_rpc_port": args["zkevm_bridge_rpc_port"], + "zkevm_rpc_http_port": args["zkevm_rpc_http_port"], + }, + ) + }, + ) + + # Start bridge service. + return plan.add_service( name="zkevm-bridge-service" + args["deployment_suffix"], config=ServiceConfig( - image=args["zkevm_bridge_service_image"], + image="hermeznetwork/zkevm-bridge-service:v0.4.2", ports={ "bridge-rpc": PortSpec( args["zkevm_bridge_rpc_port"], application_protocol="http" @@ -267,7 +350,9 @@ def run(plan, args): ), }, files={ - "/etc/": zkevm_configs, + "/etc/zkevm": Directory( + artifact_names=[bridge_config_artifact, claimtx_keystore_artifact] + ), }, entrypoint=[ "/app/zkevm-bridge", @@ -276,32 +361,21 @@ def run(plan, args): ), ) - # Fetch addresses - zkevm_bridge_address = extract_json_key_from_service( - plan, - "contracts" + args["deployment_suffix"], - "/opt/zkevm/bridge-config.toml", - "PolygonBridgeAddress", - ) # "L2PolygonBridgeAddresses" - rollup_manager_address = extract_json_key_from_service( - plan, - "contracts" + args["deployment_suffix"], - "/opt/zkevm/bridge-config.toml", - "PolygonRollupManagerAddress", + +def start_bridge_ui(plan, args, bridge_service): + l1_eth_service = plan.get_service(name="el-1-geth-lighthouse") + zkevm_node_rpc = plan.get_service(name="zkevm-node-rpc" + args["deployment_suffix"]) + zkevm_bridge_address = get_key_from_config( + plan, args, "polygonZkEVMGlobalExitRootAddress" ) - polygon_zkevm_address = extract_json_key_from_service( - plan, - "contracts" + args["deployment_suffix"], - "/opt/zkevm/bridge-config.toml", - "PolygonZkEVMAddress", + zkevm_rollup_manager_address = get_key_from_config( + plan, args, "polygonRollupManagerAddress" ) - l1_eth_service = plan.get_service(name="el-1-geth-lighthouse") - - # Fetch port - polygon_zkevm_rpc_http_port = service_map["rpc"].ports["http-rpc"] - bridge_api_http_port = zkevm_bridge_service.ports["bridge-rpc"] + zkevm_rollup_address = get_key_from_config(plan, args, "rollupAddress") + polygon_zkevm_rpc_http_port = zkevm_node_rpc.ports["http-rpc"] + bridge_api_http_port = bridge_service.ports["bridge-rpc"] - # Start bridge-ui + # Start bridge UI. plan.add_service( name="zkevm-bridge-ui" + args["deployment_suffix"], config=ServiceConfig( @@ -316,16 +390,17 @@ def run(plan, args): l1_eth_service.ip_address, l1_eth_service.ports["rpc"].number ), "POLYGON_ZK_EVM_RPC_URL": "http://{}:{}".format( - service_map["rpc"].ip_address, polygon_zkevm_rpc_http_port.number + zkevm_node_rpc.ip_address, + polygon_zkevm_rpc_http_port.number, ), "BRIDGE_API_URL": "http://{}:{}".format( - zkevm_bridge_service.ip_address, bridge_api_http_port.number + bridge_service.ip_address, bridge_api_http_port.number ), "ETHEREUM_BRIDGE_CONTRACT_ADDRESS": zkevm_bridge_address, "POLYGON_ZK_EVM_BRIDGE_CONTRACT_ADDRESS": zkevm_bridge_address, "ETHEREUM_FORCE_UPDATE_GLOBAL_EXIT_ROOT": "true", - "ETHEREUM_PROOF_OF_EFFICIENCY_CONTRACT_ADDRESS": polygon_zkevm_address, - "ETHEREUM_ROLLUP_MANAGER_ADDRESS": rollup_manager_address, + "ETHEREUM_PROOF_OF_EFFICIENCY_CONTRACT_ADDRESS": zkevm_rollup_address, + "ETHEREUM_ROLLUP_MANAGER_ADDRESS": zkevm_rollup_manager_address, "ETHEREUM_EXPLORER_URL": args["ethereum_explorer"], "POLYGON_ZK_EVM_EXPLORER_URL": args["polygon_zkevm_explorer"], "POLYGON_ZK_EVM_NETWORK_ID": "1", @@ -338,74 +413,144 @@ def run(plan, args): ), ) - # Start default permissionless node. - # Note that an additional suffix will be added to the services. - permissionless_args = dict(args) # Create a shallow copy of args. - permissionless_args["deployment_suffix"] = "-pless" + args["deployment_suffix"] - permissionless_args["genesis_artifact"] = genesis_artifact - zkevm_permissionless_node_package.run(plan, permissionless_args) - -def start_node_components( - plan, - args, - genesis_artifact, - sequencer_keystore_artifact, - aggregator_keystore_artifact, -): - # Create node configuration file. - config_template = read_file(src="./templates/trusted-node/node-config.toml") - config_artifact = plan.render_templates( - config={"node-config.toml": struct(template=config_template, data=args)}, - name="trusted-node-config", +def start_agglayer(plan, args): + # Create agglayer config. + agglayer_config_template = read_file(src="./templates/agglayer-config.toml") + rollup_manager_address = get_key_from_config( + plan, args, "polygonRollupManagerAddress" ) - - service_map = {} - # Deploy components. - service_map["synchronizer"] = zkevm_node_package.start_synchronizer( - plan, args, config_artifact, genesis_artifact + agglayer_keystore_artifact = plan.store_service_files( + name="agglayer-keystore", + service_name="contracts" + args["deployment_suffix"], + src="/opt/zkevm/agglayer.keystore", ) - service_map["sequencer"] = zkevm_node_package.start_sequencer( - plan, args, config_artifact, genesis_artifact + agglayer_config_artifact = plan.render_templates( + name="agglayer-config-artifact", + config={ + "agglayer-config.toml": struct( + template=agglayer_config_template, + # TODO: Organize those args. + data={ + "deployment_suffix": args["deployment_suffix"], + "l1_network_id": args["l1_network_id"], + "l1_rpc_url": args["l1_rpc_url"], + "zkevm_l2_keystore_password": args["zkevm_l2_keystore_password"], + # addresses + "rollup_manager_address": rollup_manager_address, + # agglayer db + "zkevm_db_agglayer_hostname": args["zkevm_db_agglayer_hostname"], + "zkevm_db_agglayer_name": args["zkevm_db_agglayer_name"], + "zkevm_db_agglayer_user": args["zkevm_db_agglayer_user"], + "zkevm_db_agglayer_password": args["zkevm_db_agglayer_password"], + # ports + "zkevm_db_postgres_port": args["zkevm_db_postgres_port"], + "zkevm_rpc_http_port": args["zkevm_rpc_http_port"], + "zkevm_agglayer_port": args["zkevm_agglayer_port"], + "zkevm_prometheus_port": args["zkevm_prometheus_port"], + }, + ) + }, ) - service_map["sequence_sender"] = zkevm_node_package.start_sequence_sender( - plan, args, config_artifact, genesis_artifact, sequencer_keystore_artifact + + # Start agglayer service. + plan.add_service( + name="zkevm-agglayer" + args["deployment_suffix"], + config=ServiceConfig( + image=args["zkevm_agglayer_image"], + ports={ + "agglayer": PortSpec( + args["zkevm_agglayer_port"], application_protocol="http" + ), + "prometheus": PortSpec( + args["zkevm_prometheus_port"], application_protocol="http" + ), + }, + files={ + "/etc/zkevm": Directory( + artifact_names=[ + agglayer_config_artifact, + agglayer_keystore_artifact, + ] + ), + }, + entrypoint=[ + "/app/agglayer", + ], + cmd=["run", "--cfg", "/etc/zkevm/agglayer-config.toml"], + ), ) - service_map["start_aggregator"] = zkevm_node_package.start_aggregator( - plan, - args, - config_artifact, - genesis_artifact, - sequencer_keystore_artifact, - aggregator_keystore_artifact, + + +def start_dac(plan, args): + # Create DAC config. + dac_config_template = read_file(src="./templates/dac-config.toml") + rollup_address = get_key_from_config(plan, args, "rollupAddress") + polygon_data_committee_address = get_key_from_config( + plan, args, "polygonDataCommitteeAddress" ) - service_map["rpc"] = zkevm_node_package.start_rpc( - plan, args, config_artifact, genesis_artifact + dac_keystore_artifact = plan.store_service_files( + name="dac-keystore", + service_name="contracts" + args["deployment_suffix"], + src="/opt/zkevm/dac.keystore", ) - service_map["eth_tx_manager"] = zkevm_node_package.start_eth_tx_manager( - plan, - args, - config_artifact, - genesis_artifact, - sequencer_keystore_artifact, - aggregator_keystore_artifact, + dac_config_artifact = plan.render_templates( + name="dac-config-artifact", + config={ + "dac-config.toml": struct( + template=dac_config_template, + # TODO: Organize those args. + data={ + "deployment_suffix": args["deployment_suffix"], + "l1_rpc_url": args["l1_rpc_url"], + "l1_ws_url": args["l1_ws_url"], + "zkevm_l2_keystore_password": args["zkevm_l2_keystore_password"], + # addresses + "rollup_address": rollup_address, + "polygon_data_committee_address": polygon_data_committee_address, + # dac db + "zkevm_db_dac_hostname": args["zkevm_db_dac_hostname"], + "zkevm_db_dac_name": args["zkevm_db_dac_name"], + "zkevm_db_dac_user": args["zkevm_db_dac_user"], + "zkevm_db_dac_password": args["zkevm_db_dac_password"], + # ports + "zkevm_db_postgres_port": args["zkevm_db_postgres_port"], + "zkevm_dac_port": args["zkevm_dac_port"], + }, + ) + }, ) - service_map["l2_gas_pricer"] = zkevm_node_package.start_l2_gas_pricer( - plan, args, config_artifact, genesis_artifact + # Start DAC service. + plan.add_service( + name="zkevm-dac" + args["deployment_suffix"], + config=ServiceConfig( + image=args["zkevm_dac_image"], + ports={ + "dac": PortSpec(args["zkevm_dac_port"], application_protocol="http"), + # Does the DAC have prometheus?! + # "prometheus": PortSpec( + # args["zkevm_prometheus_port"], application_protocol="http" + # ), + }, + files={ + "/etc/zkevm": Directory( + artifact_names=[dac_config_artifact, dac_keystore_artifact] + ), + }, + entrypoint=[ + "/app/cdk-data-availability", + ], + cmd=["run", "--cfg", "/etc/zkevm/dac-config.toml"], + ), ) - return service_map -def extract_json_key_from_service(plan, service_name, filename, key): - plan.print("Extracting contract addresses and ports...") - exec_recipe = ExecRecipe( - command=[ - "/bin/sh", - "-c", - "cat {} | grep -w '{}' | xargs -n1 | tail -1".format(filename, key), - ] +def get_key_from_config(plan, args, key): + return service_package.extract_json_key_from_service( + plan, + "contracts" + args["deployment_suffix"], + "/opt/zkevm/combined.json", + key, ) - result = plan.exec(service_name=service_name, recipe=exec_recipe) - return result["output"] diff --git a/params.yml b/params.yml index 09b62f13..c193cdd5 100644 --- a/params.yml +++ b/params.yml @@ -5,6 +5,11 @@ # Note: It should be a string. deployment_suffix: "-001" +# The deployment process is divided into various stages. +# The `stages` parameter indicates the specific stages you wish the deployment to proceed through. +# By default, it will execute all the stages. +stages: [1, 2, 3, 4, 5] + # Docker images and repositories used to spin up services. zkevm_prover_image: hermeznetwork/zkevm-prover:v5.0.7 zkevm_node_image: 0xpolygon/cdk-validium-node:0.6.2-cdk diff --git a/templates/agglayer-config.toml b/templates/agglayer-config.toml index 7425e812..a365bea6 100644 --- a/templates/agglayer-config.toml +++ b/templates/agglayer-config.toml @@ -37,7 +37,7 @@ KMSKeyName = "" # Disable for local [L1] ChainID = {{.l1_network_id}} NodeURL = "{{.l1_rpc_url}}" -RollupManagerContract = "###zkevm_l1_rollup_manager_addr###" +RollupManagerContract = "{{.rollup_manager_address}}" [Telemetry] PrometheusAddr = "0.0.0.0:{{.zkevm_prometheus_port}}" \ No newline at end of file diff --git a/templates/bridge-config.toml b/templates/bridge-config.toml index 4bf602d6..82b50cf1 100644 --- a/templates/bridge-config.toml +++ b/templates/bridge-config.toml @@ -41,12 +41,12 @@ BridgeVersion = "v1" MaxConns = 20 [NetworkConfig] -GenBlockNumber = "###zkevm_l1_deployment_block###" -PolygonBridgeAddress = "###zkevm_l1_bridge_addr###" -PolygonZkEVMGlobalExitRootAddress = "###zkevm_l1_global_exit_root_addr###" -PolygonRollupManagerAddress = "###zkevm_l1_rollup_manager_addr###" -PolygonZkEVMAddress = "###zkevm_l1_rollup_addr###" -L2PolygonBridgeAddresses = ["###zkevm_l1_bridge_addr###"] +GenBlockNumber = "{{.rollup_manager_block_number}}" +PolygonBridgeAddress = "{{.zkevm_bridge_address}}" +PolygonZkEVMGlobalExitRootAddress = "{{.zkevm_global_exit_root_address}}" +PolygonRollupManagerAddress = "{{.zkevm_rollup_manager_address}}" +PolygonZkEVMAddress = "{{.zkevm_rollup_address}}" +L2PolygonBridgeAddresses = ["{{.zkevm_bridge_address}}"] [ClaimTxManager] FrequencyToMonitorTxs = "5s" diff --git a/templates/dac-config.toml b/templates/dac-config.toml index 7a013409..bcc73fa8 100644 --- a/templates/dac-config.toml +++ b/templates/dac-config.toml @@ -3,8 +3,8 @@ PrivateKey = {Path = "/etc/zkevm/dac.keystore", Password = "{{.zkevm_l2_keystore [L1] WsURL = "{{.l1_ws_url}}" RpcURL = "{{.l1_rpc_url}}" -PolygonValidiumAddress = "###rollupAddress###" -DataCommitteeAddress = "###polygonDataCommitteeAddress###" +PolygonValidiumAddress = "{{.rollup_address}}" +DataCommitteeAddress = "{{.polygon_data_committee_address}}" Timeout = "1m" RetryPeriod = "5s" BlockBatchSize = "64" diff --git a/templates/run-contract-setup.sh b/templates/run-contract-setup.sh index 60f953e9..37564669 100755 --- a/templates/run-contract-setup.sh +++ b/templates/run-contract-setup.sh @@ -94,7 +94,6 @@ cp /opt/zkevm-contracts/deployment/v2/deploy_*.json /opt/zkevm/ cp /opt/zkevm-contracts/deployment/v2/genesis.json /opt/zkevm/ cp /opt/zkevm-contracts/deployment/v2/create_rollup_output.json /opt/zkevm/ cp /opt/zkevm-contracts/deployment/v2/create_rollup_parameters.json /opt/zkevm/ -cp /opt/contract-deploy/*-config.* /opt/zkevm/ popd pushd /opt/zkevm/ || exit 1 @@ -127,28 +126,6 @@ jq --slurpfile c combined.json '.L1Config.polygonRollupManagerAddress = $c[0].po jq --slurpfile c combined.json '.L1Config.polTokenAddress = $c[0].polTokenAddress' genesis.json > g.json; mv g.json genesis.json jq --slurpfile c combined.json '.L1Config.polygonZkEVMAddress = $c[0].rollupAddress' genesis.json > g.json; mv g.json genesis.json -# note this particular setting is different for the bridge service!! -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.NetworkConfig.GenBlockNumber = $c[0].deploymentRollupManagerBlockNumber' bridge-config.toml > b.json; mv b.json bridge-config.toml -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.NetworkConfig.PolygonBridgeAddress = $c[0].polygonZkEVMBridgeAddress' bridge-config.toml > b.json; mv b.json bridge-config.toml -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.NetworkConfig.PolygonZkEVMGlobalExitRootAddress = $c[0].polygonZkEVMGlobalExitRootAddress' bridge-config.toml > b.json; mv b.json bridge-config.toml -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.NetworkConfig.PolygonRollupManagerAddress = $c[0].polygonRollupManagerAddress' bridge-config.toml > b.json; mv b.json bridge-config.toml -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.NetworkConfig.PolygonZkEVMAddress = $c[0].rollupAddress' bridge-config.toml > b.json; mv b.json bridge-config.toml -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.NetworkConfig.L2PolygonBridgeAddresses = [$c[0].polygonZkEVMBridgeAddress]' bridge-config.toml > b.json; mv b.json bridge-config.toml - -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.L1.RollupManagerContract = $c[0].polygonRollupManagerAddress' agglayer-config.toml > a.json; mv a.json agglayer-config.toml - -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.L1.PolygonValidiumAddress = $c[0].rollupAddress' dac-config.toml > a.json; mv a.json dac-config.toml -# shellcheck disable=SC2016 -tomlq --slurpfile c combined.json -t '.L1.DataCommitteeAddress = $c[0].polygonDataCommitteeAddress' dac-config.toml > a.json; mv a.json dac-config.toml - # The sequencer needs to pay POL when it sequences batches. This gets # refunded when the batches are proved. In order for this to work the # rollup address must be approved transfer the sequencers POL