From b6de4bab764900c7110658aab8dc667d927380b0 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 12 Jun 2024 10:59:16 -0700 Subject: [PATCH 01/10] Adding empty registry address field in env.common --- env/env.common | 3 +++ 1 file changed, 3 insertions(+) diff --git a/env/env.common b/env/env.common index d955d0c..7e850b5 100644 --- a/env/env.common +++ b/env/env.common @@ -1,6 +1,9 @@ # Networking configuration ARTIFACTS_URI=http://artifacts:8080 RPC_URI=http://ethereum:8545 +# Registry address defaults to looking in artifacts +# This can be overwritten by specifying here. +REGISTRY_ADDRESS= # Migration configuration ADMIN=0x6043bF4ec8834F918242BaCA0F1F3805F2a2988D From 09568591dae97165993438aac3a57c99db4818b9 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 12 Jun 2024 13:53:09 -0700 Subject: [PATCH 02/10] Adding support for running service bots on remote chain --- docker-compose.agent0-bots.yaml | 2 +- docker-compose.anvil.yaml | 15 ++++++--- env/env.anvil | 7 +++++ env/env.bots | 5 +++ env/env.common | 5 --- setup_env.sh | 54 +++++++++++++++++++++------------ 6 files changed, 59 insertions(+), 29 deletions(-) create mode 100644 env/env.bots diff --git a/docker-compose.agent0-bots.yaml b/docker-compose.agent0-bots.yaml index 0fab327..00ad6bf 100644 --- a/docker-compose.agent0-bots.yaml +++ b/docker-compose.agent0-bots.yaml @@ -13,7 +13,7 @@ services: profiles: - "service-bot" command: | - /bin/sh -c "sleep 1; python scripts/fuzz_bot_invariant_checks.py --pool-check-sleep-blocks 30 --infra" + /bin/sh -c "sleep 1; python scripts/invariant_checks.py --pool-check-sleep-blocks 30 --infra" env_file: - .env diff --git a/docker-compose.anvil.yaml b/docker-compose.anvil.yaml index a59dd89..5ef0024 100644 --- a/docker-compose.anvil.yaml +++ b/docker-compose.anvil.yaml @@ -1,6 +1,8 @@ services: ethereum: image: ${DEVNET_IMAGE} + profiles: + - "anvil" # without --block-time we're in "automine" mode and blocks are mined # as soon as transactions are received, or with manual commands. # this is "hecking fast" mode. @@ -13,9 +15,18 @@ services: # read calls. mine: image: curlimages/curl:latest + profiles: + - "anvil" command: | /bin/sh -c "sleep 2; curl -X POST -H \"Content-Type: application/json\" --data '{\"jsonrpc\":\"2.0\", \"method\":\"anvil_mine\", \"params\":[], \"id\":1}' ${RPC_URI}" + artifacts: + image: ${ARTIFACTS_IMAGE} + profiles: + - "anvil" + volumes: + - artifacts:/var/www/artifacts/ + # Set the block timestamp interval timestamp_interval: image: curlimages/curl:latest @@ -35,10 +46,6 @@ services: env_file: - .env - artifacts: - image: ${ARTIFACTS_IMAGE} - volumes: - - artifacts:/var/www/artifacts/ volumes: artifacts: diff --git a/env/env.anvil b/env/env.anvil index ec87928..17a7158 100644 --- a/env/env.anvil +++ b/env/env.anvil @@ -1,3 +1,10 @@ +# The artifacts uri for the artifacts container +ARTIFACTS_URI=http://artifacts:8080 + # Choose a chain id for the eth node # Common choices are: 42069 for CloudChain or 31337 for local development CHAIN_ID=42069 + +# Migration configuration +ADMIN=0x6043bF4ec8834F918242BaCA0F1F3805F2a2988D +ADMIN_PRIVATE_KEY=0x0f1616bb59774b929ba5256c2204123d9dedc4c142d013179b90f929997d119b \ No newline at end of file diff --git a/env/env.bots b/env/env.bots new file mode 100644 index 0000000..c791395 --- /dev/null +++ b/env/env.bots @@ -0,0 +1,5 @@ +# The private key of the account that's paying gas for checkpoints +CHECKPOINT_BOT_KEY= + +# The rollbar api key +ROLLBAR_API_KEY= diff --git a/env/env.common b/env/env.common index 7e850b5..5cc4a31 100644 --- a/env/env.common +++ b/env/env.common @@ -1,10 +1,5 @@ # Networking configuration -ARTIFACTS_URI=http://artifacts:8080 RPC_URI=http://ethereum:8545 # Registry address defaults to looking in artifacts # This can be overwritten by specifying here. REGISTRY_ADDRESS= - -# Migration configuration -ADMIN=0x6043bF4ec8834F918242BaCA0F1F3805F2a2988D -ADMIN_PRIVATE_KEY=0x0f1616bb59774b929ba5256c2204123d9dedc4c142d013179b90f929997d119b diff --git a/setup_env.sh b/setup_env.sh index 976f595..d1c0e5d 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -7,20 +7,21 @@ if [[ $# -eq 0 ]] || [[ "$1" == "--help" ]]; then echo "Usage: ./setup_env.sh [flags]" echo "Flag Groups:" - echo " --all : Fund accounts and enable all components: anvil, data, postgres, frontend, and ports." - echo " --develop : Fund accounts and enable anvil, data, and ports. Suitable for local development work." + echo " --all : Fund accounts and enable all components." + echo " --develop : Fund accounts and enable anvil, data, and ports. Suitable for local development work." + echo " --remote-service-bots : Runs service bots on a remote chain." echo "Flags:" - echo " --anvil : Spin up an Anvil node, deploy Hyperdrive to it, and serve artifacts on an nginx server." - echo " --blocktime : Sets the anvil node to run in blocktime mode." - echo " --testnet : Uses the testnet hyperdrive image with restricted mint access." - echo " --frontend : Build the frontend container." - echo " --postgres : Runs a postgres db container for storing data." - echo " --data : Runs the data framework, querying the chain and writing to postgres." - echo " --service-bot : Runs checkpoint bot and invariance check bots on the chain." - echo " --random-bot : Runs random bots on the chain." - echo " --ports : Expose docker images to your machine, as specified in env/env.ports." - echo " --fund-accounts : Fund accounts from /accounts/balances.json." - echo " --rate-bot : Yield source will have a dynamic variable rate." + echo " --anvil : Spin up an Anvil node, deploy Hyperdrive to it, and serve artifacts on an nginx server." + echo " --blocktime : Sets the anvil node to run in blocktime mode." + echo " --testnet : Uses the testnet hyperdrive image with restricted mint access." + echo " --frontend : Build the frontend container." + echo " --postgres : Runs a postgres db container for storing data." + echo " --data : Runs the data framework, querying the chain and writing to postgres." + echo " --service-bot : Runs checkpoint bot and invariance check bots on the chain." + echo " --random-bot : Runs random bots on the chain." + echo " --ports : Expose docker images to your machine, as specified in env/env.ports." + echo " --fund-accounts : Fund accounts from /accounts/balances.json." + echo " --rate-bot : Yield source will have a dynamic variable rate." exit 0 fi @@ -62,6 +63,11 @@ while [[ $# -gt 0 ]]; do PORTS=true FUND_ACCOUNTS=true ;; + --remote-service-bots) + POSTGRES=true + PORTS=true + SERVICE_BOT=true + ;; # Flags --anvil) ANVIL=true @@ -142,6 +148,7 @@ echo $full_compose_files >>.env # Set up the COMPOSE_PROFILES environment variable. This toggles which layers # should be started. +anvil_profile="anvil" blocktime_profile="blocktime" frontend_profile="frontend" postgres_profile="postgres" @@ -151,6 +158,9 @@ random_bot_profile="random-bot" rate_bot_profile="rate-bot" fund_accounts_profile="fund-accounts" full_compose_profiles="COMPOSE_PROFILES=" +if $ANVIL; then + full_compose_profiles+="$anvil_profile," +fi if $BLOCKTIME; then full_compose_profiles+="$blocktime_profile," fi @@ -187,6 +197,13 @@ echo $full_compose_profiles >>.env # cat env.common and env.ports to the .env file echo "" >>.env cat env/env.common >> .env + +# optionally cat env.anvil to .env file if --anvil +if $ANVIL; then + echo "" >>.env + cat env/env.anvil >>.env +fi + echo "" >>.env cat env/env.ports >>.env @@ -196,12 +213,6 @@ echo "" >>.env cat env/env.images >>.env source env/env.images -# optionally cat env.anvil to .env file if --anvil -if $ANVIL; then - echo "" >>.env - cat env/env.anvil >>.env -fi - # optionally add an env.time to .env file if --blocktime if $BLOCKTIME; then echo "" >>.env @@ -222,4 +233,9 @@ if $POSTGRES || $DATA || $SERVICE_BOT || $RANDOM_BOT || $RATE_BOT; then cat env/env.postgres >> .env fi +if $SERVICE_BOT; then + echo "" >>.env + cat env/env.bots >> .env +fi + echo "Environment filed created at .env" From 752df111aff6727777c3a64161b1f73927d3c24b Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 12 Jun 2024 13:59:22 -0700 Subject: [PATCH 03/10] Bumping version of agent0 image --- env/env.images | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/env.images b/env/env.images index 60095c7..3488aab 100644 --- a/env/env.images +++ b/env/env.images @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.8 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.8 # Agent0 (bots) -AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.20.2 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.20.3 # Frontend FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-frontend/hyperdrive-frontend:1.6.5 From 7e2d253fa652cda85a22497b6048d6371fb83e05 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Tue, 18 Jun 2024 13:57:07 -0700 Subject: [PATCH 04/10] Updating readme --- README.md | 17 ++++++----------- service-bots.md | 13 +++++++++++++ 2 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 service-bots.md diff --git a/README.md b/README.md index 764c269..3da3fd2 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ compose app. The compose app is composed of several services: - artifacts: A file server that serves up artifacts about the Hyperdrive deployment. - frontend: A full monorepo with a trading UI and supporting packages - data: A service that gathers data from the chain and deploys a data dashboard. +- bots: Bot services for running auto trades on a chain. ## Setup @@ -44,19 +45,19 @@ To select an environment, run `sh setup_env.sh` with one or more of the followin - `--blocktime` : Sets the anvil node to run in blocktime mode. - `--testnet` : Uses the testnet hyperdrive image with restricted mint access. - `--frontend` : Build the frontend container. -- `--data` : Runs the data framework, queries the chain, writes to postgres, and deploys the dashboard. - `--postgres` : Launches a local postgres server for the data pipeline. +- `--data` : Runs the data framework, querying the chain and writing to postgres. +- `--service-bot` : Runs checkpoint bot and invariance check bots on the chain." +- `--random-bot` : Runs random bots on the chain. - `--ports` : Expose docker images to your machine, as specified in `env/env.ports`. - `--fund-accounts` : Fund accounts from `/accounts/balances.json`. -- `--dynamic-rate` : Yield source will have a dynamic variable rate. -- `--fuzzbot` : Runs fuzzbots on the chain. +- `--rate-bot` : Yield source will have a dynamic variable rate. We also support shortcuts for common combinations. The most inclusive tag used will take priority. - `--all` : Fund accounts and enable all components: anvil, data, postgres, frontend, and ports. -- `--competition` : Fund accounts and enable anvil on block time with testnet image, data (without postgres), and ports. Use this for a trading competition deployment. - `--develop` : Fund accounts and enable anvil, data, and ports. Suitable for local development work. -- `--fuzz` : Enable anvil on block time, data, and ports. Runs fuzzbots and outputs crash reports. See [Fuzz Agents](#fuzz-agents) for more details. +- `--remote-service-bots`: Runs service bots on a remote chain. See [Service Bots](./service-bots.md) for more details. You can also change the tags in `env/env.images` to modify which docker image you build from. @@ -76,12 +77,6 @@ See live logs with `docker logs CONTAINER_NAME -f`. Run `docker compose down -v`. The `-v` ensures that storage volumes are deleted. -## Fuzz agents - -We provide a method for launching [agents making random trades](https://github.com/delvtech/agent0/blob/main/scripts/fuzz_bots.py) on Hyperdrive along with the rest of the -infrastructure by providing the `--fuzz` argument to `setup_env.sh`. These agents also support detailed -crash reports in the event that an error is detected, located in the `.crash_report` directory. By default, -the chain will be paused when a crash happens, freezing the anvil state for further analysis. ## Disclaimer diff --git a/service-bots.md b/service-bots.md new file mode 100644 index 0000000..3b1e444 --- /dev/null +++ b/service-bots.md @@ -0,0 +1,13 @@ +# Service Bots +Infra can be used to run service bots on a remote chain (e.g., Sepolia testnet). + +- Setup infra as documented in the [README](README.md#setup). +- Run `./setup_env.sh --remote-service-bots` +- Edit the generated `.env` file and replace the following fields: + - `RPC_URI`: The URI to the RPC for connecting to the chain. + - `REGISTRY_ADDRESS`: The contract address of the hyperdrive registry. + - `CHECKPOINT_BOT_KEY`: The private key of the account that's making checkpoints. + - `ROLLBAR_API_KEY`: The API key for Rollbar for logging. +- Run `docker compose up -d` to bring up bots. + +The above will run (1) checkpoint bots and (2) invariance check bots on all registered pools. From 45c468cd07684e8b4a6f12ba59a7f7554095ead9 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Tue, 18 Jun 2024 15:23:34 -0700 Subject: [PATCH 05/10] Bumping agent0 version --- env/env.images | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/env.images b/env/env.images index 3488aab..85eac22 100644 --- a/env/env.images +++ b/env/env.images @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.8 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.8 # Agent0 (bots) -AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.20.3 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.20.4 # Frontend FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-frontend/hyperdrive-frontend:1.6.5 From 7618b4ff522aa9c565cda42060d0d20b506dfd2d Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 20 Jun 2024 14:53:08 -0700 Subject: [PATCH 06/10] Bumping protocol and agent0 image versions --- env/env.images | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/env/env.images b/env/env.images index 85eac22..61422dd 100644 --- a/env/env.images +++ b/env/env.images @@ -5,8 +5,8 @@ # - edge = The newest image regardless of stability # Anvil -DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:1.0.10 -TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:1.0.10 +DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:1.0.12 +TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:1.0.12 # Infra ARTIFACTS_IMAGE=ghcr.io/delvtech/infra/artifacts:0.0.8 @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.8 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.8 # Agent0 (bots) -AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.20.4 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.0 # Frontend FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-frontend/hyperdrive-frontend:1.6.5 From 910aa50e3ef7b893e31aa015a6b969bba1d0da39 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 20 Jun 2024 17:08:30 -0700 Subject: [PATCH 07/10] Bumping agent0 version --- env/env.images | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/env.images b/env/env.images index 61422dd..e713ef6 100644 --- a/env/env.images +++ b/env/env.images @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.8 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.8 # Agent0 (bots) -AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.0 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.1 # Frontend FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-frontend/hyperdrive-frontend:1.6.5 From 08bb5cf8bc324c4d58ca28a2c5e22b2ba7364df0 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 26 Jun 2024 13:19:24 -0700 Subject: [PATCH 08/10] Always restart service bots if they crash --- docker-compose.agent0-bots.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.agent0-bots.yaml b/docker-compose.agent0-bots.yaml index 00ad6bf..f77a7d1 100644 --- a/docker-compose.agent0-bots.yaml +++ b/docker-compose.agent0-bots.yaml @@ -3,6 +3,7 @@ services: image: ${AGENT0_IMAGE} profiles: - "service-bot" + restart: always command: | /bin/sh -c "sleep 1; python scripts/checkpoint_bots.py --pool-check-sleep-blocks 30 --infra" env_file: @@ -12,6 +13,7 @@ services: image: ${AGENT0_IMAGE} profiles: - "service-bot" + restart: always command: | /bin/sh -c "sleep 1; python scripts/invariant_checks.py --pool-check-sleep-blocks 30 --infra" env_file: From d57779cae3351e3ec724a8629ab1cfbc2d77fb35 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 27 Jun 2024 11:54:30 -0700 Subject: [PATCH 09/10] Version bumps in infra --- env/env.images | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/env/env.images b/env/env.images index e713ef6..e5d2e50 100644 --- a/env/env.images +++ b/env/env.images @@ -5,8 +5,8 @@ # - edge = The newest image regardless of stability # Anvil -DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:1.0.12 -TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:1.0.12 +DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:1.0.13 +TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:1.0.13 # Infra ARTIFACTS_IMAGE=ghcr.io/delvtech/infra/artifacts:0.0.8 @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.8 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.8 # Agent0 (bots) -AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.1 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.8 # Frontend FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-frontend/hyperdrive-frontend:1.6.5 From 36dea78733140d8e65a764bb9a5c84a913724064 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 28 Jun 2024 16:48:19 -0700 Subject: [PATCH 10/10] Bumping image versions --- env/env.images | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/env/env.images b/env/env.images index e5d2e50..db27b02 100644 --- a/env/env.images +++ b/env/env.images @@ -5,8 +5,8 @@ # - edge = The newest image regardless of stability # Anvil -DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:1.0.13 -TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:1.0.13 +DEVNET_IMAGE=ghcr.io/delvtech/hyperdrive/devnet:1.0.14 +TESTNET_IMAGE=ghcr.io/delvtech/hyperdrive/testnet:1.0.14 # Infra ARTIFACTS_IMAGE=ghcr.io/delvtech/infra/artifacts:0.0.8 @@ -14,7 +14,7 @@ FUND_ACCOUNTS_IMAGE=ghcr.io/delvtech/infra/fund-accounts:0.0.8 RATE_BOT_IMAGE=ghcr.io/delvtech/infra/rate-bot:0.0.8 # Agent0 (bots) -AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.8 +AGENT0_IMAGE=ghcr.io/delvtech/agent0/agent0:0.21.9 # Frontend FRONTEND_IMAGE=ghcr.io/delvtech/hyperdrive-frontend/hyperdrive-frontend:1.6.5