From 6a4644ffdec12d528a35d99d9c7ef3f8439cebe0 Mon Sep 17 00:00:00 2001 From: Zach Howard Date: Thu, 9 Jan 2025 17:09:31 -0600 Subject: [PATCH] kt-devnet: Adds fileserver package for serving cannon prestates (#13666) * kt-devnet: Adds fileserver package for serving cannon prestates - deploy fileserver before optimism-package - updates simple.yaml to use the template functions - small refactor of kurtosis Deploy fn to not print an expected optimism-package env * Runs go mod tidy in cannon testdata examples * Use FILESERVER_PACKAGE constant --- cannon/testdata/example/alloc/go.mod | 2 +- cannon/testdata/example/alloc/go.sum | 8 +- cannon/testdata/example/claim/go.mod | 2 +- cannon/testdata/example/claim/go.sum | 8 +- cannon/testdata/example/mt-atomic/go.mod | 1 + cannon/testdata/example/mt-map/go.mod | 1 + cannon/testdata/example/mt-mutex/go.mod | 1 + cannon/testdata/example/mt-oncefunc/go.mod | 1 + cannon/testdata/example/mt-pool/go.mod | 1 + cannon/testdata/example/mt-value/go.mod | 1 + cannon/testdata/example/mt-wg/go.mod | 1 + cannon/testdata/example/utilscheck/go.mod | 1 + cannon/testdata/example/utilscheck2/go.mod | 1 + cannon/testdata/example/utilscheck3/go.mod | 1 + cannon/testdata/example/utilscheck4/go.mod | 1 + go.mod | 6 +- go.sum | 12 +- kurtosis-devnet/cmd/main.go | 124 ++++++++++++++---- kurtosis-devnet/cmd/main_test.go | 27 +++- kurtosis-devnet/fileserver/kurtosis.yml | 4 + kurtosis-devnet/fileserver/main.star | 49 +++++++ .../static_files/nginx/default.conf | 8 ++ kurtosis-devnet/interop.yaml | 20 +-- kurtosis-devnet/justfile | 2 +- kurtosis-devnet/pkg/kurtosis/kurtosis.go | 8 +- kurtosis-devnet/pkg/util/util.go | 48 +++++++ kurtosis-devnet/simple.yaml | 18 ++- 27 files changed, 298 insertions(+), 59 deletions(-) create mode 100644 kurtosis-devnet/fileserver/kurtosis.yml create mode 100644 kurtosis-devnet/fileserver/main.star create mode 100644 kurtosis-devnet/fileserver/static_files/nginx/default.conf create mode 100644 kurtosis-devnet/pkg/util/util.go diff --git a/cannon/testdata/example/alloc/go.mod b/cannon/testdata/example/alloc/go.mod index c364668acc10..ef1555289038 100644 --- a/cannon/testdata/example/alloc/go.mod +++ b/cannon/testdata/example/alloc/go.mod @@ -8,7 +8,7 @@ require github.com/ethereum-optimism/optimism v0.0.0 require ( golang.org/x/crypto v0.28.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.29.0 // indirect ) replace github.com/ethereum-optimism/optimism v0.0.0 => ../../../.. diff --git a/cannon/testdata/example/alloc/go.sum b/cannon/testdata/example/alloc/go.sum index 3251370c60a9..6729b6fb8dc9 100644 --- a/cannon/testdata/example/alloc/go.sum +++ b/cannon/testdata/example/alloc/go.sum @@ -2,11 +2,11 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cannon/testdata/example/claim/go.mod b/cannon/testdata/example/claim/go.mod index e3904911a172..7a599a25a15f 100644 --- a/cannon/testdata/example/claim/go.mod +++ b/cannon/testdata/example/claim/go.mod @@ -8,7 +8,7 @@ require github.com/ethereum-optimism/optimism v0.0.0 require ( golang.org/x/crypto v0.28.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.29.0 // indirect ) replace github.com/ethereum-optimism/optimism v0.0.0 => ../../../.. diff --git a/cannon/testdata/example/claim/go.sum b/cannon/testdata/example/claim/go.sum index 3251370c60a9..6729b6fb8dc9 100644 --- a/cannon/testdata/example/claim/go.sum +++ b/cannon/testdata/example/claim/go.sum @@ -2,11 +2,11 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cannon/testdata/example/mt-atomic/go.mod b/cannon/testdata/example/mt-atomic/go.mod index 042386cf702d..1dca20ecc3a9 100644 --- a/cannon/testdata/example/mt-atomic/go.mod +++ b/cannon/testdata/example/mt-atomic/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/mt-map/go.mod b/cannon/testdata/example/mt-map/go.mod index 7290b372361e..f776ac11cc47 100644 --- a/cannon/testdata/example/mt-map/go.mod +++ b/cannon/testdata/example/mt-map/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/mt-mutex/go.mod b/cannon/testdata/example/mt-mutex/go.mod index 3aceb4c8cebf..3ba69881fe12 100644 --- a/cannon/testdata/example/mt-mutex/go.mod +++ b/cannon/testdata/example/mt-mutex/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/mt-oncefunc/go.mod b/cannon/testdata/example/mt-oncefunc/go.mod index e0f45e8c8790..f6efa8ec9846 100644 --- a/cannon/testdata/example/mt-oncefunc/go.mod +++ b/cannon/testdata/example/mt-oncefunc/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/mt-pool/go.mod b/cannon/testdata/example/mt-pool/go.mod index 06b49a26398e..6af9961abf91 100644 --- a/cannon/testdata/example/mt-pool/go.mod +++ b/cannon/testdata/example/mt-pool/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/mt-value/go.mod b/cannon/testdata/example/mt-value/go.mod index 602687cbcca2..1884af1fe927 100644 --- a/cannon/testdata/example/mt-value/go.mod +++ b/cannon/testdata/example/mt-value/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/mt-wg/go.mod b/cannon/testdata/example/mt-wg/go.mod index 0c10638b3d1d..f1f08f80692d 100644 --- a/cannon/testdata/example/mt-wg/go.mod +++ b/cannon/testdata/example/mt-wg/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/utilscheck/go.mod b/cannon/testdata/example/utilscheck/go.mod index 5a109a7ca8cd..70c078cc5d64 100644 --- a/cannon/testdata/example/utilscheck/go.mod +++ b/cannon/testdata/example/utilscheck/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/utilscheck2/go.mod b/cannon/testdata/example/utilscheck2/go.mod index ee0430168ad1..fcc791e6f294 100644 --- a/cannon/testdata/example/utilscheck2/go.mod +++ b/cannon/testdata/example/utilscheck2/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/utilscheck3/go.mod b/cannon/testdata/example/utilscheck3/go.mod index 3bc116499beb..614ab76cc6fa 100644 --- a/cannon/testdata/example/utilscheck3/go.mod +++ b/cannon/testdata/example/utilscheck3/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/cannon/testdata/example/utilscheck4/go.mod b/cannon/testdata/example/utilscheck4/go.mod index 7f80460beb9e..b39fa610063b 100644 --- a/cannon/testdata/example/utilscheck4/go.mod +++ b/cannon/testdata/example/utilscheck4/go.mod @@ -5,4 +5,5 @@ go 1.22 toolchain go1.22.0 require utils v0.0.0 + replace utils => ../../utils diff --git a/go.mod b/go.mod index 0ab77f180442..952ce3cc79c9 100644 --- a/go.mod +++ b/go.mod @@ -146,7 +146,7 @@ require ( github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230818182330-1a86869414d2 // indirect github.com/kurtosis-tech/kurtosis/contexts-config-store v0.0.0-20230818184218-f4e3e773463b // indirect github.com/kurtosis-tech/kurtosis/grpc-file-transfer/golang v0.0.0-20230803130419-099ee7a4e3dc // indirect - github.com/kurtosis-tech/kurtosis/path-compression v0.0.0-20240307154559-64d2929cd265 // indirect + github.com/kurtosis-tech/kurtosis/path-compression v0.0.0-20250108161014-0819b8ca912f // indirect github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -225,7 +225,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/wlynxg/anet v0.0.4 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect @@ -238,7 +238,7 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.26.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/tools v0.26.0 // indirect google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e // indirect diff --git a/go.sum b/go.sum index 9f41a82465bf..25e77ab59c23 100644 --- a/go.sum +++ b/go.sum @@ -448,8 +448,8 @@ github.com/kurtosis-tech/kurtosis/contexts-config-store v0.0.0-20230818184218-f4 github.com/kurtosis-tech/kurtosis/contexts-config-store v0.0.0-20230818184218-f4e3e773463b/go.mod h1:4pFdrRwDz5R+Fov2ZuTaPhAVgjA2jhGh1Izf832sX7A= github.com/kurtosis-tech/kurtosis/grpc-file-transfer/golang v0.0.0-20230803130419-099ee7a4e3dc h1:7IlEpSehmWcNXOFpNP24Cu5HQI3af7GCBQw//m+LnvQ= github.com/kurtosis-tech/kurtosis/grpc-file-transfer/golang v0.0.0-20230803130419-099ee7a4e3dc/go.mod h1:TOWMQgvAJH/NiWWERGXg/plT9lS7aFcXFxCa0M5sfHo= -github.com/kurtosis-tech/kurtosis/path-compression v0.0.0-20240307154559-64d2929cd265 h1:uSDftcGStwuAjHv8fV2TleNCKSWPvUKe7EaplFG3yBI= -github.com/kurtosis-tech/kurtosis/path-compression v0.0.0-20240307154559-64d2929cd265/go.mod h1:aDMrPeS7Gii8W6SDKSKyrBNgEQAUYidriyeKGf+Ml3I= +github.com/kurtosis-tech/kurtosis/path-compression v0.0.0-20250108161014-0819b8ca912f h1:kys3RDy0uHk+VwYS1mVh48YnogkRTAxUUCV7kpwMNOQ= +github.com/kurtosis-tech/kurtosis/path-compression v0.0.0-20250108161014-0819b8ca912f/go.mod h1:aDMrPeS7Gii8W6SDKSKyrBNgEQAUYidriyeKGf+Ml3I= github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409 h1:YQTATifMUwZEtZYb0LVA7DK2pj8s71iY8rzweuUQ5+g= github.com/kurtosis-tech/stacktrace v0.0.0-20211028211901-1c67a77b5409/go.mod h1:y5weVs5d9wXXHcDA1awRxkIhhHC1xxYJN8a7aXnE6S8= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -800,8 +800,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= @@ -991,8 +991,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/kurtosis-devnet/cmd/main.go b/kurtosis-devnet/cmd/main.go index c457fda9f95b..911f06be15cd 100644 --- a/kurtosis-devnet/cmd/main.go +++ b/kurtosis-devnet/cmd/main.go @@ -9,16 +9,21 @@ import ( "log" "os" "path/filepath" + "strings" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/build" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/api/engine" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/backend" + "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/sources/spec" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/serve" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl" + "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/util" "github.com/urfave/cli/v2" ) +const FILESERVER_PACKAGE = "fileserver" + type config struct { templateFile string dataFile string @@ -138,72 +143,95 @@ func (m *Main) localContractArtifactsOption(server *staticServer) tmpl.TemplateC }) } +type PrestateInfo struct { + URL string `json:"url"` + Hashes map[string]string `json:"hashes"` +} + func (m *Main) localPrestateOption(server *staticServer) tmpl.TemplateContextOptions { prestateBuilder := build.NewPrestateBuilder( build.WithPrestateBaseDir(m.cfg.baseDir), build.WithPrestateDryRun(m.cfg.dryRun), ) - return tmpl.WithFunction("localPrestate", func() (string, error) { + return tmpl.WithFunction("localPrestate", func() (*PrestateInfo, error) { // Create build directory with the final path structure buildDir := filepath.Join(server.dir, "proofs", "op-program", "cannon") if err := os.MkdirAll(buildDir, 0755); err != nil { - return "", fmt.Errorf("failed to create prestate build directory: %w", err) + return nil, fmt.Errorf("failed to create prestate build directory: %w", err) } // Get the relative path from server.dir to buildDir for the URL relPath, err := filepath.Rel(server.dir, buildDir) if err != nil { - return "", fmt.Errorf("failed to get relative path: %w", err) + return nil, fmt.Errorf("failed to get relative path: %w", err) } - url := fmt.Sprintf("%s/%s", server.URL(), relPath) + info := &PrestateInfo{ + URL: fmt.Sprintf("%s/%s", server.URL(), relPath), + Hashes: make(map[string]string), + } if m.cfg.dryRun { - return url, nil + return info, nil } - // Check if we already have prestate files. Typical in interop mode, - // where we have a prestate for each chain. - if dir, _ := os.ReadDir(buildDir); len(dir) > 0 { - return url, nil + // Map of known file prefixes to their keys + fileToKey := map[string]string{ + "prestate-proof.json": "prestate", + "prestate-proof-mt64.json": "prestate-mt64", + "prestate-proof-mt.json": "prestate-mt", } // Build all prestate files directly in the target directory if err := prestateBuilder.Build(buildDir); err != nil { - return "", fmt.Errorf("failed to build prestates: %w", err) + return nil, fmt.Errorf("failed to build prestates: %w", err) } - // Find all prestate-proof*.json files + // Find and process all prestate files matches, err := filepath.Glob(filepath.Join(buildDir, "prestate-proof*.json")) if err != nil { - return "", fmt.Errorf("failed to find prestate files: %w", err) + return nil, fmt.Errorf("failed to find prestate files: %w", err) } // Process each file to rename it to its hash for _, filePath := range matches { content, err := os.ReadFile(filePath) if err != nil { - return "", fmt.Errorf("failed to read prestate %s: %w", filepath.Base(filePath), err) + return nil, fmt.Errorf("failed to read prestate %s: %w", filepath.Base(filePath), err) } var data struct { Pre string `json:"pre"` } if err := json.Unmarshal(content, &data); err != nil { - return "", fmt.Errorf("failed to parse prestate %s: %w", filepath.Base(filePath), err) + return nil, fmt.Errorf("failed to parse prestate %s: %w", filepath.Base(filePath), err) } - // Rename the file to just the hash - hashedPath := filepath.Join(buildDir, data.Pre) - if err := os.Rename(filePath, hashedPath); err != nil { - return "", fmt.Errorf("failed to rename prestate %s: %w", filepath.Base(filePath), err) + // Store hash with its corresponding key + if key, exists := fileToKey[filepath.Base(filePath)]; exists { + info.Hashes[key] = data.Pre } - log.Printf("%s available at: %s/%s/%s\n", filepath.Base(filePath), server.URL(), relPath, data.Pre) + // Rename files to hash-based names + newFileName := data.Pre + ".json" + hashedPath := filepath.Join(buildDir, newFileName) + if err := os.Rename(filePath, hashedPath); err != nil { + return nil, fmt.Errorf("failed to rename prestate %s: %w", filepath.Base(filePath), err) + } + log.Printf("%s available at: %s/%s/%s\n", filepath.Base(filePath), server.URL(), relPath, newFileName) + + // Rename the corresponding binary file + binFilePath := strings.Replace(strings.TrimSuffix(filePath, ".json"), "-proof", "", 1) + ".bin.gz" + newBinFileName := data.Pre + ".bin.gz" + binHashedPath := filepath.Join(buildDir, newBinFileName) + if err := os.Rename(binFilePath, binHashedPath); err != nil { + return nil, fmt.Errorf("failed to rename prestate %s: %w", filepath.Base(binFilePath), err) + } + log.Printf("%s available at: %s/%s/%s\n", filepath.Base(binFilePath), server.URL(), relPath, newBinFileName) } - return url, nil + return info, nil }) } @@ -271,11 +299,16 @@ func (m *Main) deploy(ctx context.Context, r io.Reader) error { return fmt.Errorf("error creating kurtosis deployer: %w", err) } - env, err := d.Deploy(ctx, buf) + spec, err := d.Deploy(ctx, buf) if err != nil { return fmt.Errorf("error deploying kurtosis package: %w", err) } + env, err := d.GetEnvironmentInfo(ctx, spec) + if err != nil { + return fmt.Errorf("error getting environment: %w", err) + } + if err := writeEnvironment(m.cfg.environment, env); err != nil { return fmt.Errorf("error writing environment: %w", err) } @@ -283,8 +316,49 @@ func (m *Main) deploy(ctx context.Context, r io.Reader) error { return nil } +func (m *Main) deployFileserver(ctx context.Context, sourceDir string) error { + // Create a temp dir in the fileserver package + baseDir := filepath.Join(m.cfg.baseDir, FILESERVER_PACKAGE) + if err := os.MkdirAll(baseDir, 0755); err != nil { + return fmt.Errorf("error creating base directory: %w", err) + } + tempDir, err := os.MkdirTemp(baseDir, "upload-content") + if err != nil { + return fmt.Errorf("error creating temporary directory: %w", err) + } + defer os.RemoveAll(tempDir) + + // Copy build dir contents to tempDir + if err := util.CopyDir(sourceDir, tempDir); err != nil { + return fmt.Errorf("error copying directory: %w", err) + } + + buf := bytes.NewBuffer(nil) + buf.WriteString(fmt.Sprintf("source_path: %s\n", filepath.Base(tempDir))) + + opts := []kurtosis.KurtosisDeployerOptions{ + kurtosis.WithKurtosisBaseDir(m.cfg.baseDir), + kurtosis.WithKurtosisDryRun(m.cfg.dryRun), + kurtosis.WithKurtosisPackageName(FILESERVER_PACKAGE), + kurtosis.WithKurtosisEnclave(m.cfg.enclave), + } + + d, err := m.newDeployer(opts...) + if err != nil { + return fmt.Errorf("error creating kurtosis deployer: %w", err) + } + + _, err = d.Deploy(ctx, buf) + if err != nil { + return fmt.Errorf("error deploying kurtosis package: %w", err) + } + + return nil +} + type deployer interface { - Deploy(ctx context.Context, input io.Reader) (*kurtosis.KurtosisEnvironment, error) + Deploy(ctx context.Context, input io.Reader) (*spec.EnclaveSpec, error) + GetEnvironmentInfo(ctx context.Context, spec *spec.EnclaveSpec) (*kurtosis.KurtosisEnvironment, error) } func writeEnvironment(path string, env *kurtosis.KurtosisEnvironment) error { @@ -328,6 +402,12 @@ func (m *Main) run() error { return fmt.Errorf("error rendering template: %w", err) } + // TODO: clean up consumers of static server and replace with fileserver + err = m.deployFileserver(ctx, server.dir) + if err != nil { + return fmt.Errorf("error deploying fileserver: %w", err) + } + return m.deploy(ctx, buf) } diff --git a/kurtosis-devnet/cmd/main_test.go b/kurtosis-devnet/cmd/main_test.go index c419785626dc..85b54ebfa711 100644 --- a/kurtosis-devnet/cmd/main_test.go +++ b/kurtosis-devnet/cmd/main_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis" + "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/sources/spec" "github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,7 +21,11 @@ type mockDeployer struct { dryRun bool } -func (m *mockDeployer) Deploy(ctx context.Context, input io.Reader) (*kurtosis.KurtosisEnvironment, error) { +func (m *mockDeployer) Deploy(ctx context.Context, input io.Reader) (*spec.EnclaveSpec, error) { + return &spec.EnclaveSpec{}, nil +} + +func (m *mockDeployer) GetEnvironmentInfo(ctx context.Context, spec *spec.EnclaveSpec) (*kurtosis.KurtosisEnvironment, error) { return &kurtosis.KurtosisEnvironment{}, nil } @@ -221,6 +226,26 @@ func TestDeploy(t *testing.T) { require.NoError(t, err) } +func TestDeployFileserver(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tmpDir, err := os.MkdirTemp("", "deploy-fileserver-test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + envPath := filepath.Join(tmpDir, "env.json") + cfg := &config{ + baseDir: tmpDir, + environment: envPath, + dryRun: true, + } + + m := newTestMain(cfg) + err = m.deployFileserver(ctx, filepath.Join(tmpDir, "fileserver")) + require.NoError(t, err) +} + func TestMainFunc(t *testing.T) { // Create a temporary directory for test files tmpDir, err := os.MkdirTemp("", "main-test") diff --git a/kurtosis-devnet/fileserver/kurtosis.yml b/kurtosis-devnet/fileserver/kurtosis.yml new file mode 100644 index 000000000000..2f1eb17b25e8 --- /dev/null +++ b/kurtosis-devnet/fileserver/kurtosis.yml @@ -0,0 +1,4 @@ +name: github.com/ethereum-optimism/optimism/kurtosis-devnet/fileserver +description: |- + Kurtosis package for serving files from the build directory +replace: {} diff --git a/kurtosis-devnet/fileserver/main.star b/kurtosis-devnet/fileserver/main.star new file mode 100644 index 000000000000..c26357f1a3b7 --- /dev/null +++ b/kurtosis-devnet/fileserver/main.star @@ -0,0 +1,49 @@ +FILESERVER_HTTP_PORT_ID = "http" +FILESERVER_HTTP_PORT_NUM = 80 +FILESERVER_IMAGE = "nginx:latest" + + +def get_used_ports(): + used_ports = { + FILESERVER_HTTP_PORT_ID: PortSpec( + number=FILESERVER_HTTP_PORT_NUM, + ) + } + return used_ports + + +def run(plan, source_path): + service_name = "fileserver" + config = get_fileserver_config( + plan, + service_name, + source_path, + ) + service = plan.add_service(service_name, config) + return service_name + + +def get_fileserver_config(plan, service_name, source_path): + files = {} + + # Upload content to container + content_artifact = plan.upload_files( + src=source_path, + name="{}-content".format(service_name), + ) + files["/content"] = content_artifact + + # Add nginx config file + nginx_conf = plan.upload_files( + src="static_files/nginx", + name="{}-nginx-conf".format(service_name), + ) + files["/etc/nginx/conf.d"] = nginx_conf + + ports = get_used_ports() + return ServiceConfig( + image=FILESERVER_IMAGE, + ports=ports, + cmd=["nginx", "-g", "daemon off;"], + files=files, + ) diff --git a/kurtosis-devnet/fileserver/static_files/nginx/default.conf b/kurtosis-devnet/fileserver/static_files/nginx/default.conf new file mode 100644 index 000000000000..69932eb89d88 --- /dev/null +++ b/kurtosis-devnet/fileserver/static_files/nginx/default.conf @@ -0,0 +1,8 @@ +server { + listen 80; + server_name _; + root /content; + location / { + try_files $uri $uri/ =404; + } +} diff --git a/kurtosis-devnet/interop.yaml b/kurtosis-devnet/interop.yaml index 5532890c1b90..e5265222ae45 100644 --- a/kurtosis-devnet/interop.yaml +++ b/kurtosis-devnet/interop.yaml @@ -1,17 +1,17 @@ -{{- $local_images := dict +{{- $local_images := dict "op_node" (localDockerImage "op-node") - "op_batcher" (localDockerImage "op-batcher") - "op_challenger" (localDockerImage "op-challenger") - "op_proposer" (localDockerImage "op-proposer") - "op_deployer" (localDockerImage "op-deployer") - "op_supervisor" (localDockerImage "op-supervisor") + "op_batcher" (localDockerImage "op-batcher") + "op_challenger" (localDockerImage "op-challenger") + "op_proposer" (localDockerImage "op-proposer") + "op_deployer" (localDockerImage "op-deployer") + "op_supervisor" (localDockerImage "op-supervisor") -}} -{{- $urls := dict - "prestate" (localPrestate) +{{- $urls := dict + "prestate" "http://fileserver/proofs/op-program/cannon" "l1_artifacts" (localContractArtifacts "l1") "l2_artifacts" (localContractArtifacts "l2") -}} -{{- $flags := dict +{{- $flags := dict "log_level" "--log.level=info" -}} --- @@ -160,6 +160,8 @@ optimism_package: image: {{ $local_images.op_deployer }} l1_artifacts_locator: {{ $urls.l1_artifacts }} l2_artifacts_locator: {{ $urls.l2_artifacts }} + global_deploy_overrides: + faultGameAbsolutePrestate: {{ localPrestate.Hashes.prestate }} global_log_level: "info" global_node_selectors: {} global_tolerations: [] diff --git a/kurtosis-devnet/justfile b/kurtosis-devnet/justfile index af838432c30e..b89e32f8c4b2 100644 --- a/kurtosis-devnet/justfile +++ b/kurtosis-devnet/justfile @@ -9,7 +9,7 @@ _contracts-build BUNDLE='contracts-bundle.tar.gz': tar -czf {{BUNDLE}} -C ../packages/contracts-bedrock artifacts forge-artifacts cache _prestate-build PATH='.': - docker buildx build --output {{PATH}} --target export-stage-proofs --progress plain -f ../op-program/Dockerfile.repro ../ + docker buildx build --output {{PATH}} --progress plain -f ../op-program/Dockerfile.repro ../ _docker_build TAG TARGET CONTEXT DOCKERFILE *ARGS: docker buildx build -t {{TAG}} \ diff --git a/kurtosis-devnet/pkg/kurtosis/kurtosis.go b/kurtosis-devnet/pkg/kurtosis/kurtosis.go index 5767fc8d69c6..cb75b7b91122 100644 --- a/kurtosis-devnet/pkg/kurtosis/kurtosis.go +++ b/kurtosis-devnet/pkg/kurtosis/kurtosis.go @@ -151,7 +151,7 @@ func (d *KurtosisDeployer) getWallets(wallets deployer.WalletList) WalletMap { } // getEnvironmentInfo parses the input spec and inspect output to create KurtosisEnvironment -func (d *KurtosisDeployer) getEnvironmentInfo(ctx context.Context, spec *spec.EnclaveSpec) (*KurtosisEnvironment, error) { +func (d *KurtosisDeployer) GetEnvironmentInfo(ctx context.Context, spec *spec.EnclaveSpec) (*KurtosisEnvironment, error) { inspectResult, err := d.enclaveInspecter.EnclaveInspect(ctx, d.enclave) if err != nil { return nil, fmt.Errorf("failed to parse inspect output: %w", err) @@ -210,7 +210,7 @@ func (d *KurtosisDeployer) getEnvironmentInfo(ctx context.Context, spec *spec.En } // Deploy executes the Kurtosis deployment command with the provided input -func (d *KurtosisDeployer) Deploy(ctx context.Context, input io.Reader) (*KurtosisEnvironment, error) { +func (d *KurtosisDeployer) Deploy(ctx context.Context, input io.Reader) (*spec.EnclaveSpec, error) { // Parse the input spec first inputCopy := new(bytes.Buffer) tee := io.TeeReader(input, inputCopy) @@ -236,9 +236,9 @@ func (d *KurtosisDeployer) Deploy(ctx context.Context, input io.Reader) (*Kurtos // If dry run, return empty environment if d.dryRun { - return &KurtosisEnvironment{}, nil + return spec, nil } // Get environment information - return d.getEnvironmentInfo(ctx, spec) + return spec, nil } diff --git a/kurtosis-devnet/pkg/util/util.go b/kurtosis-devnet/pkg/util/util.go new file mode 100644 index 000000000000..a6c0e23d4515 --- /dev/null +++ b/kurtosis-devnet/pkg/util/util.go @@ -0,0 +1,48 @@ +package util + +import ( + "io" + "io/fs" + "os" + "path/filepath" +) + +func CopyDir(src string, dst string) error { + err := filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // Get relative path + relPath, err := filepath.Rel(src, path) + if err != nil { + return err + } + + // Construct destination path + dstPath := filepath.Join(dst, relPath) + + if info.IsDir() { + // Create directories with same permissions + return os.MkdirAll(dstPath, info.Mode()) + } + + // Copy files + srcFile, err := os.Open(path) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + return err + }) + + return err +} diff --git a/kurtosis-devnet/simple.yaml b/kurtosis-devnet/simple.yaml index 6835517a859a..be7ec67cdf64 100644 --- a/kurtosis-devnet/simple.yaml +++ b/kurtosis-devnet/simple.yaml @@ -14,7 +14,7 @@ optimism_package: el_min_mem: 0 el_max_mem: 0 cl_type: op-node - cl_image: "" + cl_image: {{ localDockerImage "op-node" }} cl_log_level: "" cl_extra_env_vars: {} cl_extra_labels: {} @@ -38,17 +38,29 @@ optimism_package: holocene_time_offset: 0 fund_dev_accounts: true batcher_params: - image: "" + image: {{ localDockerImage "op-batcher" }} extra_params: [] + challenger_params: + image: {{ localDockerImage "op-challenger" }} + cannon_prestate_path: "" + cannon_prestates_url: "http://fileserver/proofs/op-program/cannon" + extra_params: [] + proposer_params: + image: {{ localDockerImage "op-proposer" }} + extra_params: [] + game_type: 1 + proposal_interval: 10m mev_params: rollup_boost_image: "" builder_host: "" builder_port: "" additional_services: [] op_contract_deployer_params: - image: opsigma/op-deployer:v0.0.7-http + image: {{ localDockerImage "op-deployer" }} l1_artifacts_locator: {{ localContractArtifacts "l1" }} l2_artifacts_locator: {{ localContractArtifacts "l2" }} + global_deploy_overrides: + faultGameAbsolutePrestate: {{ localPrestate.Hashes.prestate }} global_log_level: "info" global_node_selectors: {} global_tolerations: []