From a1fc6646ec3cec335a3beec259a03ab42f4fbc41 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Fri, 10 Jan 2025 11:49:25 -0600 Subject: [PATCH] merge master and update geth pin --- arbos/programs/native.go | 150 ++++++++++++++++++++---------- arbos/programs/programs.go | 14 +-- arbos/programs/wasmstorehelper.go | 3 +- execution/gethexec/node.go | 3 - go-ethereum | 2 +- system_tests/common_test.go | 4 +- system_tests/program_test.go | 71 +++++++++----- 7 files changed, 165 insertions(+), 82 deletions(-) diff --git a/arbos/programs/native.go b/arbos/programs/native.go index a996d50d8a..5995d9dafe 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -72,7 +72,9 @@ func activateProgram( debug bool, burner burn.Burner, ) (*activationInfo, error) { - info, asmMap, err := activateProgramInternal(db, program, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, burner.GasLeft()) + targets := db.Database().WasmTargets() + moduleActivationMandatory := true + info, asmMap, err := activateProgramInternal(program, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, burner.GasLeft(), targets, moduleActivationMandatory) if err != nil { return nil, err } @@ -80,8 +82,7 @@ func activateProgram( return info, nil } -func activateProgramInternal( - db vm.StateDB, +func activateModule( addressForLogging common.Address, codehash common.Hash, wasm []byte, @@ -90,7 +91,7 @@ func activateProgramInternal( arbosVersionForGas uint64, debug bool, gasLeft *uint64, -) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { +) (*activationInfo, []byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -119,69 +120,120 @@ func activateProgramInternal( } return nil, nil, err } - hash := bytes32ToHash(moduleHash) - targets := db.Database().WasmTargets() + info := &activationInfo{ + moduleHash: bytes32ToHash(moduleHash), + initGas: uint16(stylusData.init_cost), + cachedInitGas: uint16(stylusData.cached_init_cost), + asmEstimate: uint32(stylusData.asm_estimate), + footprint: uint16(stylusData.footprint), + } + return info, module, nil +} + +func compileNative( + wasm []byte, + stylusVersion uint16, + debug bool, + target ethdb.WasmTarget, +) ([]byte, error) { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(stylusVersion), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := rustBytesIntoBytes(output) + if status_asm != 0 { + return nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + } + return asm, nil +} + +func activateProgramInternal( + addressForLogging common.Address, + codehash common.Hash, + wasm []byte, + page_limit uint16, + stylusVersion uint16, + arbosVersionForGas uint64, + debug bool, + gasLeft *uint64, + targets []ethdb.WasmTarget, + moduleActivationMandatory bool, +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { + var wavmFound bool + var nativeTargets []ethdb.WasmTarget + for _, target := range targets { + if target == rawdb.TargetWavm { + wavmFound = true + } else { + nativeTargets = append(nativeTargets, target) + } + } type result struct { target ethdb.WasmTarget asm []byte err error } - results := make(chan result, len(targets)) - for _, target := range targets { - target := target - if target == rawdb.TargetWavm { - results <- result{target, module, nil} - } else { - go func() { - output := &rustBytes{} - status_asm := C.stylus_compile( - goSlice(wasm), - u16(stylusVersion), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := rustBytesIntoBytes(output) - if status_asm != 0 { - results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} - return - } - results <- result{target, asm, nil} - }() + results := make(chan result) + // info will be set in separate thread, make sure to wait before reading + var info *activationInfo + asmMap := make(map[ethdb.WasmTarget][]byte, len(nativeTargets)+1) + if moduleActivationMandatory || wavmFound { + go func() { + var err error + var module []byte + info, module, err = activateModule(addressForLogging, codehash, wasm, page_limit, stylusVersion, arbosVersionForGas, debug, gasLeft) + results <- result{rawdb.TargetWavm, module, err} + }() + } + if moduleActivationMandatory { + // wait for the module activation before starting compilation for other targets + res := <-results + if res.err != nil { + return nil, nil, res.err + } else if wavmFound { + asmMap[res.target] = res.asm } } - asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) - for range targets { + for _, target := range nativeTargets { + target := target + go func() { + asm, err := compileNative(wasm, stylusVersion, debug, target) + results <- result{target, asm, err} + }() + } + expectedResults := len(nativeTargets) + if !moduleActivationMandatory && wavmFound { + // we didn't wait for module activation result, so wait for it too + expectedResults++ + } + var err error + for i := 0; i < expectedResults; i++ { res := <-results if res.err != nil { - err = errors.Join(res.err, err) + err = errors.Join(res.err, fmt.Errorf("%s:%w", res.target, err)) } else { asmMap[res.target] = res.asm } } - if err != nil { + if err != nil && moduleActivationMandatory { log.Error( "Compilation failed for one or more targets despite activation succeeding", "address", addressForLogging, - "codeHash", codeHash, - "moduleHash", hash, + "codehash", codehash, + "moduleHash", info.moduleHash, "targets", targets, "err", err, ) panic(fmt.Sprintf("Compilation of %v failed for one or more targets despite activation succeeding: %v", addressForLogging, err)) } - - info := &activationInfo{ - moduleHash: hash, - initGas: uint16(stylusData.init_cost), - cachedInitGas: uint16(stylusData.cached_init_cost), - asmEstimate: uint32(stylusData.asm_estimate), - footprint: uint16(stylusData.footprint), - } return info, asmMap, err } -func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging common.Address, code []byte, codeHash common.Hash, pagelimit uint16, time uint64, debugMode bool, program Program) ([]byte, error) { +func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging common.Address, code []byte, codehash common.Hash, pagelimit uint16, time uint64, debugMode bool, program Program) ([]byte, error) { localTarget := rawdb.LocalTarget() localAsm, err := statedb.TryGetActivatedAsm(localTarget, moduleHash) if err == nil && len(localAsm) > 0 { @@ -199,14 +251,16 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c zeroArbosVersion := uint64(0) zeroGas := uint64(0) + targets := statedb.Database().WasmTargets() // we know program is activated, so it must be in correct version and not use too much memory - info, asmMap, err := activateProgramInternal(statedb, addressForLogging, codeHash, wasm, pagelimit, program.version, zeroArbosVersion, debugMode, &zeroGas) + moduleActivationMandatory := false + info, asmMap, err := activateProgramInternal(addressForLogging, codehash, wasm, pagelimit, program.version, zeroArbosVersion, debugMode, &zeroGas, targets, moduleActivationMandatory) if err != nil { log.Error("failed to reactivate program", "address", addressForLogging, "expected moduleHash", moduleHash, "err", err) return nil, fmt.Errorf("failed to reactivate program address: %v err: %w", addressForLogging, err) } - if info.moduleHash != moduleHash { + if info != nil && info.moduleHash != moduleHash { log.Error("failed to reactivate program", "address", addressForLogging, "expected moduleHash", moduleHash, "got", info.moduleHash) return nil, fmt.Errorf("failed to reactivate program. address: %v, expected ModuleHash: %v", addressForLogging, moduleHash) } @@ -223,7 +277,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } else { // program activated recently, possibly in this eth_call // store it to statedb. It will be stored to database if statedb is commited - statedb.ActivateWasm(info.moduleHash, asmMap) + statedb.ActivateWasm(moduleHash, asmMap) } asm, exists := asmMap[localTarget] if !exists { @@ -302,10 +356,10 @@ func handleReqImpl(apiId usize, req_type u32, data *rustSlice, costPtr *u64, out // Caches a program in Rust. We write a record so that we can undo on revert. // For gas estimation and eth_call, we ignore permanent updates and rely on Rust's LRU. -func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressForLogging common.Address, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) { +func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressForLogging common.Address, code []byte, codehash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) { if runMode == core.MessageCommitMode { // address is only used for logging - asm, err := getLocalAsm(db, module, addressForLogging, code, codeHash, params.PageLimit, time, debug, program) + asm, err := getLocalAsm(db, module, addressForLogging, code, codehash, params.PageLimit, time, debug, program) if err != nil { panic("unable to recreate wasm") } diff --git a/arbos/programs/programs.go b/arbos/programs/programs.go index 524fcc0be7..c2fc1f68ad 100644 --- a/arbos/programs/programs.go +++ b/arbos/programs/programs.go @@ -164,8 +164,8 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, arbosVers return stylusVersion, codeHash, info.moduleHash, dataFee, false, p.setProgram(codeHash, programData) } -func runModeToString(runmode core.MessageRunMode) string { - switch runmode { +func runModeToString(runMode core.MessageRunMode) string { + switch runMode { case core.MessageCommitMode: return "commit_runmode" case core.MessageGasEstimationMode: @@ -187,7 +187,7 @@ func (p Programs) CallProgram( tracingInfo *util.TracingInfo, calldata []byte, reentrant bool, - runmode core.MessageRunMode, + runMode core.MessageRunMode, ) ([]byte, error) { evm := interpreter.Evm() contract := scope.Contract @@ -262,11 +262,11 @@ func (p Programs) CallProgram( address = *contract.CodeAddr } var arbos_tag uint32 - if runmode == core.MessageCommitMode { + if runMode == core.MessageCommitMode { arbos_tag = statedb.Database().WasmCacheTag() } - metrics.GetOrRegisterCounter(fmt.Sprintf("arb/arbos/stylus/program_calls/%s", runModeToString(runmode)), nil).Inc(1) + metrics.GetOrRegisterCounter(fmt.Sprintf("arb/arbos/stylus/program_calls/%s", runModeToString(runMode)), nil).Inc(1) ret, err := callProgram(address, moduleHash, localAsm, scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag) if len(ret) > 0 && arbosVersion >= gethParams.ArbosVersion_StylusFixes { // Ensure that return data costs as least as much as it would in the EVM. @@ -274,14 +274,14 @@ func (p Programs) CallProgram( if startingGas < evmCost { contract.Gas = 0 // #nosec G115 - metrics.GetOrRegisterCounter(fmt.Sprintf("arb/arbos/stylus/gas_used/%s", runModeToString(runmode)), nil).Inc(int64(startingGas)) + metrics.GetOrRegisterCounter(fmt.Sprintf("arb/arbos/stylus/gas_used/%s", runModeToString(runMode)), nil).Inc(int64(startingGas)) return nil, vm.ErrOutOfGas } maxGasToReturn := startingGas - evmCost contract.Gas = am.MinInt(contract.Gas, maxGasToReturn) } // #nosec G115 - metrics.GetOrRegisterCounter(fmt.Sprintf("arb/arbos/stylus/gas_used/%s", runModeToString(runmode)), nil).Inc(int64(startingGas - contract.Gas)) + metrics.GetOrRegisterCounter(fmt.Sprintf("arb/arbos/stylus/gas_used/%s", runModeToString(runMode)), nil).Inc(int64(startingGas - contract.Gas)) return ret, err } diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index c2d1aa65b0..1393752b72 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -62,7 +62,8 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash // We know program is activated, so it must be in correct version and not use too much memory // Empty program address is supplied because we dont have access to this during rebuilding of wasm store - info, asmMap, err := activateProgramInternal(statedb, common.Address{}, codeHash, wasm, progParams.PageLimit, program.version, zeroArbosVersion, debugMode, &zeroGas) + moduleActivationMandatory := false + info, asmMap, err := activateProgramInternal(common.Address{}, codeHash, wasm, progParams.PageLimit, program.version, zeroArbosVersion, debugMode, &zeroGas, targets, moduleActivationMandatory) if err != nil { log.Error("failed to reactivate program while rebuilding wasm store", "expected moduleHash", moduleHash, "err", err) return fmt.Errorf("failed to reactivate program while rebuilding wasm store: %w", err) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 890fa80de2..0b7cfec150 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -55,9 +55,6 @@ func (c *StylusTargetConfig) Validate() error { } targetsSet[target] = true } - if !targetsSet[rawdb.TargetWavm] { - return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) - } targetsSet[rawdb.LocalTarget()] = true targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) for target := range targetsSet { diff --git a/go-ethereum b/go-ethereum index 5bf9a75f04..9257e4f31f 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 5bf9a75f04bc59137e06816e065b8494f796ba2d +Subproject commit 9257e4f31fd9fac62748150a81c1a2296e18ae4e diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 4860c3ce4d..391111dc2a 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1429,6 +1429,7 @@ func createNonL1BlockChainWithStackConfig( if execConfig == nil { execConfig = ExecConfigDefaultTest(t) } + Require(t, execConfig.Validate()) stack, err := node.New(stackConfig) Require(t, err) @@ -1516,6 +1517,8 @@ func Create2ndNodeWithConfig( if execConfig == nil { execConfig = ExecConfigDefaultNonSequencerTest(t) } + Require(t, execConfig.Validate()) + feedErrChan := make(chan error, 10) parentChainRpcClient := parentChainStack.Attach() parentChainClient := ethclient.NewClient(parentChainRpcClient) @@ -1549,7 +1552,6 @@ func Create2ndNodeWithConfig( AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) - Require(t, execConfig.Validate()) Require(t, nodeConfig.Validate()) configFetcher := func() *gethexec.Config { return execConfig } currentExec, err := gethexec.CreateExecutionNode(ctx, chainStack, chainDb, blockchain, parentChainClient, configFetcher) diff --git a/system_tests/program_test.go b/system_tests/program_test.go index 5fbb1189c7..053cfe859d 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -59,6 +59,12 @@ func TestProgramKeccak(t *testing.T) { builder.WithExtraArchs(allWasmTargets) }) }) + + t.Run("WithOnlyLocalTarget", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs([]string{string(rawdb.LocalTarget())}) + }) + }) } func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { @@ -69,7 +75,7 @@ func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 1) wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) @@ -82,7 +88,7 @@ func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -164,6 +170,11 @@ func TestProgramActivateTwice(t *testing.T) { builder.WithExtraArchs(allWasmTargets) }) }) + t.Run("WithOnlyLocalTarget", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs([]string{string(rawdb.LocalTarget())}) + }) + }) } func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { @@ -195,7 +206,7 @@ func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 1) preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") @@ -220,7 +231,7 @@ func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder) // Calling the contract pre-activation should fail. checkReverts() - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -243,7 +254,7 @@ func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder) // Ensure the revert also reverted keccak's activation checkReverts() - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -251,7 +262,7 @@ func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder) tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 2) validateBlocks(t, 7, jit, builder) } @@ -2083,7 +2094,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) - checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.WasmTargets(), 1) // close nodeB cleanupB() @@ -2140,7 +2151,7 @@ func TestWasmStoreRebuilding(t *testing.T) { } } - checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.WasmTargets(), 1) cleanupB() } @@ -2167,25 +2178,43 @@ func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { return modules } -func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, expectedTargets []ethdb.WasmTarget, numModules int) { + t.Helper() modules := readModuleHashes(t, wasmDb) if len(modules) != numModules { t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) } + readAsm := func(module common.Hash, target string) []byte { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + return func() []byte { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + return rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } for _, module := range modules { - for _, target := range targets { - wasmTarget := ethdb.WasmTarget(target) - if !rawdb.IsSupportedWasmTarget(wasmTarget) { - t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + for _, target := range allWasmTargets { + var expected bool + for _, expectedTarget := range expectedTargets { + if ethdb.WasmTarget(target) == expectedTarget { + expected = true + break + } + } + asm := readAsm(module, target) + if expected && len(asm) == 0 { + t.Fatalf("Missing asm for target: %v, module: %v", target, module) + } + if !expected && len(asm) > 0 { + t.Fatalf("Found asm for target: %v, module: %v, expected targets: %v", target, module, expectedTargets) } - func() { - defer func() { - if r := recover(); r != nil { - t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) - } - }() - _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) - }() } } }