Merge branch 'develop' into remove-fulu-block-state

beacon-chain/rpc/prysm/v1alpha1/beacon/blocks.go

@@ -110,6 +110,10 @@ func convertToBlockContainer(blk interfaces.ReadOnlySignedBeaconBlock, root [32]
 		ctr.Block = &ethpb.BeaconBlockContainer_BlindedDenebBlock{BlindedDenebBlock: pbStruct}
 	case *ethpb.SignedBeaconBlockDeneb:
 		ctr.Block = &ethpb.BeaconBlockContainer_DenebBlock{DenebBlock: pbStruct}
+	case *ethpb.SignedBlindedBeaconBlockElectra:
+		ctr.Block = &ethpb.BeaconBlockContainer_BlindedElectraBlock{BlindedElectraBlock: pbStruct}
+	case *ethpb.SignedBeaconBlockElectra:
+		ctr.Block = &ethpb.BeaconBlockContainer_ElectraBlock{ElectraBlock: pbStruct}
 	default:
 		return nil, errors.Errorf("block type is not recognized: %d", blk.Version())
 	}

changelog/james-prysm_electra-e2e.md

@@ -0,0 +1,7 @@
+### Added
+
+- enable E2E for minimal and mainnet tests
+
+### Changed
+
+- updates geth to 1.15.0

config/params/testdata/e2e_config.yaml

@@ -46,7 +46,7 @@ DENEB_FORK_VERSION: 0x040000fd
 DENEB_FORK_EPOCH: 12
 # Electra
 ELECTRA_FORK_VERSION: 0x050000fd
-ELECTRA_FORK_EPOCH: 18446744073709551615
+ELECTRA_FORK_EPOCH: 14
 # Fulu
 FULU_FORK_VERSION: 0x060000fd
 FULU_FORK_EPOCH: 18446744073709551615

config/params/testnet_e2e_config.go

@@ -7,7 +7,7 @@ const (
 	BellatrixE2EForkEpoch = 8
 	CapellaE2EForkEpoch   = 10
 	DenebE2EForkEpoch     = 12
-	ElectraE2EForkEpoch   = math.MaxUint64
+	ElectraE2EForkEpoch   = 14
 	FuluE2EForkEpoch      = math.MaxUint64
 )
 

deps.bzl

@@ -760,8 +760,8 @@ def prysm_deps():
         patches = [
             "//third_party:com_github_ethereum_go_ethereum_secp256k1.patch",
         ],
-        sum = "h1:L81Wmv0OUP6cf4CW6wtXsr23RUrDhKs2+Y9Qto+OgHU=",
-        version = "v1.14.13",
+        sum = "h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4=",
+        version = "v1.15.0",
     )
     go_repository(
         name = "com_github_ethereum_go_verkle",
@@ -802,8 +802,8 @@ def prysm_deps():
     go_repository(
         name = "com_github_fjl_gencodec",
         importpath = "github.com/fjl/gencodec",
-        sum = "h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY=",
-        version = "v0.0.0-20230517082657-f9840df7b83e",
+        sum = "h1:B3K0xPfc52cw52BBgUbSPxYo+HlLfAgWMVKRWXUXBcs=",
+        version = "v0.1.0",
     )
     go_repository(
         name = "com_github_flosch_pongo2_v4",
@@ -2683,6 +2683,12 @@ def prysm_deps():
         sum = "h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4=",
         version = "v0.6.1",
     )
+    go_repository(
+        name = "com_github_pion_stun_v2",
+        importpath = "github.com/pion/stun/v2",
+        sum = "h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0=",
+        version = "v2.0.0",
+    )
     go_repository(
         name = "com_github_pion_transport_v2",
         importpath = "github.com/pion/transport/v2",

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/dgraph-io/ristretto v0.0.4-0.20210318174700-74754f61e018
 	github.com/dustin/go-humanize v1.0.0
 	github.com/emicklei/dot v0.11.0
-	github.com/ethereum/go-ethereum v1.14.13
+	github.com/ethereum/go-ethereum v1.15.0
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-yaml/yaml v2.1.0+incompatible
@@ -221,7 +221,9 @@ require (
 	github.com/pion/sdp/v3 v3.0.9 // indirect
 	github.com/pion/srtp/v2 v2.0.20 // indirect
 	github.com/pion/stun v0.6.1 // indirect
+	github.com/pion/stun/v2 v2.0.0 // indirect
 	github.com/pion/transport/v2 v2.2.10 // indirect
+	github.com/pion/transport/v3 v3.0.7 // indirect
 	github.com/pion/turn/v2 v2.1.6 // indirect
 	github.com/pion/webrtc/v3 v3.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -235,8 +235,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=
 github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
-github.com/ethereum/go-ethereum v1.14.13 h1:L81Wmv0OUP6cf4CW6wtXsr23RUrDhKs2+Y9Qto+OgHU=
-github.com/ethereum/go-ethereum v1.14.13/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY=
+github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4=
+github.com/ethereum/go-ethereum v1.15.0/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs=
 github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8=
 github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -825,6 +825,8 @@ github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk=
 github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA=
 github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4=
 github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8=
+github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0=
+github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ=
 github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
 github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
 github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=

runtime/interop/genesis.go

@@ -159,6 +159,18 @@ func GethTestnetGenesis(genesisTime uint64, cfg *clparams.BeaconChainConfig) *co
 		ShanghaiTime:            shanghaiTime,
 		CancunTime:              cancunTime,
 		PragueTime:              pragueTime,
+		BlobScheduleConfig: &params.BlobScheduleConfig{
+			Cancun: &params.BlobConfig{
+				Target:         3,
+				Max:            6,
+				UpdateFraction: 3338477,
+			},
+			Prague: &params.BlobConfig{
+				Target:         6,
+				Max:            9,
+				UpdateFraction: 5007716,
+			},
+		},
 	}
 	da := defaultDepositContractAllocation(cfg.DepositContractAddress)
 	ma := minerAllocation()

testing/endtoend/endtoend_setup_test.go

@@ -22,7 +22,7 @@ func e2eMinimal(t *testing.T, cfg *params.BeaconChainConfig, cfgo ...types.E2ECo
 
 	// Run for 12 epochs if not in long-running to confirm long-running has no issues.
 	var err error
-	epochsToRun := 14
+	epochsToRun := 16
 	epochStr, longRunning := os.LookupEnv("E2E_EPOCHS")
 	if longRunning {
 		epochsToRun, err = strconv.Atoi(epochStr)
@@ -64,6 +64,7 @@ func e2eMinimal(t *testing.T, cfg *params.BeaconChainConfig, cfgo ...types.E2ECo
 	evals = addIfForkSet(evals, cfg.BellatrixForkEpoch, ev.BellatrixForkTransition)
 	evals = addIfForkSet(evals, cfg.CapellaForkEpoch, ev.CapellaForkTransition)
 	evals = addIfForkSet(evals, cfg.DenebForkEpoch, ev.DenebForkTransition)
+	evals = addIfForkSet(evals, cfg.ElectraForkEpoch, ev.ElectraForkTransition)
 
 	testConfig := &types.E2EConfig{
 		BeaconFlags: []string{
@@ -104,7 +105,7 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfg *params.Beaco
 	}
 	// Run for 10 epochs if not in long-running to confirm long-running has no issues.
 	var err error
-	epochsToRun := 14
+	epochsToRun := 16
 	epochStr, longRunning := os.LookupEnv("E2E_EPOCHS")
 	if longRunning {
 		epochsToRun, err = strconv.Atoi(epochStr)
@@ -139,6 +140,7 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfg *params.Beaco
 	evals = addIfForkSet(evals, cfg.BellatrixForkEpoch, ev.BellatrixForkTransition)
 	evals = addIfForkSet(evals, cfg.CapellaForkEpoch, ev.CapellaForkTransition)
 	evals = addIfForkSet(evals, cfg.DenebForkEpoch, ev.DenebForkTransition)
+	evals = addIfForkSet(evals, cfg.ElectraForkEpoch, ev.ElectraForkTransition)
 
 	testConfig := &types.E2EConfig{
 		BeaconFlags: []string{
@@ -206,6 +208,7 @@ func scenarioEvals(cfg *params.BeaconChainConfig) []types.Evaluator {
 	evals = addIfForkSet(evals, cfg.BellatrixForkEpoch, ev.BellatrixForkTransition)
 	evals = addIfForkSet(evals, cfg.CapellaForkEpoch, ev.CapellaForkTransition)
 	evals = addIfForkSet(evals, cfg.DenebForkEpoch, ev.DenebForkTransition)
+	evals = addIfForkSet(evals, cfg.ElectraForkEpoch, ev.ElectraForkTransition)
 	return evals
 }
 
@@ -226,5 +229,6 @@ func scenarioEvalsMulti(cfg *params.BeaconChainConfig) []types.Evaluator {
 	evals = addIfForkSet(evals, cfg.BellatrixForkEpoch, ev.BellatrixForkTransition)
 	evals = addIfForkSet(evals, cfg.CapellaForkEpoch, ev.CapellaForkTransition)
 	evals = addIfForkSet(evals, cfg.DenebForkEpoch, ev.DenebForkTransition)
+	evals = addIfForkSet(evals, cfg.ElectraForkEpoch, ev.ElectraForkTransition)
 	return evals
 }

testing/endtoend/evaluators/fork.go

@@ -53,6 +53,7 @@ var CapellaForkTransition = types.Evaluator{
 	Evaluation: capellaForkOccurs,
 }
 
+// DenebForkTransition ensures that the Deneb hard fork has occurred successfully
 var DenebForkTransition = types.Evaluator{
 	Name: "deneb_fork_transition_%d",
 	Policy: func(e primitives.Epoch) bool {
@@ -62,6 +63,16 @@ var DenebForkTransition = types.Evaluator{
 	Evaluation: denebForkOccurs,
 }
 
+// ElectraForkTransition ensures that the electra hard fork has occurred successfully
+var ElectraForkTransition = types.Evaluator{
+	Name: "electra_fork_transition_%d",
+	Policy: func(e primitives.Epoch) bool {
+		fEpoch := params.BeaconConfig().ElectraForkEpoch
+		return policies.OnEpoch(fEpoch)(e)
+	},
+	Evaluation: electraForkOccurs,
+}
+
 func altairForkOccurs(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconNodeValidatorClient(conn)
@@ -234,3 +245,46 @@ func denebForkOccurs(_ *types.EvaluationContext, conns ...*grpc.ClientConn) erro
 	}
 	return nil
 }
+
+func electraForkOccurs(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
+	conn := conns[0]
+	client := ethpb.NewBeaconNodeValidatorClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), streamDeadline)
+	defer cancel()
+	stream, err := client.StreamBlocksAltair(ctx, &ethpb.StreamBlocksRequest{VerifiedOnly: true})
+	if err != nil {
+		return errors.Wrap(err, "failed to get stream")
+	}
+	fSlot, err := slots.EpochStart(params.BeaconConfig().ElectraForkEpoch)
+	if err != nil {
+		return err
+	}
+	if errors.Is(ctx.Err(), context.Canceled) {
+		return errors.New("context canceled prematurely")
+	}
+	res, err := stream.Recv()
+	if err != nil {
+		return err
+	}
+	if res == nil || res.Block == nil {
+		return errors.New("nil block returned by beacon node")
+	}
+
+	if res.GetBlock() == nil {
+		return errors.New("nil block returned by beacon node")
+	}
+	if res.GetElectraBlock() == nil {
+		return errors.Errorf("non-electra block returned after the fork with type %T", res.Block)
+	}
+	blk, err := blocks.NewSignedBeaconBlock(res.GetElectraBlock())
+	if err != nil {
+		return err
+	}
+	if err := blocks.BeaconBlockIsNil(blk); err != nil {
+		return err
+	}
+	if blk.Block().Slot() < fSlot {
+		return errors.Errorf("wanted a block at slot >= %d but received %d", fSlot, blk.Block().Slot())
+	}
+	return nil
+}

testing/endtoend/evaluators/operations.go

@@ -511,8 +511,16 @@ func validatorsVoteWithTheMajority(ec *e2etypes.EvaluationContext, conns ...*grp
 			b := blk.GetBlindedDenebBlock().Message
 			slot = b.Slot
 			vote = b.Body.Eth1Data.BlockHash
+		case *ethpb.BeaconBlockContainer_ElectraBlock:
+			b := blk.GetElectraBlock().Block
+			slot = b.Slot
+			vote = b.Body.Eth1Data.BlockHash
+		case *ethpb.BeaconBlockContainer_BlindedElectraBlock:
+			b := blk.GetBlindedElectraBlock().Message
+			slot = b.Slot
+			vote = b.Body.Eth1Data.BlockHash
 		default:
-			return errors.New("block neither phase0,altair or bellatrix")
+			return fmt.Errorf("block of type %T is unknown", blk.Block)
 		}
 		ec.SeenVotes[slot] = vote
 

testing/endtoend/evaluators/validator.go

@@ -126,6 +126,12 @@ func validatorsParticipating(_ *types.EvaluationContext, conns ...*grpc.ClientCo
 	if e2eparams.TestParams.LighthouseBeaconNodeCount != 0 {
 		expected = float32(expectedMulticlientParticipation)
 	}
+	if participation.Epoch == params.BeaconConfig().ElectraForkEpoch {
+		// The first slot of Electra will be missed due to the switching of attestation types
+		// 5/6 slots =~0.83
+		// validator REST always is slightly reduced at ~0.82
+		expected = 0.82
+	}
 	if participation.Epoch > 0 && participation.Epoch.Sub(1) == params.BeaconConfig().BellatrixForkEpoch {
 		// Reduce Participation requirement to 95% to account for longer EE calls for
 		// the merge block. Target and head will likely be missed for a few validators at
@@ -172,6 +178,18 @@ func validatorsParticipating(_ *types.EvaluationContext, conns ...*grpc.ClientCo
 				return err
 			}
 			respPrevEpochParticipation = st.PreviousEpochParticipation
+		case version.String(version.Deneb):
+			st := &structs.BeaconStateDeneb{}
+			if err = json.Unmarshal(resp.Data, st); err != nil {
+				return err
+			}
+			respPrevEpochParticipation = st.PreviousEpochParticipation
+		case version.String(version.Electra):
+			st := &structs.BeaconStateElectra{}
+			if err = json.Unmarshal(resp.Data, st); err != nil {
+				return err
+			}
+			respPrevEpochParticipation = st.PreviousEpochParticipation
 		default:
 			return fmt.Errorf("unrecognized version %s", resp.Version)
 		}
@@ -329,6 +347,12 @@ func syncCompatibleBlockFromCtr(container *ethpb.BeaconBlockContainer) (interfac
 	if container.GetBlindedDenebBlock() != nil {
 		return blocks.NewSignedBeaconBlock(container.GetBlindedDenebBlock())
 	}
+	if container.GetElectraBlock() != nil {
+		return blocks.NewSignedBeaconBlock(container.GetElectraBlock())
+	}
+	if container.GetBlindedElectraBlock() != nil {
+		return blocks.NewSignedBeaconBlock(container.GetBlindedElectraBlock())
+	}
 	return nil, errors.New("no supported block type in container")
 }
 

testing/endtoend/minimal_e2e_test.go

@@ -9,6 +9,6 @@ import (
 )
 
 func TestEndToEnd_MinimalConfig(t *testing.T) {
-	r := e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Deneb, params.E2ETestConfig()), types.WithCheckpointSync())
+	r := e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Electra, params.E2ETestConfig()), types.WithCheckpointSync())
 	r.run()
 }

testing/endtoend/minimal_scenario_e2e_test.go

@@ -26,7 +26,7 @@ func TestEndToEnd_MinimalConfig_Web3Signer_PersistentKeys(t *testing.T) {
 }
 
 func TestEndToEnd_MinimalConfig_ValidatorRESTApi(t *testing.T) {
-	e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Deneb, params.E2ETestConfig()), types.WithCheckpointSync(), types.WithValidatorRESTApi()).run()
+	e2eMinimal(t, types.InitForkCfg(version.Bellatrix, version.Electra, params.E2ETestConfig()), types.WithCheckpointSync(), types.WithValidatorRESTApi()).run()
 }
 
 func TestEndToEnd_ScenarioRun_EEOffline(t *testing.T) {

testing/endtoend/types/fork.go

@@ -25,6 +25,12 @@ func InitForkCfg(start, end int, c *params.BeaconChainConfig) *params.BeaconChai
 	if start >= version.Deneb {
 		c.DenebForkEpoch = 0
 	}
+	if start >= version.Electra {
+		c.ElectraForkEpoch = 0
+	}
+	if end < version.Electra {
+		c.ElectraForkEpoch = math.MaxUint64
+	}
 	if end < version.Deneb {
 		c.DenebForkEpoch = math.MaxUint64
 	}