chore: merge stable back to unstable

lerna.json

@@ -4,7 +4,7 @@
   ],
   "npmClient": "yarn",
   "useNx": true,
-  "version": "1.22.0",
+  "version": "1.23.0",
   "stream": true,
   "command": {
     "version": {

packages/api/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.22.0",
+  "version": "1.23.0",
   "type": "module",
   "exports": {
     ".": {
@@ -72,10 +72,10 @@
   "dependencies": {
     "@chainsafe/persistent-merkle-tree": "^0.8.0",
     "@chainsafe/ssz": "^0.18.0",
-    "@lodestar/config": "^1.22.0",
-    "@lodestar/params": "^1.22.0",
-    "@lodestar/types": "^1.22.0",
-    "@lodestar/utils": "^1.22.0",
+    "@lodestar/config": "^1.23.0",
+    "@lodestar/params": "^1.23.0",
+    "@lodestar/types": "^1.23.0",
+    "@lodestar/utils": "^1.23.0",
     "eventsource": "^2.0.2",
     "qs": "^6.11.1"
   },

packages/beacon-node/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.22.0",
+  "version": "1.23.0",
   "type": "module",
   "exports": {
     ".": {
@@ -95,17 +95,17 @@
   },
   "dependencies": {
     "@chainsafe/as-sha256": "^0.5.0",
-    "@chainsafe/blst": "^2.2.0",
+    "@chainsafe/blst": "^2.1.0",
     "@chainsafe/discv5": "^9.0.0",
     "@chainsafe/enr": "^3.0.0",
     "@chainsafe/libp2p-gossipsub": "^13.0.0",
     "@chainsafe/libp2p-identify": "^1.0.0",
     "@chainsafe/libp2p-noise": "^15.0.0",
     "@chainsafe/persistent-merkle-tree": "^0.8.0",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
+    "@chainsafe/pubkey-index-map": "2.0.0",
     "@chainsafe/ssz": "^0.18.0",
     "@chainsafe/threads": "^1.11.1",
-    "@chainsafe/pubkey-index-map": "2.0.0",
     "@ethersproject/abi": "^5.7.0",
     "@fastify/bearer-auth": "^10.0.1",
     "@fastify/cors": "^10.0.1",
@@ -120,18 +120,18 @@
     "@libp2p/peer-id-factory": "^4.1.0",
     "@libp2p/prometheus-metrics": "^3.0.21",
     "@libp2p/tcp": "9.0.23",
-    "@lodestar/api": "^1.22.0",
-    "@lodestar/config": "^1.22.0",
-    "@lodestar/db": "^1.22.0",
-    "@lodestar/fork-choice": "^1.22.0",
-    "@lodestar/light-client": "^1.22.0",
-    "@lodestar/logger": "^1.22.0",
-    "@lodestar/params": "^1.22.0",
-    "@lodestar/reqresp": "^1.22.0",
-    "@lodestar/state-transition": "^1.22.0",
-    "@lodestar/types": "^1.22.0",
-    "@lodestar/utils": "^1.22.0",
-    "@lodestar/validator": "^1.22.0",
+    "@lodestar/api": "^1.23.0",
+    "@lodestar/config": "^1.23.0",
+    "@lodestar/db": "^1.23.0",
+    "@lodestar/fork-choice": "^1.23.0",
+    "@lodestar/light-client": "^1.23.0",
+    "@lodestar/logger": "^1.23.0",
+    "@lodestar/params": "^1.23.0",
+    "@lodestar/reqresp": "^1.23.0",
+    "@lodestar/state-transition": "^1.23.0",
+    "@lodestar/types": "^1.23.0",
+    "@lodestar/utils": "^1.23.0",
+    "@lodestar/validator": "^1.23.0",
     "@multiformats/multiaddr": "^12.1.3",
     "c-kzg": "^2.1.2",
     "datastore-core": "^9.1.1",

packages/beacon-node/src/chain/archiver/archiver.ts

@@ -64,7 +64,7 @@ export class Archiver {
 
   /** Archive latest finalized state */
   async persistToDisk(): Promise<void> {
-    return this.statesArchiverStrategy.maybeArchiveState(this.chain.forkChoice.getFinalizedCheckpoint());
+    return this.statesArchiverStrategy.archiveState(this.chain.forkChoice.getFinalizedCheckpoint());
   }
 
   private onFinalizedCheckpoint = async (finalized: CheckpointWithHex): Promise<void> => {

packages/beacon-node/src/chain/archiver/interface.ts

@@ -44,4 +44,5 @@ export interface StateArchiveStrategy {
   onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise<void>;
   onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
   maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
+  archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
 }

packages/beacon-node/src/chain/archiver/strategies/frequencyStateArchiveStrategy.ts

@@ -84,7 +84,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
    * Archives finalized states from active bucket to archive bucket.
    * Only the new finalized state is stored to disk
    */
-  private async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
+  async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
     // starting from Mar 2024, the finalized state could be from disk or in memory
     const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized);
     const {rootHex} = finalized;

packages/beacon-node/src/chain/bls/multithread/index.ts

@@ -394,7 +394,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
         try {
           // Note: This can throw, must be handled per-job.
           // Pubkey and signature aggregation is defered here
-          workReq = await jobItemWorkReq(job, this.metrics);
+          workReq = jobItemWorkReq(job, this.metrics);
         } catch (e) {
           this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type});
 

packages/beacon-node/src/chain/bls/multithread/jobItem.ts

@@ -1,4 +1,4 @@
-import {PublicKey, asyncAggregateWithRandomness} from "@chainsafe/blst";
+import {PublicKey, aggregateWithRandomness} from "@chainsafe/blst";
 import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
 import {VerifySignatureOpts} from "../interface.js";
 import {getAggregatedPubkey} from "../utils.js";
@@ -48,7 +48,7 @@ export function jobItemSigSets(job: JobQueueItem): number {
  * Prepare BlsWorkReq from JobQueueItem
  * WARNING: May throw with untrusted user input
  */
-export async function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): Promise<BlsWorkReq> {
+export function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): BlsWorkReq {
   switch (job.type) {
     case JobQueueItemType.default:
       return {
@@ -61,9 +61,17 @@ export async function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null)
         })),
       };
     case JobQueueItemType.sameMessage: {
-      const {pk, sig} = await asyncAggregateWithRandomness(
-        job.sets.map((set) => ({pk: set.publicKey, sig: set.signature}))
-      );
+      // This is slow code on main thread (mainly signature deserialization + group check).
+      // Ideally it can be taken off-thread, but in the mean time, keep track of total time spent here.
+      // As of July 2024, for a node subscribing to all subnets, with 1 signature per validator per epoch,
+      // it takes around 2.02 min to perform this operation for a single epoch.
+      // cpu profile on main thread has 250s idle so this only works until we reach 3M validators
+      // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators
+      // and not a problem in the near future
+      // this is monitored on v1.21.0 https://github.com/ChainSafe/lodestar/pull/6894/files#r1687359225
+      const timer = metrics?.blsThreadPool.aggregateWithRandomnessMainThreadDuration.startTimer();
+      const {pk, sig} = aggregateWithRandomness(job.sets.map((set) => ({pk: set.publicKey, sig: set.signature})));
+      timer?.();
 
       return {
         opts: job.opts,

packages/beacon-node/src/chain/lightClient/index.ts

@@ -1,61 +1,64 @@
 import {BitArray, CompositeViewDU} from "@chainsafe/ssz";
+import {routes} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {
+  LightClientUpdateSummary,
+  isBetterUpdate,
+  toLightClientUpdateSummary,
+  upgradeLightClientHeader,
+} from "@lodestar/light-client/spec";
+import {
+  ForkExecution,
+  ForkLightClient,
+  ForkName,
+  ForkSeq,
+  MIN_SYNC_COMMITTEE_PARTICIPANTS,
+  SYNC_COMMITTEE_SIZE,
+  forkLightClient,
+  highestFork,
+  isForkPostElectra,
+} from "@lodestar/params";
+import {
+  CachedBeaconStateAltair,
+  computeStartSlotAtEpoch,
+  computeSyncPeriodAtEpoch,
+  computeSyncPeriodAtSlot,
+  executionPayloadToPayloadHeader,
+} from "@lodestar/state-transition";
 import {
-  altair,
   BeaconBlock,
   BeaconBlockBody,
   LightClientBootstrap,
   LightClientFinalityUpdate,
   LightClientHeader,
   LightClientOptimisticUpdate,
   LightClientUpdate,
-  phase0,
   Root,
   RootHex,
+  SSZTypesFor,
   Slot,
+  SyncPeriod,
+  altair,
+  electra,
+  phase0,
   ssz,
   sszTypesFor,
-  SSZTypesFor,
-  SyncPeriod,
 } from "@lodestar/types";
-import {ChainForkConfig} from "@lodestar/config";
-import {
-  CachedBeaconStateAltair,
-  computeStartSlotAtEpoch,
-  computeSyncPeriodAtEpoch,
-  computeSyncPeriodAtSlot,
-  executionPayloadToPayloadHeader,
-} from "@lodestar/state-transition";
-import {
-  isBetterUpdate,
-  toLightClientUpdateSummary,
-  LightClientUpdateSummary,
-  upgradeLightClientHeader,
-} from "@lodestar/light-client/spec";
 import {Logger, MapDef, pruneSetToMax, toRootHex} from "@lodestar/utils";
-import {routes} from "@lodestar/api";
-import {
-  MIN_SYNC_COMMITTEE_PARTICIPANTS,
-  SYNC_COMMITTEE_SIZE,
-  ForkName,
-  ForkSeq,
-  ForkExecution,
-  ForkLightClient,
-  highestFork,
-  forkLightClient,
-} from "@lodestar/params";
 
+import {ZERO_HASH} from "../../constants/index.js";
 import {IBeaconDb} from "../../db/index.js";
+import {NUM_WITNESS, NUM_WITNESS_ELECTRA} from "../../db/repositories/lightclientSyncCommitteeWitness.js";
 import {Metrics} from "../../metrics/index.js";
-import {ChainEventEmitter} from "../emitter.js";
 import {byteArrayEquals} from "../../util/bytes.js";
-import {ZERO_HASH} from "../../constants/index.js";
+import {ChainEventEmitter} from "../emitter.js";
 import {LightClientServerError, LightClientServerErrorCode} from "../errors/lightClientError.js";
 import {
+  getBlockBodyExecutionHeaderProof,
+  getCurrentSyncCommitteeBranch,
+  getFinalizedRootProof,
   getNextSyncCommitteeBranch,
   getSyncCommitteesWitness,
-  getFinalizedRootProof,
-  getCurrentSyncCommitteeBranch,
-  getBlockBodyExecutionHeaderProof,
 } from "./proofs.js";
 
 export type LightClientServerOpts = {
@@ -208,7 +211,10 @@ export class LightClientServer {
   private checkpointHeaders = new Map<BlockRooHex, LightClientHeader>();
   private latestHeadUpdate: LightClientOptimisticUpdate | null = null;
 
-  private readonly zero: Pick<altair.LightClientUpdate, "finalityBranch" | "finalizedHeader">;
+  private readonly zero: Pick<
+    altair.LightClientUpdate | electra.LightClientUpdate,
+    "finalityBranch" | "finalizedHeader"
+  >;
   private finalized: LightClientFinalityUpdate | null = null;
 
   constructor(
@@ -225,7 +231,9 @@ export class LightClientServer {
     this.zero = {
       // Assign the hightest fork's default value because it can always be typecasted down to correct fork
       finalizedHeader: sszTypesFor(highestFork(forkLightClient)).LightClientHeader.defaultValue(),
-      finalityBranch: ssz.altair.LightClientUpdate.fields.finalityBranch.defaultValue(),
+      // Electra finalityBranch has fixed length of 5 whereas altair has 4. The fifth element will be ignored
+      // when serializing as altair LightClientUpdate
+      finalityBranch: ssz.electra.LightClientUpdate.fields.finalityBranch.defaultValue(),
     };
 
     if (metrics) {
@@ -388,12 +396,13 @@ export class LightClientServer {
     parentBlockSlot: Slot
   ): Promise<void> {
     const blockSlot = block.slot;
-    const header = blockToLightClientHeader(this.config.getForkName(blockSlot), block);
+    const fork = this.config.getForkName(blockSlot);
+    const header = blockToLightClientHeader(fork, block);
 
     const blockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(header.beacon);
     const blockRootHex = toRootHex(blockRoot);
 
-    const syncCommitteeWitness = getSyncCommitteesWitness(postState);
+    const syncCommitteeWitness = getSyncCommitteesWitness(fork, postState);
 
     // Only store current sync committee once per run
     if (!this.storedCurrentSyncCommittee) {
@@ -621,6 +630,16 @@ export class LightClientServer {
     if (!syncCommitteeWitness) {
       throw Error(`syncCommitteeWitness not available at ${toRootHex(attestedData.blockRoot)}`);
     }
+
+    const attestedFork = this.config.getForkName(attestedHeader.beacon.slot);
+    const numWitness = syncCommitteeWitness.witness.length;
+    if (isForkPostElectra(attestedFork) && numWitness !== NUM_WITNESS_ELECTRA) {
+      throw Error(`Expected ${NUM_WITNESS_ELECTRA} witnesses in post-Electra numWitness=${numWitness}`);
+    }
+    if (!isForkPostElectra(attestedFork) && numWitness !== NUM_WITNESS) {
+      throw Error(`Expected ${NUM_WITNESS} witnesses in pre-Electra numWitness=${numWitness}`);
+    }
+
     const nextSyncCommittee = await this.db.syncCommittee.get(syncCommitteeWitness.nextSyncCommitteeRoot);
     if (!nextSyncCommittee) {
       throw Error("nextSyncCommittee not available");
@@ -641,7 +660,6 @@ export class LightClientServer {
       finalityBranch = attestedData.finalityBranch;
       finalizedHeader = finalizedHeaderAttested;
       // Fork of LightClientUpdate is based off on attested header's fork
-      const attestedFork = this.config.getForkName(attestedHeader.beacon.slot);
       if (this.config.getForkName(finalizedHeader.beacon.slot) !== attestedFork) {
         finalizedHeader = upgradeLightClientHeader(this.config, attestedFork, finalizedHeader);
       }