Merge branch 'master' into reuse-earliest-blob-slot

build.gradle

@@ -1,11 +1,10 @@
 import com.github.jk1.license.filter.LicenseBundleNormalizer
+import groovy.json.JsonSlurper
 import tech.pegasys.internal.license.reporter.GroupedLicenseHtmlRenderer
 import tech.pegasys.teku.depcheck.DepCheckPlugin
 
 import java.text.SimpleDateFormat
 
-import groovy.json.JsonSlurper
-
 import static tech.pegasys.teku.repackage.Repackage.repackage
 
 buildscript {
@@ -325,7 +324,7 @@ allprojects {
 }
 
 def nightly = System.getenv("NIGHTLY") != null
-def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.10"
+def refTestVersion = nightly ? "nightly" : "v1.5.0-beta.1"
 def blsRefTestVersion = 'v0.1.2'
 def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
 def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'

data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java

@@ -214,7 +214,7 @@ private void verifyBlocksWithReferenceState(
             block.getParentRoot().equals(previousArchiveLastBlock.getRoot()),
             "First block in archive does not match last block of previous archive.");
       }
-      // TODO should verify signature
+      // when fully implemented, we would check signature also
       ++populatedSlots;
     }
     System.out.println(

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java

@@ -53,6 +53,7 @@ public abstract class Eth2ReferenceTestCase {
           .put("light_client/single_merkle_proof", TestExecutor.IGNORE_TESTS)
           .put("light_client/sync", TestExecutor.IGNORE_TESTS)
           .put("light_client/update_ranking", TestExecutor.IGNORE_TESTS)
+          .put("light_client/data_collection", TestExecutor.IGNORE_TESTS)
           .build();
 
   private static final ImmutableMap<String, TestExecutor> PHASE_0_TEST_TYPES =

ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java

@@ -97,7 +97,6 @@ public void changeSpecConfigTest() {
 
   @Test
   void roundTripViaSsz() {
-    // TODO - generate random version-specific state
     BeaconState beaconState = randomState();
     Bytes bytes = beaconState.sszSerialize();
     BeaconState state = schema.sszDeserialize(bytes);

infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java

@@ -278,6 +278,10 @@ public void finishInitializingChainData() {
     log.info("Storage initialization complete");
   }
 
+  public void emptyChainData() {
+    log.info("Empty storage. Initialization complete.");
+  }
+
   public void recordedFinalizedBlocks(final int numberRecorded, final int totalToRecord) {
     log.info("Recorded {} of {} finalized blocks", numberRecorded, totalToRecord);
   }

infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java

@@ -49,7 +49,7 @@ default SszCollectionT createFromElements(final List<? extends SszElementT> elem
   }
 
   default TreeNode createTreeFromElements(final List<? extends SszElementT> elements) {
-    // TODO: probably suboptimal method implementation:
+    // https://github.com/Consensys/teku/issues/9035
     // This is a generic implementation which works for both Vector and List but it potentially
     // could do better if construct the tree directly in List/Vector subclasses
     checkArgument(elements.size() <= getMaxLength(), "Too many elements for this collection type");

networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java

@@ -54,8 +54,6 @@ class ScoringConfig {
 
   private ScoringConfig(final Spec spec, final int d) {
     this.spec = spec;
-    // TODO(#3356) Use spec provider through-out rather than relying only on genesis constants and
-    //  genesis spec
     this.genesisConfig = spec.getGenesisSpecConfig();
     this.genesisSpec = spec.getGenesisSpec();
     this.d = d;

networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java

@@ -215,9 +215,9 @@ protected Host createHost(final PrivKey privKey, final List<Multiaddr> advertise
           b.getTransports().add(TcpTransport::new);
           b.getSecureChannels().add(NoiseXXSecureChannel::new);
 
-          // yamux MUST take precedence during negotiation
+          // Yamux must take precedence during negotiation
           if (config.isYamuxEnabled()) {
-            // TODO: https://github.com/Consensys/teku/issues/7532
+            // https://github.com/Consensys/teku/issues/7532
             final int maxBufferedConnectionWrites = 150 * 1024 * 1024;
             b.getMuxers().add(StreamMuxerProtocol.getYamux(maxBufferedConnectionWrites));
           }

storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java

@@ -166,7 +166,7 @@ private SafeFuture<RecentChainData> processStoreFuture(
     return storeFuture.thenApply(
         maybeData -> {
           if (maybeData.isEmpty()) {
-            STATUS_LOG.finishInitializingChainData();
+            STATUS_LOG.emptyChainData();
             return this;
           }
 

storage/src/main/java/tech/pegasys/teku/storage/server/Database.java

@@ -215,6 +215,8 @@ default Stream<SlotAndBlockRootAndBlobIndex> streamBlobSidecarKeys(final UInt64
 
   Map<String, Long> getColumnCounts(final Optional<String> maybeColumnFilter);
 
+  Map<String, Optional<String>> getVariables();
+
   long getBlobSidecarColumnCount();
 
   long getNonCanonicalBlobSidecarColumnCount();

storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java

@@ -432,6 +432,37 @@ public Map<String, Long> getColumnCounts(final Optional<String> maybeColumnFilte
     return columnCounts;
   }
 
+  @Override
+  public Map<String, Optional<String>> getVariables() {
+    Map<String, Optional<String>> variables = new LinkedHashMap<>();
+    variables.put("GENESIS_TIME", getGenesisTime().map(UInt64::toString));
+    variables.put("JUSTIFIED_CHECKPOINT", getJustifiedCheckpoint().map(Checkpoint::toString));
+    variables.put(
+        "BEST_JUSTIFIED_CHECKPOINT", getBestJustifiedCheckpoint().map(Checkpoint::toString));
+    variables.put("FINALIZED_CHECKPOINT", getFinalizedCheckpoint().map(Checkpoint::toString));
+    variables.put(
+        "WEAK_SUBJECTIVITY_CHECKPOINT", getWeakSubjectivityCheckpoint().map(Checkpoint::toString));
+    variables.put("ANCHOR_CHECKPOINT", getAnchor().map(Checkpoint::toString));
+    variables.put(
+        "FINALIZED_DEPOSIT_SNAPSHOT",
+        getFinalizedDepositSnapshot().map(DepositTreeSnapshot::toString));
+    try {
+      variables.put(
+          "LATEST_FINALIZED_STATE",
+          getLatestFinalizedState()
+              .map(
+                  state ->
+                      "BeaconState{slot="
+                          + state.getSlot()
+                          + ", root="
+                          + state.hashTreeRoot().toHexString()
+                          + "}"));
+    } catch (final Exception e) {
+      variables.put("FINALIZED_STATE", Optional.of(e.toString()));
+    }
+    return variables;
+  }
+
   @Override
   public long getBlobSidecarColumnCount() {
     final KvStoreColumn<?, ?> column =

storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java

@@ -154,6 +154,8 @@ List<SlotAndBlockRootAndBlobIndex> getNonCanonicalBlobSidecarKeys(
 
   Map<String, Long> getColumnCounts(final Optional<String> maybeColumnFilter);
 
+  Map<String, Optional<String>> getVariables();
+
   long getBlobSidecarColumnCount();
 
   long getNonCanonicalBlobSidecarColumnCount();

storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java

@@ -228,6 +228,11 @@ public Map<String, Long> getColumnCounts(final Optional<String> maybeColumnFilte
     return result;
   }
 
+  @Override
+  public Map<String, Optional<String>> getVariables() {
+    return Map.of();
+  }
+
   @Override
   public long getBlobSidecarColumnCount() {
     return finalizedDao.getBlobSidecarColumnCount();

storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Consensys Software Inc., 2025
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.storage.server.leveldb;
+
+import org.iq80.leveldb.DBIterator;
+
+/**
+ * This interface extends the DBIterator interface to provide additional methods for peeking at the
+ * next key which are used to avoid unnecessary value allocations
+ */
+public interface CustomDBIterator extends DBIterator {
+
+  byte[] peekNextKey();
+
+  byte[] nextKey();
+}

storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Consensys Software Inc., 2025
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.storage.server.leveldb;
+
+import org.fusesource.leveldbjni.internal.JniDB;
+import org.fusesource.leveldbjni.internal.NativeCache;
+import org.fusesource.leveldbjni.internal.NativeComparator;
+import org.fusesource.leveldbjni.internal.NativeDB;
+import org.fusesource.leveldbjni.internal.NativeLogger;
+import org.fusesource.leveldbjni.internal.NativeReadOptions;
+import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.DBIterator;
+import org.iq80.leveldb.ReadOptions;
+
+/** This class extends the JniDB class to provide a custom DBIterator. */
+public class CustomJniDB extends JniDB {
+  private final NativeDB db;
+
+  public CustomJniDB(
+      final NativeDB db,
+      final NativeCache cache,
+      final NativeComparator comparator,
+      final NativeLogger logger) {
+    super(db, cache, comparator, logger);
+    this.db = db;
+  }
+
+  @Override
+  public DBIterator iterator(final ReadOptions options) {
+    if (this.db == null) {
+      throw new DBException("Closed");
+    } else {
+      return new CustomJniDBIterator(this.db.iterator(this.convert(options)));
+    }
+  }
+
+  // this method is private in the super class, so has been copied here
+  private NativeReadOptions convert(final ReadOptions options) {
+    if (options == null) {
+      return null;
+    } else {
+      NativeReadOptions rc = new NativeReadOptions();
+      rc.fillCache(options.fillCache());
+      rc.verifyChecksums(options.verifyChecksums());
+      if (options.snapshot() != null) {
+        throw new UnsupportedOperationException("Snapshots are not supported");
+      }
+
+      return rc;
+    }
+  }
+}