From 8117ebfd2b60ce6a15d8dceff77c7c9af3ac437e Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Fri, 24 Jan 2025 11:16:00 +0000 Subject: [PATCH 1/8] Update ref tests to v1.5.0-beta.0 (#9032) --- build.gradle | 5 ++--- .../tech/pegasys/teku/reference/Eth2ReferenceTestCase.java | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index e06251b765f..c5652d69cf8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,10 @@ import com.github.jk1.license.filter.LicenseBundleNormalizer +import groovy.json.JsonSlurper import tech.pegasys.internal.license.reporter.GroupedLicenseHtmlRenderer import tech.pegasys.teku.depcheck.DepCheckPlugin import java.text.SimpleDateFormat -import groovy.json.JsonSlurper - import static tech.pegasys.teku.repackage.Repackage.repackage buildscript { @@ -325,7 +324,7 @@ allprojects { } def nightly = System.getenv("NIGHTLY") != null -def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.10" +def refTestVersion = nightly ? "nightly" : "v1.5.0-beta.0" def blsRefTestVersion = 'v0.1.2' def slashingProtectionInterchangeRefTestVersion = 'v5.3.0' def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download' diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java index 39c53c17b92..8047e00cfda 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java @@ -53,6 +53,7 @@ public abstract class Eth2ReferenceTestCase { .put("light_client/single_merkle_proof", TestExecutor.IGNORE_TESTS) .put("light_client/sync", TestExecutor.IGNORE_TESTS) .put("light_client/update_ranking", TestExecutor.IGNORE_TESTS) + .put("light_client/data_collection", TestExecutor.IGNORE_TESTS) .build(); private static final ImmutableMap PHASE_0_TEST_TYPES = From d9b07ea7fbe34a16ee761175fb68c5becfa64f80 Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Sat, 25 Jan 2025 08:52:02 +1100 Subject: [PATCH 2/8] cleanup stale todo's (#9028) Removed the SlashingProtectedValidatorSource todo as its had several PRs from bots, and it's actually grossly over-simplifying. Re-worded to remove TODO in EraFile, as its just a POC, and we won't be accepting any tiny changes there, the note was for when its rewritten. Removed the TODO from SszCollectionSchema but left the content, as it does seem valid. remove stale reference to a PR #3356 Fixes #7532 Signed-off-by: Paul Harris --- .../java/tech/pegasys/teku/data/eraFileFormat/EraFile.java | 2 +- .../beaconstate/common/AbstractBeaconStateSchemaTest.java | 1 - .../teku/infrastructure/ssz/schema/SszCollectionSchema.java | 1 - .../teku/networking/eth2/gossip/config/ScoringConfig.java | 2 -- .../teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java | 6 ++---- .../client/loader/SlashingProtectedValidatorSource.java | 2 -- 6 files changed, 3 insertions(+), 11 deletions(-) diff --git a/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java b/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java index ee64be4a9bd..ff1e8e98943 100644 --- a/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java +++ b/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java @@ -214,7 +214,7 @@ private void verifyBlocksWithReferenceState( block.getParentRoot().equals(previousArchiveLastBlock.getRoot()), "First block in archive does not match last block of previous archive."); } - // TODO should verify signature + // when fully implemented, we would check signature also ++populatedSlots; } System.out.println( diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java index 353f9bc3ea1..e112e5f5e2a 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java @@ -97,7 +97,6 @@ public void changeSpecConfigTest() { @Test void roundTripViaSsz() { - // TODO - generate random version-specific state BeaconState beaconState = randomState(); Bytes bytes = beaconState.sszSerialize(); BeaconState state = schema.sszDeserialize(bytes); diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java index 3b672c40c10..4484373b73c 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java @@ -49,7 +49,6 @@ default SszCollectionT createFromElements(final List elem } default TreeNode createTreeFromElements(final List elements) { - // TODO: probably suboptimal method implementation: // This is a generic implementation which works for both Vector and List but it potentially // could do better if construct the tree directly in List/Vector subclasses checkArgument(elements.size() <= getMaxLength(), "Too many elements for this collection type"); diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java index e35bb4370bc..edf1e1905ef 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java @@ -54,8 +54,6 @@ class ScoringConfig { private ScoringConfig(final Spec spec, final int d) { this.spec = spec; - // TODO(#3356) Use spec provider through-out rather than relying only on genesis constants and - // genesis spec this.genesisConfig = spec.getGenesisSpecConfig(); this.genesisSpec = spec.getGenesisSpec(); this.d = d; diff --git a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java index 5b809f77d14..10a79d33644 100644 --- a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java +++ b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java @@ -215,11 +215,9 @@ protected Host createHost(final PrivKey privKey, final List advertise b.getTransports().add(TcpTransport::new); b.getSecureChannels().add(NoiseXXSecureChannel::new); - // yamux MUST take precedence during negotiation + // Yamux must take precedence during negotiation if (config.isYamuxEnabled()) { - // TODO: https://github.com/Consensys/teku/issues/7532 - final int maxBufferedConnectionWrites = 150 * 1024 * 1024; - b.getMuxers().add(StreamMuxerProtocol.getYamux(maxBufferedConnectionWrites)); + b.getMuxers().add(StreamMuxerProtocol.getYamux()); } b.getMuxers().add(StreamMuxerProtocol.getMplex()); diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java index 08abd012b6e..85d76ec62ab 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java @@ -90,8 +90,6 @@ public boolean isReadOnly() { @Override public Signer createSigner() { - // TODO: Consider caching these to guarantee we can't possible use different - // `SlashingProtectedSigner` instances with the same key return new SlashingProtectedSigner( getPublicKey(), slashingProtector, delegate.createSigner()); } From 28f69d39de2f5ddaa9fc21bc52539f9910d12a91 Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Sun, 26 Jan 2025 22:15:49 +1100 Subject: [PATCH 3/8] undo yamux revert (#9036) Using a plain ticket reference without TODO should still ensure we can track the issue. also added ticket reference for SszCollectionSchema.java Signed-off-by: Paul Harris --- .../teku/infrastructure/ssz/schema/SszCollectionSchema.java | 1 + .../teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java index 4484373b73c..9b04c382f48 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java @@ -49,6 +49,7 @@ default SszCollectionT createFromElements(final List elem } default TreeNode createTreeFromElements(final List elements) { + // https://github.com/Consensys/teku/issues/9035 // This is a generic implementation which works for both Vector and List but it potentially // could do better if construct the tree directly in List/Vector subclasses checkArgument(elements.size() <= getMaxLength(), "Too many elements for this collection type"); diff --git a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java index 10a79d33644..eb9d1c0df68 100644 --- a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java +++ b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java @@ -217,7 +217,9 @@ protected Host createHost(final PrivKey privKey, final List advertise // Yamux must take precedence during negotiation if (config.isYamuxEnabled()) { - b.getMuxers().add(StreamMuxerProtocol.getYamux()); + // https://github.com/Consensys/teku/issues/7532 + final int maxBufferedConnectionWrites = 150 * 1024 * 1024; + b.getMuxers().add(StreamMuxerProtocol.getYamux(maxBufferedConnectionWrites)); } b.getMuxers().add(StreamMuxerProtocol.getMplex()); From 5c8ccdbfadef68d3f504e2817f59da7489ac2cd2 Mon Sep 17 00:00:00 2001 From: Enrico Del Fante Date: Mon, 27 Jan 2025 11:27:04 +0100 Subject: [PATCH 4/8] Add custom iterator for LevelDB (#9033) --- .../server/leveldb/CustomDBIterator.java | 27 +++ .../storage/server/leveldb/CustomJniDB.java | 63 ++++++ .../server/leveldb/CustomJniDBFactory.java | 156 +++++++++++++++ .../server/leveldb/CustomJniDBIterator.java | 181 ++++++++++++++++++ .../server/leveldb/LevelDbInstance.java | 9 +- .../leveldb/LevelDbInstanceFactory.java | 4 +- .../server/leveldb/LevelDbIterator.java | 7 +- .../server/leveldb/LevelDbKeyIterator.java | 9 +- 8 files changed, 441 insertions(+), 15 deletions(-) create mode 100644 storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java create mode 100644 storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java create mode 100644 storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java create mode 100644 storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java new file mode 100644 index 00000000000..a41bd0cf624 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java @@ -0,0 +1,27 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import org.iq80.leveldb.DBIterator; + +/** + * This interface extends the DBIterator interface to provide additional methods for peeking at the + * next key which are used to avoid unnecessary value allocations + */ +public interface CustomDBIterator extends DBIterator { + + byte[] peekNextKey(); + + byte[] nextKey(); +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java new file mode 100644 index 00000000000..13ef74b9073 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java @@ -0,0 +1,63 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import org.fusesource.leveldbjni.internal.JniDB; +import org.fusesource.leveldbjni.internal.NativeCache; +import org.fusesource.leveldbjni.internal.NativeComparator; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.fusesource.leveldbjni.internal.NativeLogger; +import org.fusesource.leveldbjni.internal.NativeReadOptions; +import org.iq80.leveldb.DBException; +import org.iq80.leveldb.DBIterator; +import org.iq80.leveldb.ReadOptions; + +/** This class extends the JniDB class to provide a custom DBIterator. */ +public class CustomJniDB extends JniDB { + private final NativeDB db; + + public CustomJniDB( + final NativeDB db, + final NativeCache cache, + final NativeComparator comparator, + final NativeLogger logger) { + super(db, cache, comparator, logger); + this.db = db; + } + + @Override + public DBIterator iterator(final ReadOptions options) { + if (this.db == null) { + throw new DBException("Closed"); + } else { + return new CustomJniDBIterator(this.db.iterator(this.convert(options))); + } + } + + // this method is private in the super class, so has been copied here + private NativeReadOptions convert(final ReadOptions options) { + if (options == null) { + return null; + } else { + NativeReadOptions rc = new NativeReadOptions(); + rc.fillCache(options.fillCache()); + rc.verifyChecksums(options.verifyChecksums()); + if (options.snapshot() != null) { + throw new UnsupportedOperationException("Snapshots are not supported"); + } + + return rc; + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java new file mode 100644 index 00000000000..81c879cab79 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java @@ -0,0 +1,156 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import org.fusesource.leveldbjni.JniDBFactory; +import org.fusesource.leveldbjni.internal.NativeCache; +import org.fusesource.leveldbjni.internal.NativeComparator; +import org.fusesource.leveldbjni.internal.NativeCompressionType; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.fusesource.leveldbjni.internal.NativeLogger; +import org.fusesource.leveldbjni.internal.NativeOptions; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBComparator; +import org.iq80.leveldb.Logger; +import org.iq80.leveldb.Options; + +/** + * This class extends the JniDBFactory class to provide a custom DBIterator. Only {@link + * CustomJniDBFactory#open} has been changed from the original class + */ +@SuppressWarnings("EmptyCatch") +public class CustomJniDBFactory extends JniDBFactory { + public static final CustomJniDBFactory FACTORY = new CustomJniDBFactory(); + public static final String VERSION; + + @Override + public DB open(final File path, final Options options) throws IOException { + final OptionsResourceHolder holder = new OptionsResourceHolder(); + + NativeDB db = null; + try { + holder.init(options); + db = NativeDB.open(holder.options, path); + } finally { + if (db == null) { + holder.close(); + } + } + + // this is the only line that has been functionally changed from the original class + return new CustomJniDB(db, holder.cache, holder.comparator, holder.logger); + } + + static { + NativeDB.LIBRARY.load(); + String v = "unknown"; + + try (final InputStream is = JniDBFactory.class.getResourceAsStream("version.txt")) { + if (is != null) { + try (final BufferedReader reader = + new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + v = reader.readLine(); + } + } + } catch (final Throwable ignored) { + } + + VERSION = v; + } + + private static class OptionsResourceHolder { + NativeCache cache; + NativeComparator comparator; + NativeLogger logger; + NativeOptions options; + + private OptionsResourceHolder() { + this.cache = null; + this.comparator = null; + this.logger = null; + } + + private void init(final Options value) { + this.options = new NativeOptions(); + this.options.blockRestartInterval(value.blockRestartInterval()); + this.options.blockSize((long) value.blockSize()); + this.options.createIfMissing(value.createIfMissing()); + this.options.errorIfExists(value.errorIfExists()); + this.options.maxOpenFiles(value.maxOpenFiles()); + this.options.paranoidChecks(value.paranoidChecks()); + this.options.writeBufferSize((long) value.writeBufferSize()); + switch (value.compressionType()) { + case NONE: + this.options.compression(NativeCompressionType.kNoCompression); + break; + case SNAPPY: + this.options.compression(NativeCompressionType.kSnappyCompression); + } + + if (value.cacheSize() > 0L) { + this.cache = new NativeCache(value.cacheSize()); + this.options.cache(this.cache); + } + + final DBComparator userComparator = value.comparator(); + if (userComparator != null) { + this.comparator = + new NativeComparator() { + @Override + public int compare(final byte[] key1, final byte[] key2) { + return userComparator.compare(key1, key2); + } + + @Override + public String name() { + return userComparator.name(); + } + }; + this.options.comparator(this.comparator); + } + + final Logger userLogger = value.logger(); + if (userLogger != null) { + this.logger = + new NativeLogger() { + @Override + public void log(final String message) { + userLogger.log(message); + } + }; + this.options.infoLog(this.logger); + } + } + + private void close() { + if (this.cache != null) { + this.cache.delete(); + } + + if (this.comparator != null) { + this.comparator.delete(); + } + + if (this.logger != null) { + this.logger.delete(); + } + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java new file mode 100644 index 00000000000..95c53310a7a --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java @@ -0,0 +1,181 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import java.util.AbstractMap; +import java.util.Map; +import java.util.NoSuchElementException; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.fusesource.leveldbjni.internal.NativeIterator; + +/** + * This is a copy of + * which also implements the methods from {@link CustomDBIterator} + */ +public class CustomJniDBIterator implements CustomDBIterator { + private final NativeIterator iterator; + + CustomJniDBIterator(final NativeIterator iterator) { + this.iterator = iterator; + } + + @Override + public void close() { + this.iterator.delete(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(final byte[] key) { + try { + this.iterator.seek(key); + } catch (final NativeDB.DBException e) { + if (e.isNotFound()) { + throw new NoSuchElementException(); + } else { + throw new RuntimeException(e); + } + } + } + + @Override + public void seekToFirst() { + this.iterator.seekToFirst(); + } + + @Override + public void seekToLast() { + this.iterator.seekToLast(); + } + + @Override + public Map.Entry peekNext() { + if (!this.iterator.isValid()) { + throw new NoSuchElementException(); + } else { + try { + return new AbstractMap.SimpleImmutableEntry<>(this.iterator.key(), this.iterator.value()); + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public boolean hasNext() { + return this.iterator.isValid(); + } + + @Override + public Map.Entry next() { + final Map.Entry entry = this.peekNext(); + + try { + this.iterator.next(); + return entry; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean hasPrev() { + if (!this.iterator.isValid()) { + return false; + } else { + try { + this.iterator.prev(); + + final boolean ret; + try { + ret = this.iterator.isValid(); + } finally { + if (this.iterator.isValid()) { + this.iterator.next(); + } else { + this.iterator.seekToFirst(); + } + } + + return ret; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public Map.Entry peekPrev() { + try { + this.iterator.prev(); + + final Map.Entry entry; + try { + entry = this.peekNext(); + } finally { + if (this.iterator.isValid()) { + this.iterator.next(); + } else { + this.iterator.seekToFirst(); + } + } + + return entry; + } catch (final NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + + @Override + public Map.Entry prev() { + final Map.Entry rc = this.peekPrev(); + + try { + this.iterator.prev(); + return rc; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + + @Override + public byte[] peekNextKey() { + if (!this.iterator.isValid()) { + throw new NoSuchElementException(); + } else { + try { + return this.iterator.key(); + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public byte[] nextKey() { + final byte[] key = this.peekNextKey(); + + try { + this.iterator.next(); + return key; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java index ef92af2c47b..634bfac9bf2 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java @@ -296,7 +296,7 @@ private Stream streamKeys( private Stream> streamRaw( final KvStoreColumn column, final byte[] fromBytes, final byte[] toBytes) { assertOpen(); - final DBIterator iterator = createIterator(); + final CustomDBIterator iterator = createIterator(); iterator.seek(fromBytes); return new LevelDbIterator<>(this, iterator, column, toBytes) .toStream() @@ -307,7 +307,7 @@ private Stream> streamRaw( private Stream streamKeysRaw( final KvStoreColumn column, final byte[] fromBytes, final byte[] toBytes) { assertOpen(); - final DBIterator iterator = createIterator(); + final CustomDBIterator iterator = createIterator(); iterator.seek(fromBytes); return new LevelDbKeyIterator<>(this, iterator, column, toBytes) .toStream() @@ -371,8 +371,9 @@ synchronized void onTransactionClosed(final LevelDbTransaction transaction) { } } - private DBIterator createIterator() { - final DBIterator iterator = db.iterator(new ReadOptions().fillCache(false)); + private CustomDBIterator createIterator() { + final CustomDBIterator iterator = + (CustomDBIterator) db.iterator(new ReadOptions().fillCache(false)); openIterators.add(iterator); openedIteratorsCounter.inc(); return iterator; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java index f3142174644..da1082c68f6 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.util.Collection; -import org.fusesource.leveldbjni.JniDBFactory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.iq80.leveldb.DB; @@ -45,7 +44,8 @@ public static KvStoreAccessor create( .writeBufferSize(configuration.getLeveldbWriteBufferSize()); try { - final DB db = JniDBFactory.factory.open(configuration.getDatabaseDir().toFile(), options); + final DB db = + CustomJniDBFactory.FACTORY.open(configuration.getDatabaseDir().toFile(), options); return new LevelDbInstance(db, metricsSystem, metricCategory); } catch (final IOException e) { throw DatabaseStorageException.unrecoverable("Failed to open database", e); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java index d40e0e2f938..41612fca0e4 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java @@ -22,20 +22,19 @@ import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.iq80.leveldb.DBIterator; import tech.pegasys.teku.storage.server.kvstore.ColumnEntry; import tech.pegasys.teku.storage.server.kvstore.schema.KvStoreColumn; public class LevelDbIterator implements Iterator> { private final LevelDbInstance dbInstance; - private final DBIterator iterator; + private final CustomDBIterator iterator; private final KvStoreColumn column; private final byte[] lastKey; public LevelDbIterator( final LevelDbInstance dbInstance, - final DBIterator iterator, + final CustomDBIterator iterator, final KvStoreColumn column, final byte[] lastKey) { this.dbInstance = dbInstance; @@ -53,7 +52,7 @@ public boolean hasNext() { } private boolean isValidKey() { - final byte[] nextKey = iterator.peekNext().getKey(); + final byte[] nextKey = iterator.peekNextKey(); return isFromColumn(column, nextKey) && Arrays.compareUnsigned(nextKey, lastKey) <= 0; } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java index 3a4f6e2c6e0..7cb93e9bce9 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java @@ -22,19 +22,18 @@ import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.iq80.leveldb.DBIterator; import tech.pegasys.teku.storage.server.kvstore.schema.KvStoreColumn; public class LevelDbKeyIterator implements Iterator { private final LevelDbInstance dbInstance; - private final DBIterator iterator; + private final CustomDBIterator iterator; private final KvStoreColumn column; private final byte[] lastKey; public LevelDbKeyIterator( final LevelDbInstance dbInstance, - final DBIterator iterator, + final CustomDBIterator iterator, final KvStoreColumn column, final byte[] lastKey) { this.dbInstance = dbInstance; @@ -52,7 +51,7 @@ public boolean hasNext() { } private boolean isValidKey() { - final byte[] nextKey = iterator.peekNext().getKey(); + final byte[] nextKey = iterator.peekNextKey(); return isFromColumn(column, nextKey) && Arrays.compareUnsigned(nextKey, lastKey) <= 0; } @@ -60,7 +59,7 @@ private boolean isValidKey() { public byte[] next() { synchronized (dbInstance) { dbInstance.assertOpen(); - return removeKeyPrefix(column, iterator.next().getKey()); + return removeKeyPrefix(column, iterator.nextKey()); } } From e44a47c461ff0ccd7054bf8de131ef6bae400a5c Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Mon, 27 Jan 2025 18:02:19 +0100 Subject: [PATCH 5/8] update empty storage initialization log (#9043) --- .../pegasys/teku/infrastructure/logging/StatusLogger.java | 4 ++++ .../teku/storage/client/StorageBackedRecentChainData.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java index ee86cd6406d..c62402e2966 100644 --- a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java +++ b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java @@ -278,6 +278,10 @@ public void finishInitializingChainData() { log.info("Storage initialization complete"); } + public void emptyChainData() { + log.info("Empty storage. Initialization complete."); + } + public void recordedFinalizedBlocks(final int numberRecorded, final int totalToRecord) { log.info("Recorded {} of {} finalized blocks", numberRecorded, totalToRecord); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java index 5ffa9a7e22e..d011f4fdfa2 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java @@ -166,7 +166,7 @@ private SafeFuture processStoreFuture( return storeFuture.thenApply( maybeData -> { if (maybeData.isEmpty()) { - STATUS_LOG.finishInitializingChainData(); + STATUS_LOG.emptyChainData(); return this; } From ed1bedbd50e3d68d2da6355e9dc2ab3e800000de Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:37:24 -0600 Subject: [PATCH 6/8] Update ref tests to v1.5.0-beta.1 (#9044) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c5652d69cf8..4e743c110e8 100644 --- a/build.gradle +++ b/build.gradle @@ -324,7 +324,7 @@ allprojects { } def nightly = System.getenv("NIGHTLY") != null -def refTestVersion = nightly ? "nightly" : "v1.5.0-beta.0" +def refTestVersion = nightly ? "nightly" : "v1.5.0-beta.1" def blsRefTestVersion = 'v0.1.2' def slashingProtectionInterchangeRefTestVersion = 'v5.3.0' def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download' From 07b70e217cdf03cdb62d5c5801c9c53ec8c0dfb7 Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Tue, 28 Jan 2025 13:04:09 +1100 Subject: [PATCH 7/8] debug command to see what values are in variables (#9030) Signed-off-by: Paul Harris --- .../pegasys/teku/storage/server/Database.java | 2 ++ .../server/kvstore/KvStoreDatabase.java | 5 +++ .../dataaccess/CombinedKvStoreDao.java | 31 +++++++++++++++++ .../dataaccess/KvStoreCombinedDao.java | 2 ++ .../dataaccess/KvStoreCombinedDaoAdapter.java | 5 +++ .../storage/server/noop/NoOpDatabase.java | 5 +++ .../cli/subcommand/debug/DebugDbCommand.java | 34 +++++++++++++++++++ 7 files changed, 84 insertions(+) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java index 4f8751cc44b..162db230511 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java @@ -215,6 +215,8 @@ default Stream streamBlobSidecarKeys(final UInt64 Map getColumnCounts(final Optional maybeColumnFilter); + Map> getVariables(); + long getBlobSidecarColumnCount(); long getNonCanonicalBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index c7d5ead766b..2cd6d3c943d 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -380,6 +380,11 @@ public Map getColumnCounts(final Optional maybeColumnFilte return dao.getColumnCounts(maybeColumnFilter); } + @Override + public Map> getVariables() { + return dao.getVariables(); + } + @Override public long getBlobSidecarColumnCount() { return dao.getBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java index 83bda83c9a8..6e9f46d0c1e 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java @@ -432,6 +432,37 @@ public Map getColumnCounts(final Optional maybeColumnFilte return columnCounts; } + @Override + public Map> getVariables() { + Map> variables = new LinkedHashMap<>(); + variables.put("GENESIS_TIME", getGenesisTime().map(UInt64::toString)); + variables.put("JUSTIFIED_CHECKPOINT", getJustifiedCheckpoint().map(Checkpoint::toString)); + variables.put( + "BEST_JUSTIFIED_CHECKPOINT", getBestJustifiedCheckpoint().map(Checkpoint::toString)); + variables.put("FINALIZED_CHECKPOINT", getFinalizedCheckpoint().map(Checkpoint::toString)); + variables.put( + "WEAK_SUBJECTIVITY_CHECKPOINT", getWeakSubjectivityCheckpoint().map(Checkpoint::toString)); + variables.put("ANCHOR_CHECKPOINT", getAnchor().map(Checkpoint::toString)); + variables.put( + "FINALIZED_DEPOSIT_SNAPSHOT", + getFinalizedDepositSnapshot().map(DepositTreeSnapshot::toString)); + try { + variables.put( + "LATEST_FINALIZED_STATE", + getLatestFinalizedState() + .map( + state -> + "BeaconState{slot=" + + state.getSlot() + + ", root=" + + state.hashTreeRoot().toHexString() + + "}")); + } catch (final Exception e) { + variables.put("FINALIZED_STATE", Optional.of(e.toString())); + } + return variables; + } + @Override public long getBlobSidecarColumnCount() { final KvStoreColumn column = diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java index 2dba5e05e70..438c0b16e51 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java @@ -154,6 +154,8 @@ List getNonCanonicalBlobSidecarKeys( Map getColumnCounts(final Optional maybeColumnFilter); + Map> getVariables(); + long getBlobSidecarColumnCount(); long getNonCanonicalBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java index b9d918f98dc..da50d25e596 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java @@ -228,6 +228,11 @@ public Map getColumnCounts(final Optional maybeColumnFilte return result; } + @Override + public Map> getVariables() { + return Map.of(); + } + @Override public long getBlobSidecarColumnCount() { return finalizedDao.getBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java index af522b0c619..dc1b3bd4969 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java @@ -256,6 +256,11 @@ public Map getColumnCounts(final Optional maybeColumnFilte return new HashMap<>(); } + @Override + public Map> getVariables() { + return new HashMap<>(); + } + @Override public long getBlobSidecarColumnCount() { return 0L; diff --git a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java index aa32be3ba42..04a048a2cdd 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java @@ -305,6 +305,40 @@ public int getColumnCounts( return 0; } + @Command( + name = "get-variables", + description = "Check variable values", + mixinStandardHelpOptions = true, + showDefaultValues = true, + abbreviateSynopsis = true, + versionProvider = PicoCliVersionProvider.class, + synopsisHeading = "%n", + descriptionHeading = "%nDescription:%n%n", + optionListHeading = "%nOptions:%n", + footerHeading = "%n", + footer = "Teku is licensed under the Apache License 2.0") + public int getVariables( + @Mixin final BeaconNodeDataOptions beaconNodeDataOptions, + @Mixin final Eth2NetworkOptions eth2NetworkOptions, + @Option( + names = {"--filter"}, + description = "Only get variables that match a given filter.") + final String filter) + throws Exception { + try (final Database database = createDatabase(beaconNodeDataOptions, eth2NetworkOptions)) { + final Map> variables = database.getVariables(); + variables + .keySet() + .forEach( + k -> { + if (filter == null || k.contains(filter)) { + System.out.printf("%-30s: %s\n", k, variables.get(k).orElse("EMPTY")); + } + }); + } + return 0; + } + @Command( name = "validate-block-history", description = "Validate the chain of finalized blocks via parent references", From 05a0c70d652a860e7af95201ce8bcedcfe9c3041 Mon Sep 17 00:00:00 2001 From: Paul Harris Date: Tue, 28 Jan 2025 15:00:08 +1100 Subject: [PATCH 8/8] rocksdb nits (#9046) - also not sure that it's a good plan to not call assertOpen during setup of the streams, so asserted open in those fns. Signed-off-by: Paul Harris --- .../storage/server/rocksdb/RocksDbInstance.java | 4 +++- .../server/rocksdb/RocksDbKeyIterator.java | 2 +- .../teku/storage/server/rocksdb/RocksDbStats.java | 15 +++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java index 8ed14e3ab5d..e5f5110e75c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java @@ -201,7 +201,7 @@ public , V> Stream streamKeys( @MustBeClosed public synchronized KvStoreTransaction startTransaction() { assertOpen(); - RocksDbTransaction tx = + final RocksDbTransaction tx = new RocksDbTransaction(db, defaultHandle, columnHandles, openTransactions::remove); openTransactions.add(tx); return tx; @@ -249,6 +249,7 @@ private Stream> createStreamRaw( final KvStoreColumn column, final Consumer setupIterator, final Predicate continueTest) { + assertOpen(); final ColumnFamilyHandle handle = columnHandles.get(column); final RocksIterator rocksDbIterator = db.newIterator(handle); setupIterator.accept(rocksDbIterator); @@ -261,6 +262,7 @@ private Stream createStreamKeyRaw( final KvStoreColumn column, final Consumer setupIterator, final Predicate continueTest) { + assertOpen(); final ColumnFamilyHandle handle = columnHandles.get(column); final RocksIterator rocksDbIterator = db.newIterator(handle); setupIterator.accept(rocksDbIterator); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java index d45e48ea92d..ffd32385e19 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java @@ -98,7 +98,7 @@ public Stream toStream() { } private void assertOpen() { - if (this.isDatabaseClosed.get()) { + if (isDatabaseClosed.get()) { throw new ShuttingDownException(); } if (closed.get()) { diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java index d9801c8a1ff..1a34ffcad87 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -150,7 +151,7 @@ public class RocksDbStats implements AutoCloseable { HistogramType.READ_NUM_MERGE_OPERANDS, }; - private boolean closed = false; + private final AtomicBoolean closed = new AtomicBoolean(false); private final Statistics stats; private final MetricsSystem metricsSystem; private final MetricCategory category; @@ -230,16 +231,14 @@ private long getLongProperty(final RocksDB database, final String name) { } @Override - public synchronized void close() { - if (closed) { - return; + public void close() { + if (closed.compareAndSet(false, true)) { + stats.close(); } - closed = true; - stats.close(); } - private synchronized T ifOpen(final Supplier supplier, final T defaultValue) { - if (closed) { + private T ifOpen(final Supplier supplier, final T defaultValue) { + if (closed.get()) { return defaultValue; } return supplier.get();