diff --git a/build.gradle b/build.gradle index e06251b765f..4e743c110e8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,10 @@ import com.github.jk1.license.filter.LicenseBundleNormalizer +import groovy.json.JsonSlurper import tech.pegasys.internal.license.reporter.GroupedLicenseHtmlRenderer import tech.pegasys.teku.depcheck.DepCheckPlugin import java.text.SimpleDateFormat -import groovy.json.JsonSlurper - import static tech.pegasys.teku.repackage.Repackage.repackage buildscript { @@ -325,7 +324,7 @@ allprojects { } def nightly = System.getenv("NIGHTLY") != null -def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.10" +def refTestVersion = nightly ? "nightly" : "v1.5.0-beta.1" def blsRefTestVersion = 'v0.1.2' def slashingProtectionInterchangeRefTestVersion = 'v5.3.0' def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download' diff --git a/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java b/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java index ee64be4a9bd..ff1e8e98943 100644 --- a/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java +++ b/data/dataexchange/src/main/java/tech/pegasys/teku/data/eraFileFormat/EraFile.java @@ -214,7 +214,7 @@ private void verifyBlocksWithReferenceState( block.getParentRoot().equals(previousArchiveLastBlock.getRoot()), "First block in archive does not match last block of previous archive."); } - // TODO should verify signature + // when fully implemented, we would check signature also ++populatedSlots; } System.out.println( diff --git a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java index 39c53c17b92..8047e00cfda 100644 --- a/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java +++ b/eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java @@ -53,6 +53,7 @@ public abstract class Eth2ReferenceTestCase { .put("light_client/single_merkle_proof", TestExecutor.IGNORE_TESTS) .put("light_client/sync", TestExecutor.IGNORE_TESTS) .put("light_client/update_ranking", TestExecutor.IGNORE_TESTS) + .put("light_client/data_collection", TestExecutor.IGNORE_TESTS) .build(); private static final ImmutableMap PHASE_0_TEST_TYPES = diff --git a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java index 353f9bc3ea1..e112e5f5e2a 100644 --- a/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java +++ b/ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/state/beaconstate/common/AbstractBeaconStateSchemaTest.java @@ -97,7 +97,6 @@ public void changeSpecConfigTest() { @Test void roundTripViaSsz() { - // TODO - generate random version-specific state BeaconState beaconState = randomState(); Bytes bytes = beaconState.sszSerialize(); BeaconState state = schema.sszDeserialize(bytes); diff --git a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java index ee86cd6406d..c62402e2966 100644 --- a/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java +++ b/infrastructure/logging/src/main/java/tech/pegasys/teku/infrastructure/logging/StatusLogger.java @@ -278,6 +278,10 @@ public void finishInitializingChainData() { log.info("Storage initialization complete"); } + public void emptyChainData() { + log.info("Empty storage. Initialization complete."); + } + public void recordedFinalizedBlocks(final int numberRecorded, final int totalToRecord) { log.info("Recorded {} of {} finalized blocks", numberRecorded, totalToRecord); } diff --git a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java index 3b672c40c10..9b04c382f48 100644 --- a/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java +++ b/infrastructure/ssz/src/main/java/tech/pegasys/teku/infrastructure/ssz/schema/SszCollectionSchema.java @@ -49,7 +49,7 @@ default SszCollectionT createFromElements(final List elem } default TreeNode createTreeFromElements(final List elements) { - // TODO: probably suboptimal method implementation: + // https://github.com/Consensys/teku/issues/9035 // This is a generic implementation which works for both Vector and List but it potentially // could do better if construct the tree directly in List/Vector subclasses checkArgument(elements.size() <= getMaxLength(), "Too many elements for this collection type"); diff --git a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java index e35bb4370bc..edf1e1905ef 100644 --- a/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java +++ b/networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/gossip/config/ScoringConfig.java @@ -54,8 +54,6 @@ class ScoringConfig { private ScoringConfig(final Spec spec, final int d) { this.spec = spec; - // TODO(#3356) Use spec provider through-out rather than relying only on genesis constants and - // genesis spec this.genesisConfig = spec.getGenesisSpecConfig(); this.genesisSpec = spec.getGenesisSpec(); this.d = d; diff --git a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java index 5b809f77d14..eb9d1c0df68 100644 --- a/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java +++ b/networking/p2p/src/main/java/tech/pegasys/teku/networking/p2p/libp2p/LibP2PNetworkBuilder.java @@ -215,9 +215,9 @@ protected Host createHost(final PrivKey privKey, final List advertise b.getTransports().add(TcpTransport::new); b.getSecureChannels().add(NoiseXXSecureChannel::new); - // yamux MUST take precedence during negotiation + // Yamux must take precedence during negotiation if (config.isYamuxEnabled()) { - // TODO: https://github.com/Consensys/teku/issues/7532 + // https://github.com/Consensys/teku/issues/7532 final int maxBufferedConnectionWrites = 150 * 1024 * 1024; b.getMuxers().add(StreamMuxerProtocol.getYamux(maxBufferedConnectionWrites)); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java index 5ffa9a7e22e..d011f4fdfa2 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/StorageBackedRecentChainData.java @@ -166,7 +166,7 @@ private SafeFuture processStoreFuture( return storeFuture.thenApply( maybeData -> { if (maybeData.isEmpty()) { - STATUS_LOG.finishInitializingChainData(); + STATUS_LOG.emptyChainData(); return this; } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java index 4f8751cc44b..162db230511 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java @@ -215,6 +215,8 @@ default Stream streamBlobSidecarKeys(final UInt64 Map getColumnCounts(final Optional maybeColumnFilter); + Map> getVariables(); + long getBlobSidecarColumnCount(); long getNonCanonicalBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java index 83bda83c9a8..6e9f46d0c1e 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/CombinedKvStoreDao.java @@ -432,6 +432,37 @@ public Map getColumnCounts(final Optional maybeColumnFilte return columnCounts; } + @Override + public Map> getVariables() { + Map> variables = new LinkedHashMap<>(); + variables.put("GENESIS_TIME", getGenesisTime().map(UInt64::toString)); + variables.put("JUSTIFIED_CHECKPOINT", getJustifiedCheckpoint().map(Checkpoint::toString)); + variables.put( + "BEST_JUSTIFIED_CHECKPOINT", getBestJustifiedCheckpoint().map(Checkpoint::toString)); + variables.put("FINALIZED_CHECKPOINT", getFinalizedCheckpoint().map(Checkpoint::toString)); + variables.put( + "WEAK_SUBJECTIVITY_CHECKPOINT", getWeakSubjectivityCheckpoint().map(Checkpoint::toString)); + variables.put("ANCHOR_CHECKPOINT", getAnchor().map(Checkpoint::toString)); + variables.put( + "FINALIZED_DEPOSIT_SNAPSHOT", + getFinalizedDepositSnapshot().map(DepositTreeSnapshot::toString)); + try { + variables.put( + "LATEST_FINALIZED_STATE", + getLatestFinalizedState() + .map( + state -> + "BeaconState{slot=" + + state.getSlot() + + ", root=" + + state.hashTreeRoot().toHexString() + + "}")); + } catch (final Exception e) { + variables.put("FINALIZED_STATE", Optional.of(e.toString())); + } + return variables; + } + @Override public long getBlobSidecarColumnCount() { final KvStoreColumn column = diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java index 2dba5e05e70..438c0b16e51 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDao.java @@ -154,6 +154,8 @@ List getNonCanonicalBlobSidecarKeys( Map getColumnCounts(final Optional maybeColumnFilter); + Map> getVariables(); + long getBlobSidecarColumnCount(); long getNonCanonicalBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java index b9d918f98dc..da50d25e596 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/dataaccess/KvStoreCombinedDaoAdapter.java @@ -228,6 +228,11 @@ public Map getColumnCounts(final Optional maybeColumnFilte return result; } + @Override + public Map> getVariables() { + return Map.of(); + } + @Override public long getBlobSidecarColumnCount() { return finalizedDao.getBlobSidecarColumnCount(); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java new file mode 100644 index 00000000000..a41bd0cf624 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomDBIterator.java @@ -0,0 +1,27 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import org.iq80.leveldb.DBIterator; + +/** + * This interface extends the DBIterator interface to provide additional methods for peeking at the + * next key which are used to avoid unnecessary value allocations + */ +public interface CustomDBIterator extends DBIterator { + + byte[] peekNextKey(); + + byte[] nextKey(); +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java new file mode 100644 index 00000000000..13ef74b9073 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDB.java @@ -0,0 +1,63 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import org.fusesource.leveldbjni.internal.JniDB; +import org.fusesource.leveldbjni.internal.NativeCache; +import org.fusesource.leveldbjni.internal.NativeComparator; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.fusesource.leveldbjni.internal.NativeLogger; +import org.fusesource.leveldbjni.internal.NativeReadOptions; +import org.iq80.leveldb.DBException; +import org.iq80.leveldb.DBIterator; +import org.iq80.leveldb.ReadOptions; + +/** This class extends the JniDB class to provide a custom DBIterator. */ +public class CustomJniDB extends JniDB { + private final NativeDB db; + + public CustomJniDB( + final NativeDB db, + final NativeCache cache, + final NativeComparator comparator, + final NativeLogger logger) { + super(db, cache, comparator, logger); + this.db = db; + } + + @Override + public DBIterator iterator(final ReadOptions options) { + if (this.db == null) { + throw new DBException("Closed"); + } else { + return new CustomJniDBIterator(this.db.iterator(this.convert(options))); + } + } + + // this method is private in the super class, so has been copied here + private NativeReadOptions convert(final ReadOptions options) { + if (options == null) { + return null; + } else { + NativeReadOptions rc = new NativeReadOptions(); + rc.fillCache(options.fillCache()); + rc.verifyChecksums(options.verifyChecksums()); + if (options.snapshot() != null) { + throw new UnsupportedOperationException("Snapshots are not supported"); + } + + return rc; + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java new file mode 100644 index 00000000000..81c879cab79 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBFactory.java @@ -0,0 +1,156 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import org.fusesource.leveldbjni.JniDBFactory; +import org.fusesource.leveldbjni.internal.NativeCache; +import org.fusesource.leveldbjni.internal.NativeComparator; +import org.fusesource.leveldbjni.internal.NativeCompressionType; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.fusesource.leveldbjni.internal.NativeLogger; +import org.fusesource.leveldbjni.internal.NativeOptions; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBComparator; +import org.iq80.leveldb.Logger; +import org.iq80.leveldb.Options; + +/** + * This class extends the JniDBFactory class to provide a custom DBIterator. Only {@link + * CustomJniDBFactory#open} has been changed from the original class + */ +@SuppressWarnings("EmptyCatch") +public class CustomJniDBFactory extends JniDBFactory { + public static final CustomJniDBFactory FACTORY = new CustomJniDBFactory(); + public static final String VERSION; + + @Override + public DB open(final File path, final Options options) throws IOException { + final OptionsResourceHolder holder = new OptionsResourceHolder(); + + NativeDB db = null; + try { + holder.init(options); + db = NativeDB.open(holder.options, path); + } finally { + if (db == null) { + holder.close(); + } + } + + // this is the only line that has been functionally changed from the original class + return new CustomJniDB(db, holder.cache, holder.comparator, holder.logger); + } + + static { + NativeDB.LIBRARY.load(); + String v = "unknown"; + + try (final InputStream is = JniDBFactory.class.getResourceAsStream("version.txt")) { + if (is != null) { + try (final BufferedReader reader = + new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + v = reader.readLine(); + } + } + } catch (final Throwable ignored) { + } + + VERSION = v; + } + + private static class OptionsResourceHolder { + NativeCache cache; + NativeComparator comparator; + NativeLogger logger; + NativeOptions options; + + private OptionsResourceHolder() { + this.cache = null; + this.comparator = null; + this.logger = null; + } + + private void init(final Options value) { + this.options = new NativeOptions(); + this.options.blockRestartInterval(value.blockRestartInterval()); + this.options.blockSize((long) value.blockSize()); + this.options.createIfMissing(value.createIfMissing()); + this.options.errorIfExists(value.errorIfExists()); + this.options.maxOpenFiles(value.maxOpenFiles()); + this.options.paranoidChecks(value.paranoidChecks()); + this.options.writeBufferSize((long) value.writeBufferSize()); + switch (value.compressionType()) { + case NONE: + this.options.compression(NativeCompressionType.kNoCompression); + break; + case SNAPPY: + this.options.compression(NativeCompressionType.kSnappyCompression); + } + + if (value.cacheSize() > 0L) { + this.cache = new NativeCache(value.cacheSize()); + this.options.cache(this.cache); + } + + final DBComparator userComparator = value.comparator(); + if (userComparator != null) { + this.comparator = + new NativeComparator() { + @Override + public int compare(final byte[] key1, final byte[] key2) { + return userComparator.compare(key1, key2); + } + + @Override + public String name() { + return userComparator.name(); + } + }; + this.options.comparator(this.comparator); + } + + final Logger userLogger = value.logger(); + if (userLogger != null) { + this.logger = + new NativeLogger() { + @Override + public void log(final String message) { + userLogger.log(message); + } + }; + this.options.infoLog(this.logger); + } + } + + private void close() { + if (this.cache != null) { + this.cache.delete(); + } + + if (this.comparator != null) { + this.comparator.delete(); + } + + if (this.logger != null) { + this.logger.delete(); + } + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java new file mode 100644 index 00000000000..95c53310a7a --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/CustomJniDBIterator.java @@ -0,0 +1,181 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.server.leveldb; + +import java.util.AbstractMap; +import java.util.Map; +import java.util.NoSuchElementException; +import org.fusesource.leveldbjni.internal.NativeDB; +import org.fusesource.leveldbjni.internal.NativeIterator; + +/** + * This is a copy of + * which also implements the methods from {@link CustomDBIterator} + */ +public class CustomJniDBIterator implements CustomDBIterator { + private final NativeIterator iterator; + + CustomJniDBIterator(final NativeIterator iterator) { + this.iterator = iterator; + } + + @Override + public void close() { + this.iterator.delete(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public void seek(final byte[] key) { + try { + this.iterator.seek(key); + } catch (final NativeDB.DBException e) { + if (e.isNotFound()) { + throw new NoSuchElementException(); + } else { + throw new RuntimeException(e); + } + } + } + + @Override + public void seekToFirst() { + this.iterator.seekToFirst(); + } + + @Override + public void seekToLast() { + this.iterator.seekToLast(); + } + + @Override + public Map.Entry peekNext() { + if (!this.iterator.isValid()) { + throw new NoSuchElementException(); + } else { + try { + return new AbstractMap.SimpleImmutableEntry<>(this.iterator.key(), this.iterator.value()); + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public boolean hasNext() { + return this.iterator.isValid(); + } + + @Override + public Map.Entry next() { + final Map.Entry entry = this.peekNext(); + + try { + this.iterator.next(); + return entry; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean hasPrev() { + if (!this.iterator.isValid()) { + return false; + } else { + try { + this.iterator.prev(); + + final boolean ret; + try { + ret = this.iterator.isValid(); + } finally { + if (this.iterator.isValid()) { + this.iterator.next(); + } else { + this.iterator.seekToFirst(); + } + } + + return ret; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public Map.Entry peekPrev() { + try { + this.iterator.prev(); + + final Map.Entry entry; + try { + entry = this.peekNext(); + } finally { + if (this.iterator.isValid()) { + this.iterator.next(); + } else { + this.iterator.seekToFirst(); + } + } + + return entry; + } catch (final NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + + @Override + public Map.Entry prev() { + final Map.Entry rc = this.peekPrev(); + + try { + this.iterator.prev(); + return rc; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + + @Override + public byte[] peekNextKey() { + if (!this.iterator.isValid()) { + throw new NoSuchElementException(); + } else { + try { + return this.iterator.key(); + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public byte[] nextKey() { + final byte[] key = this.peekNextKey(); + + try { + this.iterator.next(); + return key; + } catch (NativeDB.DBException e) { + throw new RuntimeException(e); + } + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java index ef92af2c47b..634bfac9bf2 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstance.java @@ -296,7 +296,7 @@ private Stream streamKeys( private Stream> streamRaw( final KvStoreColumn column, final byte[] fromBytes, final byte[] toBytes) { assertOpen(); - final DBIterator iterator = createIterator(); + final CustomDBIterator iterator = createIterator(); iterator.seek(fromBytes); return new LevelDbIterator<>(this, iterator, column, toBytes) .toStream() @@ -307,7 +307,7 @@ private Stream> streamRaw( private Stream streamKeysRaw( final KvStoreColumn column, final byte[] fromBytes, final byte[] toBytes) { assertOpen(); - final DBIterator iterator = createIterator(); + final CustomDBIterator iterator = createIterator(); iterator.seek(fromBytes); return new LevelDbKeyIterator<>(this, iterator, column, toBytes) .toStream() @@ -371,8 +371,9 @@ synchronized void onTransactionClosed(final LevelDbTransaction transaction) { } } - private DBIterator createIterator() { - final DBIterator iterator = db.iterator(new ReadOptions().fillCache(false)); + private CustomDBIterator createIterator() { + final CustomDBIterator iterator = + (CustomDBIterator) db.iterator(new ReadOptions().fillCache(false)); openIterators.add(iterator); openedIteratorsCounter.inc(); return iterator; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java index f3142174644..da1082c68f6 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbInstanceFactory.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.util.Collection; -import org.fusesource.leveldbjni.JniDBFactory; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.plugin.services.metrics.MetricCategory; import org.iq80.leveldb.DB; @@ -45,7 +44,8 @@ public static KvStoreAccessor create( .writeBufferSize(configuration.getLeveldbWriteBufferSize()); try { - final DB db = JniDBFactory.factory.open(configuration.getDatabaseDir().toFile(), options); + final DB db = + CustomJniDBFactory.FACTORY.open(configuration.getDatabaseDir().toFile(), options); return new LevelDbInstance(db, metricsSystem, metricCategory); } catch (final IOException e) { throw DatabaseStorageException.unrecoverable("Failed to open database", e); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java index d40e0e2f938..41612fca0e4 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbIterator.java @@ -22,20 +22,19 @@ import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.iq80.leveldb.DBIterator; import tech.pegasys.teku.storage.server.kvstore.ColumnEntry; import tech.pegasys.teku.storage.server.kvstore.schema.KvStoreColumn; public class LevelDbIterator implements Iterator> { private final LevelDbInstance dbInstance; - private final DBIterator iterator; + private final CustomDBIterator iterator; private final KvStoreColumn column; private final byte[] lastKey; public LevelDbIterator( final LevelDbInstance dbInstance, - final DBIterator iterator, + final CustomDBIterator iterator, final KvStoreColumn column, final byte[] lastKey) { this.dbInstance = dbInstance; @@ -53,7 +52,7 @@ public boolean hasNext() { } private boolean isValidKey() { - final byte[] nextKey = iterator.peekNext().getKey(); + final byte[] nextKey = iterator.peekNextKey(); return isFromColumn(column, nextKey) && Arrays.compareUnsigned(nextKey, lastKey) <= 0; } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java index 3a4f6e2c6e0..7cb93e9bce9 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/leveldb/LevelDbKeyIterator.java @@ -22,19 +22,18 @@ import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.iq80.leveldb.DBIterator; import tech.pegasys.teku.storage.server.kvstore.schema.KvStoreColumn; public class LevelDbKeyIterator implements Iterator { private final LevelDbInstance dbInstance; - private final DBIterator iterator; + private final CustomDBIterator iterator; private final KvStoreColumn column; private final byte[] lastKey; public LevelDbKeyIterator( final LevelDbInstance dbInstance, - final DBIterator iterator, + final CustomDBIterator iterator, final KvStoreColumn column, final byte[] lastKey) { this.dbInstance = dbInstance; @@ -52,7 +51,7 @@ public boolean hasNext() { } private boolean isValidKey() { - final byte[] nextKey = iterator.peekNext().getKey(); + final byte[] nextKey = iterator.peekNextKey(); return isFromColumn(column, nextKey) && Arrays.compareUnsigned(nextKey, lastKey) <= 0; } @@ -60,7 +59,7 @@ private boolean isValidKey() { public byte[] next() { synchronized (dbInstance) { dbInstance.assertOpen(); - return removeKeyPrefix(column, iterator.next().getKey()); + return removeKeyPrefix(column, iterator.nextKey()); } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java index af522b0c619..dc1b3bd4969 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java @@ -256,6 +256,11 @@ public Map getColumnCounts(final Optional maybeColumnFilte return new HashMap<>(); } + @Override + public Map> getVariables() { + return new HashMap<>(); + } + @Override public long getBlobSidecarColumnCount() { return 0L; diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java index 8ed14e3ab5d..e5f5110e75c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstance.java @@ -201,7 +201,7 @@ public , V> Stream streamKeys( @MustBeClosed public synchronized KvStoreTransaction startTransaction() { assertOpen(); - RocksDbTransaction tx = + final RocksDbTransaction tx = new RocksDbTransaction(db, defaultHandle, columnHandles, openTransactions::remove); openTransactions.add(tx); return tx; @@ -249,6 +249,7 @@ private Stream> createStreamRaw( final KvStoreColumn column, final Consumer setupIterator, final Predicate continueTest) { + assertOpen(); final ColumnFamilyHandle handle = columnHandles.get(column); final RocksIterator rocksDbIterator = db.newIterator(handle); setupIterator.accept(rocksDbIterator); @@ -261,6 +262,7 @@ private Stream createStreamKeyRaw( final KvStoreColumn column, final Consumer setupIterator, final Predicate continueTest) { + assertOpen(); final ColumnFamilyHandle handle = columnHandles.get(column); final RocksIterator rocksDbIterator = db.newIterator(handle); setupIterator.accept(rocksDbIterator); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java index d45e48ea92d..ffd32385e19 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbKeyIterator.java @@ -98,7 +98,7 @@ public Stream toStream() { } private void assertOpen() { - if (this.isDatabaseClosed.get()) { + if (isDatabaseClosed.get()) { throw new ShuttingDownException(); } if (closed.get()) { diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java index d9801c8a1ff..1a34ffcad87 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbStats.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -150,7 +151,7 @@ public class RocksDbStats implements AutoCloseable { HistogramType.READ_NUM_MERGE_OPERANDS, }; - private boolean closed = false; + private final AtomicBoolean closed = new AtomicBoolean(false); private final Statistics stats; private final MetricsSystem metricsSystem; private final MetricCategory category; @@ -230,16 +231,14 @@ private long getLongProperty(final RocksDB database, final String name) { } @Override - public synchronized void close() { - if (closed) { - return; + public void close() { + if (closed.compareAndSet(false, true)) { + stats.close(); } - closed = true; - stats.close(); } - private synchronized T ifOpen(final Supplier supplier, final T defaultValue) { - if (closed) { + private T ifOpen(final Supplier supplier, final T defaultValue) { + if (closed.get()) { return defaultValue; } return supplier.get(); diff --git a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java index aa32be3ba42..04a048a2cdd 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/debug/DebugDbCommand.java @@ -305,6 +305,40 @@ public int getColumnCounts( return 0; } + @Command( + name = "get-variables", + description = "Check variable values", + mixinStandardHelpOptions = true, + showDefaultValues = true, + abbreviateSynopsis = true, + versionProvider = PicoCliVersionProvider.class, + synopsisHeading = "%n", + descriptionHeading = "%nDescription:%n%n", + optionListHeading = "%nOptions:%n", + footerHeading = "%n", + footer = "Teku is licensed under the Apache License 2.0") + public int getVariables( + @Mixin final BeaconNodeDataOptions beaconNodeDataOptions, + @Mixin final Eth2NetworkOptions eth2NetworkOptions, + @Option( + names = {"--filter"}, + description = "Only get variables that match a given filter.") + final String filter) + throws Exception { + try (final Database database = createDatabase(beaconNodeDataOptions, eth2NetworkOptions)) { + final Map> variables = database.getVariables(); + variables + .keySet() + .forEach( + k -> { + if (filter == null || k.contains(filter)) { + System.out.printf("%-30s: %s\n", k, variables.get(k).orElse("EMPTY")); + } + }); + } + return 0; + } + @Command( name = "validate-block-history", description = "Validate the chain of finalized blocks via parent references", diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java index 08abd012b6e..85d76ec62ab 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/loader/SlashingProtectedValidatorSource.java @@ -90,8 +90,6 @@ public boolean isReadOnly() { @Override public Signer createSigner() { - // TODO: Consider caching these to guarantee we can't possible use different - // `SlashingProtectedSigner` instances with the same key return new SlashingProtectedSigner( getPublicKey(), slashingProtector, delegate.createSigner()); }