diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index acf8daafb79..1faf7e45a4a 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -41,6 +41,7 @@ test.consensus > xrpl.basics test.consensus > xrpld.app test.consensus > xrpld.consensus test.consensus > xrpld.ledger +test.consensus > xrpl.json test.core > test.jtx test.core > test.toplevel test.core > test.unit_test diff --git a/docs/consensus.md b/docs/consensus.md index 1b0063663a2..4ee5aa70dca 100644 --- a/docs/consensus.md +++ b/docs/consensus.md @@ -558,7 +558,7 @@ struct ConsensusResult ConsensusTimer roundTime; // Indicates state in which consensus ended. Once in the accept phase - // will be either Yes or MovedOn + // will be either Yes or MovedOn or Expired ConsensusState state = ConsensusState::No; }; diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp index 88280994c10..aa0eb94cd70 100644 --- a/src/test/consensus/Consensus_test.cpp +++ b/src/test/consensus/Consensus_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace ripple { @@ -40,6 +41,7 @@ class Consensus_test : public beast::unit_test::suite testShouldCloseLedger() { using namespace std::chrono_literals; + testcase("should close ledger"); // Use default parameters ConsensusParms const p{}; @@ -78,46 +80,101 @@ class Consensus_test : public beast::unit_test::suite testCheckConsensus() { using namespace std::chrono_literals; + testcase("check consensus"); // Use default parameterss ConsensusParms const p{}; + /////////////// + // Disputes still in doubt + // // Not enough time has elapsed BEAST_EXPECT( ConsensusState::No == - checkConsensus(10, 2, 2, 0, 3s, 2s, p, true, journal_)); + checkConsensus(10, 2, 2, 0, 3s, 2s, false, p, true, journal_)); // If not enough peers have propsed, ensure // more time for proposals BEAST_EXPECT( ConsensusState::No == - checkConsensus(10, 2, 2, 0, 3s, 4s, p, true, journal_)); + checkConsensus(10, 2, 2, 0, 3s, 4s, false, p, true, journal_)); // Enough time has elapsed and we all agree BEAST_EXPECT( ConsensusState::Yes == - checkConsensus(10, 2, 2, 0, 3s, 10s, p, true, journal_)); + checkConsensus(10, 2, 2, 0, 3s, 10s, false, p, true, journal_)); // Enough time has elapsed and we don't yet agree BEAST_EXPECT( ConsensusState::No == - checkConsensus(10, 2, 1, 0, 3s, 10s, p, true, journal_)); + checkConsensus(10, 2, 1, 0, 3s, 10s, false, p, true, journal_)); // Our peers have moved on // Enough time has elapsed and we all agree BEAST_EXPECT( ConsensusState::MovedOn == - checkConsensus(10, 2, 1, 8, 3s, 10s, p, true, journal_)); + checkConsensus(10, 2, 1, 8, 3s, 10s, false, p, true, journal_)); + + // If no peers, don't agree until time has passed. + BEAST_EXPECT( + ConsensusState::No == + checkConsensus(0, 0, 0, 0, 3s, 10s, false, p, true, journal_)); + + // Agree if no peers and enough time has passed. + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(0, 0, 0, 0, 3s, 16s, false, p, true, journal_)); + + // Expire if too much time has passed without agreement + BEAST_EXPECT( + ConsensusState::Expired == + checkConsensus(10, 8, 1, 0, 1s, 19s, false, p, true, journal_)); + /////////////// + // Stable state + // + // Not enough time has elapsed + BEAST_EXPECT( + ConsensusState::No == + checkConsensus(10, 2, 2, 0, 3s, 2s, true, p, true, journal_)); + + // If not enough peers have propsed, ensure + // more time for proposals + BEAST_EXPECT( + ConsensusState::No == + checkConsensus(10, 2, 2, 0, 3s, 4s, true, p, true, journal_)); + + // Enough time has elapsed and we all agree + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 2, 2, 0, 3s, 10s, true, p, true, journal_)); + + // Enough time has elapsed and we don't yet agree, but there's nothing + // left to dispute + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 2, 1, 0, 3s, 10s, true, p, true, journal_)); + + // Our peers have moved on + // Enough time has elapsed and we all agree, nothing left to dispute + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 2, 1, 8, 3s, 10s, true, p, true, journal_)); // If no peers, don't agree until time has passed. BEAST_EXPECT( ConsensusState::No == - checkConsensus(0, 0, 0, 0, 3s, 10s, p, true, journal_)); + checkConsensus(0, 0, 0, 0, 3s, 10s, true, p, true, journal_)); // Agree if no peers and enough time has passed. BEAST_EXPECT( ConsensusState::Yes == - checkConsensus(0, 0, 0, 0, 3s, 16s, p, true, journal_)); + checkConsensus(0, 0, 0, 0, 3s, 16s, true, p, true, journal_)); + + // We are done if there's nothing left to dispute, no matter how much + // time has passed + BEAST_EXPECT( + ConsensusState::Yes == + checkConsensus(10, 8, 1, 0, 1s, 19s, true, p, true, journal_)); } void @@ -125,6 +182,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace std::chrono_literals; using namespace csf; + testcase("standalone"); Sim s; PeerGroup peers = s.createGroup(1); @@ -149,6 +207,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("peers agree"); ConsensusParms const parms{}; Sim sim; @@ -186,6 +245,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("slow peers"); // Several tests of a complete trust graph with a subset of peers // that have significantly longer network delays to the rest of the @@ -351,6 +411,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("close time disagree"); // This is a very specialized test to get ledgers to disagree on // the close time. It unfortunately assumes knowledge about current @@ -417,6 +478,8 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("wrong LCL"); + // Specialized test to exercise a temporary fork in which some peers // are working on an incorrect prior ledger. @@ -589,6 +652,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("consensus close time rounding"); // This is a specialized test engineered to yield ledgers with different // close times even though the peers believe they had close time @@ -604,9 +668,6 @@ class Consensus_test : public beast::unit_test::suite PeerGroup fast = sim.createGroup(4); PeerGroup network = fast + slow; - for (Peer* peer : network) - peer->consensusParms = parms; - // Connected trust graph network.trust(network); @@ -692,6 +753,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("fork"); std::uint32_t numPeers = 10; // Vary overlap between two UNLs @@ -748,6 +810,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("hub network"); // Simulate a set of 5 validators that aren't directly connected but // rely on a single hub node for communication @@ -835,6 +898,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("preferred by branch"); // Simulate network splits that are prevented from forking when using // preferred ledger by trie. This is a contrived example that involves @@ -967,6 +1031,7 @@ class Consensus_test : public beast::unit_test::suite { using namespace csf; using namespace std::chrono; + testcase("pause for laggards"); // Test that validators that jump ahead of the network slow // down. @@ -1052,6 +1117,98 @@ class Consensus_test : public beast::unit_test::suite BEAST_EXPECT(sim.synchronized()); } + void + testDisputes() + { + // WIP: Try to create conditions where messaging is unreliable and all + // peers have different initial proposals + using namespace csf; + using namespace std::chrono; + testcase("disputes"); + + std::uint32_t const numPeers = 35; + + ConsensusParms const parms{}; + Sim sim; + + std::vector peerGroups; + peerGroups.reserve(numPeers); + PeerGroup network; + for (int i = 0; i < numPeers; ++i) + { + peerGroups.emplace_back(sim.createGroup(1)); + network = network + peerGroups.back(); + } + + // Fully connected trust graph + network.trust(network); + + for (int i = 0; i < peerGroups.size(); ++i) + { + auto const delay = i * 0.2 * parms.ledgerGRANULARITY; + auto& group = peerGroups[i]; + auto& nextGroup = peerGroups[(i + 1) % numPeers]; + group.connect(nextGroup, round(delay)); + auto& oppositeGroup = peerGroups[(i + numPeers / 2) % numPeers]; + group.connect(oppositeGroup, round(delay)); + } + + // Initial round to set prior state + sim.run(1); + for (Peer* peer : network) + { + // Every transaction will be seen by every node but two. + // To accomplish that, every peer will add the ids of every peer + // except itself, and the one following. + auto const myId = peer->id; + auto const nextId = (peer->id + PeerID(1)) % PeerID(numPeers); + for (Peer* to : sim.trustGraph.trustedPeers(peer)) + { + if (to->id == myId || to->id == nextId) + continue; + peer->openTxs.insert(Tx{static_cast(to->id)}); + } + } + sim.run(1); + + // Peers are out of sync + if (BEAST_EXPECT(!sim.synchronized())) + { + BEAST_EXPECT(sim.branches() == 1); + for (auto const& grp : peerGroups) + { + Peer const* peer = grp[0]; + BEAST_EXPECT( + peer->fullyValidatedLedger.seq() == Ledger::Seq{1}); + + auto const& lcl = peer->lastClosedLedger; + BEAST_EXPECT(lcl.id() == peer->prevLedgerID()); + log << "Peer " << peer->id << ", lcl seq: " << lcl.seq() + << ", prevProposers: " << peer->prevProposers + << ", txs in lcl: " << lcl.txs().size() << ", validations: " + << peer->validations.sizeOfByLedgerCache() << std::endl; + for (auto const& [id, positions] : peer->peerPositions) + { + log << "\tLedger ID: " << id + << ", #positions: " << positions.size() << std::endl; + } + /* + log << "\t" << to_string(peer->consensus.getJson(true)) + << std::endl + << std::endl; + */ + /* + BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}, to_string); + // All peers proposed + BEAST_EXPECT(peer->prevProposers == network.size() - 1); + // All transactions were accepted + for (std::uint32_t i = 0; i < network.size(); ++i) + BEAST_EXPECT(lcl.txs().find(Tx{i}) != lcl.txs().end()); + */ + } + } + } + void run() override { @@ -1068,6 +1225,7 @@ class Consensus_test : public beast::unit_test::suite testHubNetwork(); testPreferredByBranch(); testPauseForLaggards(); + testDisputes(); } }; diff --git a/src/xrpld/app/consensus/RCLValidations.cpp b/src/xrpld/app/consensus/RCLValidations.cpp index c4d9389e896..ac8bd7a0794 100644 --- a/src/xrpld/app/consensus/RCLValidations.cpp +++ b/src/xrpld/app/consensus/RCLValidations.cpp @@ -189,8 +189,9 @@ handleNewValidation( auto& validations = app.getValidations(); // masterKey is seated only if validator is trusted or listed - auto const outcome = - validations.add(calcNodeID(masterKey.value_or(signingKey)), val); + auto const nodeKey = masterKey.value_or(signingKey); + // assert(nodeKey != app.getValidationPublicKey()); + auto const outcome = validations.add(calcNodeID(nodeKey), val); if (outcome == ValStatus::current) { diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 996a1fdf748..cdd2dc9d22f 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -1887,6 +1887,15 @@ NetworkOPsImp::beginConsensus(uint256 const& networkClosed) bool NetworkOPsImp::processTrustedProposal(RCLCxPeerPos peerPos) { + if (auto const localPk = app_.getValidationPublicKey()) + { + auto const localID = calcNodeID(*localPk); + auto const& peerID = peerPos.proposal().nodeID(); + assert(localID != peerID); + if (localID == peerID) + return false; + } + return mConsensus.peerProposal(app_.timeKeeper().closeTime(), peerPos); } diff --git a/src/xrpld/consensus/Consensus.cpp b/src/xrpld/consensus/Consensus.cpp index 01451c6a255..3c46dd7c1e8 100644 --- a/src/xrpld/consensus/Consensus.cpp +++ b/src/xrpld/consensus/Consensus.cpp @@ -88,7 +88,8 @@ checkConsensusReached( std::size_t total, bool count_self, std::size_t minConsensusPct, - bool reachedMax) + bool reachedMax, + bool stableState) { // If we are alone for too long, we have consensus. // Delaying consensus like this avoids a circumstance where a peer @@ -114,7 +115,7 @@ checkConsensusReached( std::size_t currentPercentage = (agreeing * 100) / total; - return currentPercentage >= minConsensusPct; + return currentPercentage >= minConsensusPct || stableState; } ConsensusState @@ -125,6 +126,7 @@ checkConsensus( std::size_t currentFinished, std::chrono::milliseconds previousAgreeTime, std::chrono::milliseconds currentAgreeTime, + bool stableState, ConsensusParms const& parms, bool proposing, beast::Journal j) @@ -137,7 +139,7 @@ checkConsensus( << " minimum duration to reach consensus: " << parms.ledgerMIN_CONSENSUS.count() << "ms" << " max consensus time " - << parms.ledgerMAX_CONSENSUS.count() << "s" + << parms.ledgerMAX_CONSENSUS.count() << "ms" << " minimum consensus percentage: " << parms.minCONSENSUS_PCT; @@ -162,9 +164,11 @@ checkConsensus( currentProposers, proposing, parms.minCONSENSUS_PCT, - currentAgreeTime > parms.ledgerMAX_CONSENSUS)) + currentAgreeTime > parms.ledgerMAX_CONSENSUS, + stableState)) { - JLOG(j.debug()) << "normal consensus"; + JLOG(j.debug()) << "normal consensus with " << (stableState ? "" : "un") + << "stable state"; return ConsensusState::Yes; } @@ -175,12 +179,24 @@ checkConsensus( currentProposers, false, parms.minCONSENSUS_PCT, - currentAgreeTime > parms.ledgerMAX_CONSENSUS)) + currentAgreeTime > parms.ledgerMAX_CONSENSUS, + false)) { JLOG(j.warn()) << "We see no consensus, but 80% of nodes have moved on"; return ConsensusState::MovedOn; } + std::chrono::milliseconds const maxAgreeTime = + previousAgreeTime * parms.ledgerABANDON_CONSENSUS_FACTOR; + if (currentAgreeTime > std::clamp( + maxAgreeTime, + parms.ledgerMAX_CONSENSUS, + parms.ledgerABANDON_CONSENSUS)) + { + JLOG(j.warn()) << "consensus taken too long"; + return ConsensusState::Expired; + } + // no consensus yet JLOG(j.trace()) << "no consensus"; return ConsensusState::No; diff --git a/src/xrpld/consensus/Consensus.h b/src/xrpld/consensus/Consensus.h index daad520c77f..b0bf22816da 100644 --- a/src/xrpld/consensus/Consensus.h +++ b/src/xrpld/consensus/Consensus.h @@ -79,6 +79,10 @@ shouldCloseLedger( last ledger @param currentAgreeTime how long, in milliseconds, we've been trying to agree + @param stableState the network appears to be in a stable state, where + neither we nor our peers have changed their vote on any disputes in a + while. This is undesirable, and will cause us to end consensus + without 80% agreement. @param parms Consensus constant parameters @param proposing whether we should count ourselves @param j journal for logging @@ -91,6 +95,7 @@ checkConsensus( std::size_t currentFinished, std::chrono::milliseconds previousAgreeTime, std::chrono::milliseconds currentAgreeTime, + bool stableState, ConsensusParms const& parms, bool proposing, beast::Journal j); @@ -559,6 +564,9 @@ class Consensus NetClock::duration closeResolution_ = ledgerDefaultTimeResolution; + ConsensusParms::AvalancheState closeTimeAvalancheState_ = + ConsensusParms::init; + // Time it took for the last consensus round to converge std::chrono::milliseconds prevRoundTime_; @@ -676,6 +684,7 @@ Consensus::startRoundInternal( previousLedger_ = prevLedger; result_.reset(); convergePercent_ = 0; + closeTimeAvalancheState_ = ConsensusParms::init; haveCloseTimeConsensus_ = false; openTime_.reset(clock_.now()); currPeerPositions_.clear(); @@ -832,6 +841,8 @@ Consensus::timerEntry(NetClock::time_point const& now) } else if (phase_ == ConsensusPhase::establish) { + if (result_) + ++result_->peerUnchangedCounter; phaseEstablish(); } } @@ -1443,16 +1454,10 @@ Consensus::updateOurPositions() } else { - int neededWeight; - - if (convergePercent_ < parms.avMID_CONSENSUS_TIME) - neededWeight = parms.avINIT_CONSENSUS_PCT; - else if (convergePercent_ < parms.avLATE_CONSENSUS_TIME) - neededWeight = parms.avMID_CONSENSUS_PCT; - else if (convergePercent_ < parms.avSTUCK_CONSENSUS_TIME) - neededWeight = parms.avLATE_CONSENSUS_PCT; - else - neededWeight = parms.avSTUCK_CONSENSUS_PCT; + auto const [neededWeight, newState] = getNeededWeight( + parms, closeTimeAvalancheState_, convergePercent_, {}); + if (newState) + closeTimeAvalancheState_ = *newState; int participants = currPeerPositions_.size(); if (mode_.get() == ConsensusMode::proposing) @@ -1566,7 +1571,8 @@ Consensus::haveConsensus() } else { - JLOG(j_.debug()) << nodeId << " has " << peerProp.position(); + JLOG(j_.debug()) << "Proposal disagreement: Peer " << nodeId + << " has " << peerProp.position(); ++disagree; } } @@ -1576,6 +1582,18 @@ Consensus::haveConsensus() JLOG(j_.debug()) << "Checking for TX consensus: agree=" << agree << ", disagree=" << disagree; + ConsensusParms const& parms = adaptor_.parms(); + // stable state is NOT desirable if we don't have 80% agreement + bool stableState = true; + for (auto const& [txid, dt] : result_->disputes) + { + if (!dt.stableState(parms, result_->peerUnchangedCounter)) + { + stableState = false; + break; + } + } + // Determine if we actually have consensus or not result_->state = checkConsensus( prevProposers_, @@ -1584,13 +1602,21 @@ Consensus::haveConsensus() currentFinished, prevRoundTime_, result_->roundTime.read(), - adaptor_.parms(), + stableState, + parms, mode_.get() == ConsensusMode::proposing, j_); if (result_->state == ConsensusState::No) return false; + // Consensus has taken far too long. Drop out of the round. + if (result_->state == ConsensusState::Expired) + { + JLOG(j_.error()) << "Nobody can reach consensus"; + JLOG(j_.error()) << Json::Compact{getJson(true)}; + leaveConsensus(); + } // There is consensus, but we need to track if the network moved on // without us. if (result_->state == ConsensusState::MovedOn) @@ -1669,8 +1695,9 @@ Consensus::createDisputes(TxSet_t const& o) { Proposal_t const& peerProp = peerPos.proposal(); auto const cit = acquired_.find(peerProp.position()); - if (cit != acquired_.end()) - dtx.setVote(nodeId, cit->second.exists(txID)); + if (cit != acquired_.end() && + dtx.setVote(nodeId, cit->second.exists(txID))) + result_->peerUnchangedCounter = 0; } adaptor_.share(dtx.tx()); @@ -1694,7 +1721,8 @@ Consensus::updateDisputes(NodeID_t const& node, TxSet_t const& other) for (auto& it : result_->disputes) { auto& d = it.second; - d.setVote(node, other.exists(d.tx().id())); + if (d.setVote(node, other.exists(d.tx().id()))) + result_->peerUnchangedCounter = 0; } } diff --git a/src/xrpld/consensus/ConsensusParms.h b/src/xrpld/consensus/ConsensusParms.h index a0b6c6be8d4..ee0e2966ca1 100644 --- a/src/xrpld/consensus/ConsensusParms.h +++ b/src/xrpld/consensus/ConsensusParms.h @@ -20,8 +20,12 @@ #ifndef RIPPLE_CONSENSUS_CONSENSUS_PARMS_H_INCLUDED #define RIPPLE_CONSENSUS_CONSENSUS_PARMS_H_INCLUDED +#include #include #include +#include +#include +#include namespace ripple { @@ -43,7 +47,7 @@ struct ConsensusParms This is a safety to protect against very old validations and the time it takes to adjust the close time accuracy window. */ - std::chrono::seconds validationVALID_WALL = std::chrono::minutes{5}; + std::chrono::seconds const validationVALID_WALL = std::chrono::minutes{5}; /** Duration a validation remains current after first observed. @@ -51,33 +55,34 @@ struct ConsensusParms first saw it. This provides faster recovery in very rare cases where the number of validations produced by the network is lower than normal */ - std::chrono::seconds validationVALID_LOCAL = std::chrono::minutes{3}; + std::chrono::seconds const validationVALID_LOCAL = std::chrono::minutes{3}; /** Duration pre-close in which validations are acceptable. The number of seconds before a close time that we consider a validation acceptable. This protects against extreme clock errors */ - std::chrono::seconds validationVALID_EARLY = std::chrono::minutes{3}; + std::chrono::seconds const validationVALID_EARLY = std::chrono::minutes{3}; //! How long we consider a proposal fresh - std::chrono::seconds proposeFRESHNESS = std::chrono::seconds{20}; + std::chrono::seconds const proposeFRESHNESS = std::chrono::seconds{20}; //! How often we force generating a new proposal to keep ours fresh - std::chrono::seconds proposeINTERVAL = std::chrono::seconds{12}; + std::chrono::seconds const proposeINTERVAL = std::chrono::seconds{12}; //------------------------------------------------------------------------- // Consensus durations are relative to the internal Consensus clock and use // millisecond resolution. //! The percentage threshold above which we can declare consensus. - std::size_t minCONSENSUS_PCT = 80; + std::size_t const minCONSENSUS_PCT = 80; //! The duration a ledger may remain idle before closing - std::chrono::milliseconds ledgerIDLE_INTERVAL = std::chrono::seconds{15}; + std::chrono::milliseconds const ledgerIDLE_INTERVAL = + std::chrono::seconds{15}; //! The number of seconds we wait minimum to ensure participation - std::chrono::milliseconds ledgerMIN_CONSENSUS = + std::chrono::milliseconds const ledgerMIN_CONSENSUS = std::chrono::milliseconds{1950}; /** The maximum amount of time to spend pausing for laggards. @@ -86,13 +91,26 @@ struct ConsensusParms * validators don't appear to be offline that are merely waiting for * laggards. */ - std::chrono::milliseconds ledgerMAX_CONSENSUS = std::chrono::seconds{15}; + std::chrono::milliseconds const ledgerMAX_CONSENSUS = + std::chrono::seconds{15}; //! Minimum number of seconds to wait to ensure others have computed the LCL - std::chrono::milliseconds ledgerMIN_CLOSE = std::chrono::seconds{2}; + std::chrono::milliseconds const ledgerMIN_CLOSE = std::chrono::seconds{2}; //! How often we check state or change positions - std::chrono::milliseconds ledgerGRANULARITY = std::chrono::seconds{1}; + std::chrono::milliseconds const ledgerGRANULARITY = std::chrono::seconds{1}; + + //! How long to wait before completely abandoning consensus + std::size_t const ledgerABANDON_CONSENSUS_FACTOR = 10; + + /** + * Maximum amount of time to give a consensus round + * + * Does not include the time to build the LCL, so there is no reason for a + * round to go this long, regardless of how big the ledger is. + */ + std::chrono::milliseconds const ledgerABANDON_CONSENSUS = + std::chrono::seconds{60}; /** The minimum amount of time to consider the previous round to have taken. @@ -104,7 +122,8 @@ struct ConsensusParms twice the interval between proposals (0.7s) divided by the interval between mid and late consensus ([85-50]/100). */ - std::chrono::milliseconds avMIN_CONSENSUS_TIME = std::chrono::seconds{5}; + std::chrono::milliseconds const avMIN_CONSENSUS_TIME = + std::chrono::seconds{5}; //------------------------------------------------------------------------------ // Avalanche tuning @@ -112,30 +131,68 @@ struct ConsensusParms // we increase the threshold for yes votes to add a transaction to our // position. - //! Percentage of nodes on our UNL that must vote yes - std::size_t avINIT_CONSENSUS_PCT = 50; - - //! Percentage of previous round duration before we advance - std::size_t avMID_CONSENSUS_TIME = 50; - - //! Percentage of nodes that most vote yes after advancing - std::size_t avMID_CONSENSUS_PCT = 65; - - //! Percentage of previous round duration before we advance - std::size_t avLATE_CONSENSUS_TIME = 85; - - //! Percentage of nodes that most vote yes after advancing - std::size_t avLATE_CONSENSUS_PCT = 70; - - //! Percentage of previous round duration before we are stuck - std::size_t avSTUCK_CONSENSUS_TIME = 200; - - //! Percentage of nodes that must vote yes after we are stuck - std::size_t avSTUCK_CONSENSUS_PCT = 95; - //! Percentage of nodes required to reach agreement on ledger close time - std::size_t avCT_CONSENSUS_PCT = 75; + std::size_t const avCT_CONSENSUS_PCT = 75; + + //! Number of rounds before certain actions can happen. + // (Moving to the next avalanche level, considering that votes are in a + // stable state without consensus.) + std::size_t const avMIN_ROUNDS = 2; + + //! Number of rounds before a stuck vote is considered unlikely to change + //! because voting is in an undesriable stable state + std::size_t const avSTUCK_VOTE_ROUNDS = 5; + + enum AvalancheState { init, mid, late, stuck }; + struct AvalancheCutoff + { + std::size_t const consensusTime; + std::size_t const consensusPct; + AvalancheState const next; + }; + std::map const avalancheCutoffs{ + // {state, {time, percent, nextState}}, + // Initial state: 50% of nodes must vote yes + {init, {0, 50, mid}}, + // mid-consensus starts after 50% of the previous round time, and + // requires 65% yes + {mid, {50, 65, late}}, + // late consensus starts after 85% time, and requires 70% yes + {late, {85, 70, stuck}}, + // we're stuck after 2x time, requires 95% yes votes + {stuck, {200, 95, stuck}}, + }; }; +inline std::pair> +getNeededWeight( + ConsensusParms const& p, + ConsensusParms::AvalancheState currentState, + int percentTime, + std::function + considerNextCallback) +{ + // at() can throw, but the map is built by hand to ensure all valid + // values are available. + auto const& currentCutoff = p.avalancheCutoffs.at(currentState); + // Should we consider moving to the next state? + if (currentCutoff.next != currentState && + (!considerNextCallback || considerNextCallback(currentCutoff))) + { + // at() can throw, but the map is built by hand to ensure all + // valid values are available. + auto const& nextCutoff = p.avalancheCutoffs.at(currentCutoff.next); + // See if enough time has passed to move on to the next. + XRPL_ASSERT( + nextCutoff.consensusTime, + "ripple::DisputedTx::updateVote next state valid"); + if (percentTime >= nextCutoff.consensusTime) + { + return {nextCutoff.consensusPct, currentCutoff.next}; + } + } + return {currentCutoff.consensusPct, {}}; +} + } // namespace ripple #endif diff --git a/src/xrpld/consensus/ConsensusTypes.h b/src/xrpld/consensus/ConsensusTypes.h index ba8e0a8b1ac..1bf6323aef2 100644 --- a/src/xrpld/consensus/ConsensusTypes.h +++ b/src/xrpld/consensus/ConsensusTypes.h @@ -186,6 +186,7 @@ struct ConsensusCloseTimes enum class ConsensusState { No, //!< We do not have consensus MovedOn, //!< The network has consensus without us + Expired, //!< Consensus time limit has hard-expired Yes //!< We have consensus along with the network }; @@ -234,8 +235,12 @@ struct ConsensusResult // Measures the duration of the establish phase for this consensus round ConsensusTimer roundTime; + // The number of attempts to establish consensus where none of our peers + // have changed any votes on disputed transactions. + std::size_t peerUnchangedCounter; + // Indicates state in which consensus ended. Once in the accept phase - // will be either Yes or MovedOn + // will be either Yes or MovedOn or Expired ConsensusState state = ConsensusState::No; // The number of peers proposing during the round diff --git a/src/xrpld/consensus/DisputedTx.h b/src/xrpld/consensus/DisputedTx.h index bffb1009323..1eeee0dfecf 100644 --- a/src/xrpld/consensus/DisputedTx.h +++ b/src/xrpld/consensus/DisputedTx.h @@ -84,6 +84,31 @@ class DisputedTx return ourVote_; } + //! Are we and our peers at a "stable state" where we probably won't change + //! our vote? + bool + stableState(ConsensusParms const& p, int peersUnchanged) const + { + // at() can throw, but the map is built by hand to ensure all valid + // values are available. + auto const& currentCutoff = p.avalancheCutoffs.at(avalancheState_); + + // We're not at the final avalanche state, so there's room for change + if (avalancheState_ != currentCutoff.next) + return false; + + // We've haven't had this vote for 2 rounds yet. Things could change. + if (currentVoteCounter_ < p.avMIN_ROUNDS) + return false; + + // If we or any peers have changed a vote in the last 5 rounds, then + // things could still change. But if _either_ has not changed in that + // long, we're unlikely to change our vote any time soon. (This prevents + // a malicious peer from flip-flopping a vote to prevent consensus.) + return peersUnchanged >= p.avSTUCK_VOTE_ROUNDS || + currentVoteCounter_ >= p.avSTUCK_VOTE_ROUNDS; + } + //! The disputed transaction. Tx_t const& tx() const @@ -102,8 +127,12 @@ class DisputedTx @param peer Identifier of peer. @param votesYes Whether peer votes to include the disputed transaction. + + @return bool Whether the peer changed its vote. (A new vote counts as a + change.) */ - void + [[nodiscard]] + bool setVote(NodeID_t const& peer, bool votesYes); /** Remove a peer's vote @@ -137,12 +166,16 @@ class DisputedTx bool ourVote_; //< Our vote (true is yes) Tx_t tx_; //< Transaction under dispute Map_t votes_; //< Map from NodeID to vote + //! The number of rounds we've gone without changing our vote + std::size_t currentVoteCounter_ = 0; + ConsensusParms::AvalancheState avalancheState_ = ConsensusParms::init; + std::size_t avalancheCounter_ = 0; beast::Journal const j_; }; // Track a peer's yes/no vote on a particular disputed tx_ template -void +bool DisputedTx::setVote(NodeID_t const& peer, bool votesYes) { auto const [it, inserted] = votes_.insert(std::make_pair(peer, votesYes)); @@ -160,6 +193,7 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) JLOG(j_.debug()) << "Peer " << peer << " votes NO on " << tx_.id(); ++nays_; } + return true; } // changes vote to yes else if (votesYes && !it->second) @@ -168,6 +202,7 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) --nays_; ++yays_; it->second = true; + return true; } // changes vote to no else if (!votesYes && it->second) @@ -176,7 +211,9 @@ DisputedTx::setVote(NodeID_t const& peer, bool votesYes) ++nays_; --yays_; it->second = false; + return true; } + return false; } // Remove a peer's vote on this disputed transaction @@ -218,16 +255,26 @@ DisputedTx::updateVote( // This is basically the percentage of nodes voting 'yes' (including us) weight = (yays_ * 100 + (ourVote_ ? 100 : 0)) / (nays_ + yays_ + 1); - // To prevent avalanche stalls, we increase the needed weight slightly - // over time. - if (percentTime < p.avMID_CONSENSUS_TIME) - newPosition = weight > p.avINIT_CONSENSUS_PCT; - else if (percentTime < p.avLATE_CONSENSUS_TIME) - newPosition = weight > p.avMID_CONSENSUS_PCT; - else if (percentTime < p.avSTUCK_CONSENSUS_TIME) - newPosition = weight > p.avLATE_CONSENSUS_PCT; - else - newPosition = weight > p.avSTUCK_CONSENSUS_PCT; + // To prevent avalanche stalls, we increase the needed weight + // slightly over time. We also need to ensure that the consensus has + // made a minimum number of attempts at each "state" before moving + // to the next. + auto const [requiredPct, newState] = getNeededWeight( + p, + avalancheState_, + percentTime, + [&](ConsensusParms::AvalancheCutoff const& currentState) { + // Have we spent sufficient rounds at this step. + return ++avalancheCounter_ >= p.avMIN_ROUNDS && + currentState.consensusPct; + }); + if (newState) + { + avalancheState_ = *newState; + avalancheCounter_ = 0; + } + + newPosition = weight > requiredPct; } else { @@ -238,13 +285,16 @@ DisputedTx::updateVote( if (newPosition == ourVote_) { + ++currentVoteCounter_; JLOG(j_.info()) << "No change (" << (ourVote_ ? "YES" : "NO") << ") : weight " << weight << ", percent " - << percentTime; + << percentTime + << ", round(s) with this vote: " << currentVoteCounter_; JLOG(j_.debug()) << Json::Compact{getJson()}; return false; } + currentVoteCounter_ = 0; ourVote_ = newPosition; JLOG(j_.debug()) << "We now vote " << (ourVote_ ? "YES" : "NO") << " on " << tx_.id();