From 94e61f96b584d4790d416a3500c79911e377b21a Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 26 Sep 2024 16:22:50 +0200 Subject: [PATCH 01/71] adds a new overload of queryPastEvents allowing to query past events based on timestamp in the past --- codex/contracts/market.nim | 78 +++++++++++++++++++++++++++++++++++++- codex/market.nim | 6 +++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 049e38bb6..dbe745b49 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,6 +1,7 @@ -import std/sequtils +# import std/sequtils import std/strutils -import std/sugar +import std/strformat +# import std/sugar import pkg/ethers import pkg/upraises import pkg/questionable @@ -482,3 +483,76 @@ method queryPastEvents*[T: MarketplaceEvent]( return await contract.queryFilter(T, fromBlock, BlockTag.latest) + +proc blockNumberAndTimestamp(provider: Provider, blockTag: BlockTag): + Future[(UInt256, UInt256)] {.async.} = + without latestBlock =? await provider.getBlock(blockTag), error: + raise error + + without latestBlockNumber =? latestBlock.number: + raise newException(EthersError, "Could not get latest block number") + + (latestBlockNumber, latestBlock.timestamp) + +proc blockNumberForEpoch(epochTime: int64, provider: Provider): Future[UInt256] + {.async.} = + let avgBlockTime = 13.u256 + let epochTimeUInt256 = epochTime.u256 + let (latestBlockNumber, latestBlockTimestamp) = + await blockNumberAndTimestamp(provider, BlockTag.latest) + + let timeDiff = latestBlockTimestamp - epochTimeUInt256 + let blockDiff = timeDiff div avgBlockTime + let estimatedBlockNumber = latestBlockNumber - blockDiff + + let (estimatedBlockTimestamp, _) = await blockNumberAndTimestamp( + provider, BlockTag.init(estimatedBlockNumber)) + + var low = 0.u256 + var high = latestBlockNumber + if estimatedBlockTimestamp < epochTimeUInt256: + low = estimatedBlockNumber + else: + high = estimatedBlockNumber + + while low <= high: + let mid = (low + high) div 2 + let (midBlockTimestamp, midBlockNumber) = + await blockNumberAndTimestamp(provider, BlockTag.init(mid)) + + if midBlockTimestamp < epochTimeUInt256: + low = mid + 1 + elif midBlockTimestamp > epochTimeUInt256: + high = mid - 1 + else: + return midBlockNumber + + let (_, lowTimestamp) = await blockNumberAndTimestamp( + provider, BlockTag.init(low)) + let (_, highTimestamp) = await blockNumberAndTimestamp( + provider, BlockTag.init(high)) + try: + if abs(lowTimestamp.stint(256) - epochTimeUInt256.stint(256)) < + abs(highTimestamp.stint(256) - epochTimeUInt256.stint(256)): + low + else: + high + except ValueError as e: + raise newException(EthersError, fmt"Conversion error: {e.msg}") + +method queryPastEvents*[T: MarketplaceEvent]( + market: OnChainMarket, + _: type T, + fromTime: int64): Future[seq[T]] {.async.} = + + convertEthersError: + let contract = market.contract + let provider = contract.provider + + let blockNumberForEpoch = await blockNumberForEpoch(fromTime, provider) + + let fromBlock = BlockTag.init(blockNumberForEpoch) + + return await contract.queryFilter(T, + fromBlock, + BlockTag.latest) diff --git a/codex/market.nim b/codex/market.nim index cb86e0d79..decf11e74 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -251,3 +251,9 @@ method queryPastEvents*[T: MarketplaceEvent]( _: type T, blocksAgo: int): Future[seq[T]] {.base, async.} = raiseAssert("not implemented") + +method queryPastEvents*[T: MarketplaceEvent]( + market: Market, + _: type T, + fromTime: int64): Future[seq[T]] {.base, async.} = + raiseAssert("not implemented") From e080295901bf3aac277c24fd5be252f61cea2cd7 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 26 Sep 2024 16:23:29 +0200 Subject: [PATCH 02/71] adds state restoration to validator --- codex/validation.nim | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/codex/validation.nim b/codex/validation.nim index d00f5772f..80e15c8f4 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -1,3 +1,4 @@ +import std/times import std/sets import std/sequtils import pkg/chronos @@ -23,6 +24,9 @@ type proofTimeout: UInt256 config: ValidationConfig +const + MaxStorageRequestDuration: times.Duration = initDuration(days = 30) + logScope: topics = "codex validator" @@ -119,10 +123,26 @@ proc run(validation: Validation) {.async.} = except CatchableError as e: error "Validation failed", msg = e.msg +proc epochForDurationBackFromNow(duration: times.Duration): int64 = + let now = getTime().toUnix + return now - duration.inSeconds + +proc restoreHistoricalState(validation: Validation) {.async} = + let startTimeEpoch = epochForDurationBackFromNow(MaxStorageRequestDuration) + let slotFilledEvents = await validation.market.queryPastEvents(SlotFilled, + fromTime = startTimeEpoch) + for event in slotFilledEvents: + let slotId = slotId(event.requestId, event.slotIndex) + if validation.shouldValidateSlot(slotId): + trace "Adding slot", slotId + validation.slots.incl(slotId) + await removeSlotsThatHaveEnded(validation) + proc start*(validation: Validation) {.async.} = validation.periodicity = await validation.market.periodicity() validation.proofTimeout = await validation.market.proofTimeout() await validation.subscribeSlotFilled() + await validation.restoreHistoricalState() validation.running = validation.run() proc stop*(validation: Validation) {.async.} = From 0587461d536a3867e35aaa2606ba294bf7e9ba81 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 27 Sep 2024 23:53:02 +0200 Subject: [PATCH 03/71] refactors a bit to get the tests back to work --- codex/contracts/market.nim | 8 +++----- codex/market.nim | 5 ++--- codex/validation.nim | 2 +- tests/codex/helpers/mockmarket.nim | 7 +++++++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index dbe745b49..0051529f0 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -540,11 +540,9 @@ proc blockNumberForEpoch(epochTime: int64, provider: Provider): Future[UInt256] except ValueError as e: raise newException(EthersError, fmt"Conversion error: {e.msg}") -method queryPastEvents*[T: MarketplaceEvent]( +method queryPastSlotFilledEvents*( market: OnChainMarket, - _: type T, - fromTime: int64): Future[seq[T]] {.async.} = - + fromTime: int64): Future[seq[SlotFilled]] {.async.} = convertEthersError: let contract = market.contract let provider = contract.provider @@ -553,6 +551,6 @@ method queryPastEvents*[T: MarketplaceEvent]( let fromBlock = BlockTag.init(blockNumberForEpoch) - return await contract.queryFilter(T, + return await contract.queryFilter(SlotFilled, fromBlock, BlockTag.latest) diff --git a/codex/market.nim b/codex/market.nim index decf11e74..9a3e56758 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -252,8 +252,7 @@ method queryPastEvents*[T: MarketplaceEvent]( blocksAgo: int): Future[seq[T]] {.base, async.} = raiseAssert("not implemented") -method queryPastEvents*[T: MarketplaceEvent]( +method queryPastSlotFilledEvents*( market: Market, - _: type T, - fromTime: int64): Future[seq[T]] {.base, async.} = + fromTime: int64): Future[seq[SlotFilled]] {.base, async.} = raiseAssert("not implemented") diff --git a/codex/validation.nim b/codex/validation.nim index 80e15c8f4..ac1a94001 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -129,7 +129,7 @@ proc epochForDurationBackFromNow(duration: times.Duration): int64 = proc restoreHistoricalState(validation: Validation) {.async} = let startTimeEpoch = epochForDurationBackFromNow(MaxStorageRequestDuration) - let slotFilledEvents = await validation.market.queryPastEvents(SlotFilled, + let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) for event in slotFilledEvents: let slotId = slotId(event.requestId, event.slotIndex) diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 07eeb856e..72c0c57f9 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -488,6 +488,13 @@ method queryPastEvents*[T: MarketplaceEvent]( SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) ) +method queryPastSlotFilledEvents*( + market: MockMarket, + fromTime: int64): Future[seq[SlotFilled]] {.async.} = + return market.filled.map(slot => + SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) + ) + method unsubscribe*(subscription: RequestSubscription) {.async.} = subscription.market.subscriptions.onRequest.keepItIf(it != subscription) From 0c2dd1d5f0135029bd92b1ef03996823499cd860 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 7 Oct 2024 20:07:22 +0200 Subject: [PATCH 04/71] replaces deprecated generic methods from Market with methods for specific event types --- codex/contracts/market.nim | 74 +++++++++++++++++++----------- codex/market.nim | 26 ++++++++--- tests/codex/helpers/mockmarket.nim | 49 ++++++++++++++------ tests/contracts/testMarket.nim | 10 ++-- 4 files changed, 108 insertions(+), 51 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 0051529f0..2502e5b71 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -468,21 +468,10 @@ method subscribeProofSubmission*(market: OnChainMarket, method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() -method queryPastEvents*[T: MarketplaceEvent]( - market: OnChainMarket, - _: type T, - blocksAgo: int): Future[seq[T]] {.async.} = - - convertEthersError: - let contract = market.contract - let provider = contract.provider - - let head = await provider.getBlockNumber() - let fromBlock = BlockTag.init(head - blocksAgo.abs.u256) - - return await contract.queryFilter(T, - fromBlock, - BlockTag.latest) +proc blockNumberForBlocksEgo(provider: Provider, + blocksAgo: int): Future[BlockTag] {.async.} = + let head = await provider.getBlockNumber() + return BlockTag.init(head - blocksAgo.abs.u256) proc blockNumberAndTimestamp(provider: Provider, blockTag: BlockTag): Future[(UInt256, UInt256)] {.async.} = @@ -494,7 +483,7 @@ proc blockNumberAndTimestamp(provider: Provider, blockTag: BlockTag): (latestBlockNumber, latestBlock.timestamp) -proc blockNumberForEpoch(epochTime: int64, provider: Provider): Future[UInt256] +proc blockNumberForEpoch(provider: Provider, epochTime: int64): Future[BlockTag] {.async.} = let avgBlockTime = 13.u256 let epochTimeUInt256 = epochTime.u256 @@ -525,7 +514,7 @@ proc blockNumberForEpoch(epochTime: int64, provider: Provider): Future[UInt256] elif midBlockTimestamp > epochTimeUInt256: high = mid - 1 else: - return midBlockNumber + return BlockTag.init(midBlockNumber) let (_, lowTimestamp) = await blockNumberAndTimestamp( provider, BlockTag.init(low)) @@ -534,23 +523,56 @@ proc blockNumberForEpoch(epochTime: int64, provider: Provider): Future[UInt256] try: if abs(lowTimestamp.stint(256) - epochTimeUInt256.stint(256)) < abs(highTimestamp.stint(256) - epochTimeUInt256.stint(256)): - low + BlockTag.init(low) else: - high + BlockTag.init(high) except ValueError as e: raise newException(EthersError, fmt"Conversion error: {e.msg}") +method queryPastSlotFilledEvents*( + market: OnChainMarket, + fromBlock: BlockTag): Future[seq[SlotFilled]] {.async.} = + + convertEthersError: + return await market.contract.queryFilter(SlotFilled, + fromBlock, + BlockTag.latest) + +method queryPastSlotFilledEvents*( + market: OnChainMarket, + blocksAgo: int): Future[seq[SlotFilled]] {.async.} = + + convertEthersError: + let fromBlock = + await blockNumberForBlocksEgo(market.contract.provider, blocksAgo) + + return await market.queryPastSlotFilledEvents(fromBlock) + method queryPastSlotFilledEvents*( market: OnChainMarket, fromTime: int64): Future[seq[SlotFilled]] {.async.} = + + convertEthersError: + let fromBlock = await blockNumberForEpoch(market.contract.provider, + fromTime) + + return await market.queryPastSlotFilledEvents(fromBlock) + +method queryPastStorageRequestedEvents*( + market: OnChainMarket, + fromBlock: BlockTag): Future[seq[StorageRequested]] {.async.} = + convertEthersError: - let contract = market.contract - let provider = contract.provider + return await market.contract.queryFilter(StorageRequested, + fromBlock, + BlockTag.latest) - let blockNumberForEpoch = await blockNumberForEpoch(fromTime, provider) +method queryPastStorageRequestedEvents*( + market: OnChainMarket, + blocksAgo: int): Future[seq[StorageRequested]] {.async.} = - let fromBlock = BlockTag.init(blockNumberForEpoch) + convertEthersError: + let fromBlock = + await blockNumberForBlocksEgo(market.contract.provider, blocksAgo) - return await contract.queryFilter(SlotFilled, - fromBlock, - BlockTag.latest) + return await market.queryPastStorageRequestedEvents(fromBlock) diff --git a/codex/market.nim b/codex/market.nim index 9a3e56758..0fbcbff78 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -246,13 +246,27 @@ method subscribeProofSubmission*(market: Market, method unsubscribe*(subscription: Subscription) {.base, async, upraises:[].} = raiseAssert("not implemented") -method queryPastEvents*[T: MarketplaceEvent]( - market: Market, - _: type T, - blocksAgo: int): Future[seq[T]] {.base, async.} = +method queryPastSlotFilledEvents*( + market: Market, + fromBlock: BlockTag): Future[seq[SlotFilled]] {.base, async.} = raiseAssert("not implemented") method queryPastSlotFilledEvents*( - market: Market, - fromTime: int64): Future[seq[SlotFilled]] {.base, async.} = + market: Market, + blocksAgo: int): Future[seq[SlotFilled]] {.base, async.} = + raiseAssert("not implemented") + +method queryPastStorageRequestedEvents*( + market: Market, + fromBlock: BlockTag): Future[seq[StorageRequested]] {.base, async.} = + raiseAssert("not implemented") + +method queryPastStorageRequestedEvents*( + market: Market, + blocksAgo: int): Future[seq[StorageRequested]] {.base, async.} = + raiseAssert("not implemented") + +method queryPastSlotFilledEvents*( + market: Market, + fromTime: int64): Future[seq[SlotFilled]] {.base, async.} = raiseAssert("not implemented") diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 72c0c57f9..6f393ca3e 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -8,6 +8,9 @@ import pkg/codex/market import pkg/codex/contracts/requests import pkg/codex/contracts/proofs import pkg/codex/contracts/config + +from pkg/ethers import BlockTag + import ../examples export market @@ -472,21 +475,37 @@ method subscribeProofSubmission*(mock: MockMarket, mock.subscriptions.onProofSubmitted.add(subscription) return subscription -method queryPastEvents*[T: MarketplaceEvent]( - market: MockMarket, - _: type T, - blocksAgo: int): Future[seq[T]] {.async.} = - - if T of StorageRequested: - return market.requested.map(request => - StorageRequested(requestId: request.id, - ask: request.ask, - expiry: request.expiry) - ) - elif T of SlotFilled: - return market.filled.map(slot => - SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) - ) +method queryPastStorageRequestedEvents*( + market: MockMarket, + fromBlock: BlockTag): Future[seq[StorageRequested]] {.async.} = + return market.requested.map(request => + StorageRequested(requestId: request.id, + ask: request.ask, + expiry: request.expiry) + ) + +method queryPastStorageRequestedEvents*( + market: MockMarket, + blocksAgo: int): Future[seq[StorageRequested]] {.async.} = + return market.requested.map(request => + StorageRequested(requestId: request.id, + ask: request.ask, + expiry: request.expiry) + ) + +method queryPastSlotFilledEvents*( + market: MockMarket, + fromBlock: BlockTag): Future[seq[SlotFilled]] {.async.} = + return market.filled.map(slot => + SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) + ) + +method queryPastSlotFilledEvents*( + market: MockMarket, + blocksAgo: int): Future[seq[SlotFilled]] {.async.} = + return market.filled.map(slot => + SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) + ) method queryPastSlotFilledEvents*( market: MockMarket, diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index a32590d6f..e37f64119 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -412,7 +412,8 @@ ethersuite "On-Chain Market": # ago". proc getsPastRequest(): Future[bool] {.async.} = - let reqs = await market.queryPastEvents(StorageRequested, 5) + let reqs = + await market.queryPastStorageRequestedEvents(blocksAgo = 5) reqs.mapIt(it.requestId) == @[request.id, request1.id, request2.id] check eventually await getsPastRequest() @@ -431,7 +432,8 @@ ethersuite "On-Chain Market": # two PoA blocks per `fillSlot` call (6 blocks for 3 calls). We don't need # to check the `approve` for the first `fillSlot` call, so we only need to # check 5 "blocks ago". - let events = await market.queryPastEvents(SlotFilled, 5) + let events = + await market.queryPastSlotFilledEvents(blocksAgo = 5) check events == @[ SlotFilled(requestId: request.id, slotIndex: 0.u256), SlotFilled(requestId: request.id, slotIndex: 1.u256), @@ -442,8 +444,8 @@ ethersuite "On-Chain Market": await market.requestStorage(request) check eventually ( - (await market.queryPastEvents(StorageRequested, blocksAgo = -2)) == - (await market.queryPastEvents(StorageRequested, blocksAgo = 2)) + (await market.queryPastStorageRequestedEvents(blocksAgo = -2)) == + (await market.queryPastStorageRequestedEvents(blocksAgo = 2)) ) test "pays rewards and collateral to host": From c19c775eba1a312b5841387e5a9dc197e62dbe3d Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 8 Oct 2024 07:13:52 +0200 Subject: [PATCH 05/71] Refactors binary search --- codex/contracts/market.nim | 99 +++++++++++++++++++++++--------------- codex/validation.nim | 6 ++- 2 files changed, 65 insertions(+), 40 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 2502e5b71..bd4ce8587 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -483,51 +483,72 @@ proc blockNumberAndTimestamp(provider: Provider, blockTag: BlockTag): (latestBlockNumber, latestBlock.timestamp) -proc blockNumberForEpoch(provider: Provider, epochTime: int64): Future[BlockTag] - {.async.} = - let avgBlockTime = 13.u256 - let epochTimeUInt256 = epochTime.u256 - let (latestBlockNumber, latestBlockTimestamp) = - await blockNumberAndTimestamp(provider, BlockTag.latest) - - let timeDiff = latestBlockTimestamp - epochTimeUInt256 - let blockDiff = timeDiff div avgBlockTime - let estimatedBlockNumber = latestBlockNumber - blockDiff - - let (estimatedBlockTimestamp, _) = await blockNumberAndTimestamp( - provider, BlockTag.init(estimatedBlockNumber)) +proc estimateAverageBlockTime(provider: Provider): Future[UInt256] {.async.} = + let (latestBlockNumber, latestBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.latest) + let (_, previousBlockTimestamp) = + await provider.blockNumberAndTimestamp( + BlockTag.init(latestBlockNumber - 1.u256)) + trace "[estimateAverageBlockTime]:", latestBlockNumber = latestBlockNumber, + latestBlockTimestamp = latestBlockTimestamp, + previousBlockTimestamp = previousBlockTimestamp + return latestBlockTimestamp - previousBlockTimestamp + +proc binarySearchFindClosestBlock(provider: Provider, + epochTime: int, + low: BlockTag, + high: BlockTag): Future[BlockTag] {.async.} = + let (_, lowTimestamp) = + await provider.blockNumberAndTimestamp(low) + let (_, highTimestamp) = + await provider.blockNumberAndTimestamp(high) + if abs(lowTimestamp.truncate(int) - epochTime) < + abs(highTimestamp.truncate(int) - epochTime): + return low + else: + return high +proc binarySearchBlockNumberForEpoch(provider: Provider, + epochTime: UInt256, + latestBlockNumber: UInt256): + Future[BlockTag] {.async.} = var low = 0.u256 var high = latestBlockNumber - if estimatedBlockTimestamp < epochTimeUInt256: - low = estimatedBlockNumber - else: - high = estimatedBlockNumber + trace "[binarySearchBlockNumberForEpoch]:", low = low, high = high while low <= high: - let mid = (low + high) div 2 - let (midBlockTimestamp, midBlockNumber) = - await blockNumberAndTimestamp(provider, BlockTag.init(mid)) + let mid = (low + high) div 2.u256 + let (midBlockNumber, midBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.init(mid)) - if midBlockTimestamp < epochTimeUInt256: - low = mid + 1 - elif midBlockTimestamp > epochTimeUInt256: - high = mid - 1 + if midBlockTimestamp < epochTime: + low = mid + 1.u256 + elif midBlockTimestamp > epochTime: + high = mid - 1.u256 else: return BlockTag.init(midBlockNumber) + await provider.binarySearchFindClosestBlock( + epochTime.truncate(int), BlockTag.init(low), BlockTag.init(high)) - let (_, lowTimestamp) = await blockNumberAndTimestamp( - provider, BlockTag.init(low)) - let (_, highTimestamp) = await blockNumberAndTimestamp( - provider, BlockTag.init(high)) - try: - if abs(lowTimestamp.stint(256) - epochTimeUInt256.stint(256)) < - abs(highTimestamp.stint(256) - epochTimeUInt256.stint(256)): - BlockTag.init(low) - else: - BlockTag.init(high) - except ValueError as e: - raise newException(EthersError, fmt"Conversion error: {e.msg}") +proc blockNumberForEpoch(provider: Provider, epochTime: int64): Future[BlockTag] + {.async.} = + let avgBlockTime = await provider.estimateAverageBlockTime() + trace "[blockNumberForEpoch]:", avgBlockTime = avgBlockTime + let epochTimeUInt256 = epochTime.u256 + let (latestBlockNumber, latestBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.latest) + + trace "[blockNumberForEpoch]:", latestBlockNumber = latestBlockNumber, + latestBlockTimestamp = latestBlockTimestamp + + let timeDiff = latestBlockTimestamp - epochTimeUInt256 + let blockDiff = timeDiff div avgBlockTime + + if blockDiff >= latestBlockNumber: + return BlockTag.earliest + + return await provider.binarySearchBlockNumberForEpoch( + epochTimeUInt256, latestBlockNumber) method queryPastSlotFilledEvents*( market: OnChainMarket, @@ -553,9 +574,9 @@ method queryPastSlotFilledEvents*( fromTime: int64): Future[seq[SlotFilled]] {.async.} = convertEthersError: - let fromBlock = await blockNumberForEpoch(market.contract.provider, - fromTime) - + let fromBlock = + await market.contract.provider.blockNumberForEpoch(fromTime) + trace "queryPastSlotFilledEvents fromTime", fromTime=fromTime, fromBlock=fromBlock return await market.queryPastSlotFilledEvents(fromBlock) method queryPastStorageRequestedEvents*( diff --git a/codex/validation.nim b/codex/validation.nim index ac1a94001..b245ebc28 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -128,15 +128,19 @@ proc epochForDurationBackFromNow(duration: times.Duration): int64 = return now - duration.inSeconds proc restoreHistoricalState(validation: Validation) {.async} = + trace "Restoring historical state..." let startTimeEpoch = epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) + trace "Found slot filled events", numberOfSlots = slotFilledEvents.len for event in slotFilledEvents: let slotId = slotId(event.requestId, event.slotIndex) if validation.shouldValidateSlot(slotId): - trace "Adding slot", slotId + trace "Adding slot [historical]", slotId validation.slots.incl(slotId) + trace "Removing slots that have ended..." await removeSlotsThatHaveEnded(validation) + trace "Historical state restored", numberOfSlots = validation.slots.len proc start*(validation: Validation) {.async.} = validation.periodicity = await validation.market.periodicity() From b8b2b4959d7250eb9346636946c32e3146ae719a Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 9 Oct 2024 03:52:27 +0200 Subject: [PATCH 06/71] adds market tests for querying past SlotFilled events and binary search --- codex/contracts/market.nim | 45 +++++---- tests/contracts/testMarket.nim | 169 +++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 20 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index bd4ce8587..49cb6c849 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -468,12 +468,12 @@ method subscribeProofSubmission*(market: OnChainMarket, method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() -proc blockNumberForBlocksEgo(provider: Provider, +proc blockNumberForBlocksEgo*(provider: Provider, blocksAgo: int): Future[BlockTag] {.async.} = let head = await provider.getBlockNumber() return BlockTag.init(head - blocksAgo.abs.u256) -proc blockNumberAndTimestamp(provider: Provider, blockTag: BlockTag): +proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag): Future[(UInt256, UInt256)] {.async.} = without latestBlock =? await provider.getBlock(blockTag), error: raise error @@ -483,39 +483,41 @@ proc blockNumberAndTimestamp(provider: Provider, blockTag: BlockTag): (latestBlockNumber, latestBlock.timestamp) -proc estimateAverageBlockTime(provider: Provider): Future[UInt256] {.async.} = +proc estimateAverageBlockTime*(provider: Provider): Future[UInt256] {.async.} = let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) let (_, previousBlockTimestamp) = await provider.blockNumberAndTimestamp( BlockTag.init(latestBlockNumber - 1.u256)) - trace "[estimateAverageBlockTime]:", latestBlockNumber = latestBlockNumber, + debug "[estimateAverageBlockTime]:", latestBlockNumber = latestBlockNumber, latestBlockTimestamp = latestBlockTimestamp, previousBlockTimestamp = previousBlockTimestamp return latestBlockTimestamp - previousBlockTimestamp -proc binarySearchFindClosestBlock(provider: Provider, +proc binarySearchFindClosestBlock*(provider: Provider, epochTime: int, - low: BlockTag, - high: BlockTag): Future[BlockTag] {.async.} = + low: UInt256, + high: UInt256): Future[UInt256] {.async.} = let (_, lowTimestamp) = - await provider.blockNumberAndTimestamp(low) + await provider.blockNumberAndTimestamp(BlockTag.init(low)) let (_, highTimestamp) = - await provider.blockNumberAndTimestamp(high) + await provider.blockNumberAndTimestamp(BlockTag.init(high)) + trace "[binarySearchFindClosestBlock]:", epochTime = epochTime, + lowTimestamp = lowTimestamp, highTimestamp = highTimestamp, low = low, high = high if abs(lowTimestamp.truncate(int) - epochTime) < abs(highTimestamp.truncate(int) - epochTime): return low else: return high -proc binarySearchBlockNumberForEpoch(provider: Provider, +proc binarySearchBlockNumberForEpoch*(provider: Provider, epochTime: UInt256, latestBlockNumber: UInt256): - Future[BlockTag] {.async.} = + Future[UInt256] {.async.} = var low = 0.u256 var high = latestBlockNumber - trace "[binarySearchBlockNumberForEpoch]:", low = low, high = high + debug "[binarySearchBlockNumberForEpoch]:", low = low, high = high while low <= high: let mid = (low + high) div 2.u256 let (midBlockNumber, midBlockTimestamp) = @@ -526,26 +528,29 @@ proc binarySearchBlockNumberForEpoch(provider: Provider, elif midBlockTimestamp > epochTime: high = mid - 1.u256 else: - return BlockTag.init(midBlockNumber) + return midBlockNumber + # NOTICE that by how the binaty search is implemented, when it finishes + # low is always greater than high - this is why we return high, where + # intuitively we would return low. await provider.binarySearchFindClosestBlock( - epochTime.truncate(int), BlockTag.init(low), BlockTag.init(high)) + epochTime.truncate(int), low=high, high=low) -proc blockNumberForEpoch(provider: Provider, epochTime: int64): Future[BlockTag] +proc blockNumberForEpoch*(provider: Provider, epochTime: int64): Future[UInt256] {.async.} = let avgBlockTime = await provider.estimateAverageBlockTime() - trace "[blockNumberForEpoch]:", avgBlockTime = avgBlockTime + debug "[blockNumberForEpoch]:", avgBlockTime = avgBlockTime let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) - trace "[blockNumberForEpoch]:", latestBlockNumber = latestBlockNumber, + debug "[blockNumberForEpoch]:", latestBlockNumber = latestBlockNumber, latestBlockTimestamp = latestBlockTimestamp let timeDiff = latestBlockTimestamp - epochTimeUInt256 let blockDiff = timeDiff div avgBlockTime if blockDiff >= latestBlockNumber: - return BlockTag.earliest + return 0.u256 return await provider.binarySearchBlockNumberForEpoch( epochTimeUInt256, latestBlockNumber) @@ -576,8 +581,8 @@ method queryPastSlotFilledEvents*( convertEthersError: let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime) - trace "queryPastSlotFilledEvents fromTime", fromTime=fromTime, fromBlock=fromBlock - return await market.queryPastSlotFilledEvents(fromBlock) + debug "[queryPastSlotFilledEvents]", fromTime=fromTime, fromBlock=parseHexInt($fromBlock) + return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock)) method queryPastStorageRequestedEvents*( market: OnChainMarket, diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index e37f64119..423f141f6 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -10,6 +10,12 @@ import ./deployment privateAccess(OnChainMarket) # enable access to private fields +# to see supportive information in the test output +# use `-d:"chronicles_enabled_topics:testMarket:DEBUG` option +# when compiling the test file +logScope: + topics = "testMarket" + ethersuite "On-Chain Market": let proof = Groth16Proof.example @@ -57,6 +63,10 @@ ethersuite "On-Chain Market": proc advanceToCancelledRequest(request: StorageRequest) {.async.} = let expiry = (await market.requestExpiresAt(request.id)) + 1 await ethProvider.advanceTimeTo(expiry.u256) + + proc mineNBlocks(provider: JsonRpcProvider, n: int) {.async.} = + for _ in 0.. 291 + # 1728436104 => 291 + # 1728436105 => 292 + # 1728436106 => 292 + # 1728436110 => 292 + proc generateExpectations( + blocks: seq[(UInt256, UInt256)]): seq[Expectations] = + var expectations: seq[Expectations] = @[] + for i in 0.. Date: Wed, 9 Oct 2024 18:55:10 +0200 Subject: [PATCH 07/71] Takes into account that <> block available is not necessarily the genesis block --- codex/contracts/market.nim | 17 +++++++++++------ tests/contracts/testMarket.nim | 20 ++++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 49cb6c849..bfd606cbf 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -502,7 +502,7 @@ proc binarySearchFindClosestBlock*(provider: Provider, await provider.blockNumberAndTimestamp(BlockTag.init(low)) let (_, highTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(high)) - trace "[binarySearchFindClosestBlock]:", epochTime = epochTime, + debug "[binarySearchFindClosestBlock]:", epochTime = epochTime, lowTimestamp = lowTimestamp, highTimestamp = highTimestamp, low = low, high = high if abs(lowTimestamp.truncate(int) - epochTime) < abs(highTimestamp.truncate(int) - epochTime): @@ -512,9 +512,10 @@ proc binarySearchFindClosestBlock*(provider: Provider, proc binarySearchBlockNumberForEpoch*(provider: Provider, epochTime: UInt256, - latestBlockNumber: UInt256): + latestBlockNumber: UInt256, + earliestBlockNumber: UInt256): Future[UInt256] {.async.} = - var low = 0.u256 + var low = earliestBlockNumber var high = latestBlockNumber debug "[binarySearchBlockNumberForEpoch]:", low = low, high = high @@ -542,18 +543,22 @@ proc blockNumberForEpoch*(provider: Provider, epochTime: int64): Future[UInt256] let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) + let (earliestBlockNumber, earliestBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.earliest) debug "[blockNumberForEpoch]:", latestBlockNumber = latestBlockNumber, latestBlockTimestamp = latestBlockTimestamp + debug "[blockNumberForEpoch]:", earliestBlockNumber = earliestBlockNumber, + earliestBlockTimestamp = earliestBlockTimestamp let timeDiff = latestBlockTimestamp - epochTimeUInt256 let blockDiff = timeDiff div avgBlockTime - if blockDiff >= latestBlockNumber: - return 0.u256 + if blockDiff >= latestBlockNumber - earliestBlockNumber: + return earliestBlockNumber return await provider.binarySearchBlockNumberForEpoch( - epochTimeUInt256, latestBlockNumber) + epochTimeUInt256, latestBlockNumber, earliestBlockNumber) method queryPastSlotFilledEvents*( market: OnChainMarket, diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 423f141f6..4de6f4d0c 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -511,17 +511,25 @@ ethersuite "On-Chain Market": check expected == simulatedBlockTime check actual == expected - test "blockNumberForEpoch returns the earliest block when block height " & - "is less than the given epoch time": - let (_, timestampEarliest) = + test "blockNumberForEpoch returns the earliest block when retained history " & + "is shorter than the given epoch time": + # create predictable conditions for computing average block time + let averageBlockTime = 10.u256 + await ethProvider.mineNBlocks(1) + await ethProvider.advanceTime(averageBlockTime) + let (earliestBlockNumber, earliestTimestamp) = await ethProvider.blockNumberAndTimestamp(BlockTag.earliest) - let fromTime = timestampEarliest - 1 + let fromTime = earliestTimestamp - 1 - let expected = await ethProvider.blockNumberForEpoch( + let actual = await ethProvider.blockNumberForEpoch( fromTime.truncate(int64)) - check expected == 0.u256 + # Notice this could fail in a network where "earliest" block is + # not the genesis block - we run the tests agains local network + # so we know the earliest block is the same as genesis block + # earliestBlockNumber is 0.u256 in our case. + check actual == earliestBlockNumber test "blockNumberForEpoch finds closest blockNumber for given epoch time": proc createBlockHistory(n: int, blockTime: int): From e462511d417806a92bc1e5010a97baae49d13f63 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 10 Oct 2024 06:22:10 +0200 Subject: [PATCH 08/71] Adds more logging and makes testing earliest block boundary more reliable --- codex/contracts/market.nim | 21 ++++++++++++--------- codex/market.nim | 2 +- tests/contracts/testMarket.nim | 18 ++++++++---------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index bfd606cbf..50a9b8cb2 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,7 +1,4 @@ -# import std/sequtils import std/strutils -import std/strformat -# import std/sugar import pkg/ethers import pkg/upraises import pkg/questionable @@ -520,14 +517,17 @@ proc binarySearchBlockNumberForEpoch*(provider: Provider, debug "[binarySearchBlockNumberForEpoch]:", low = low, high = high while low <= high: - let mid = (low + high) div 2.u256 + if low == 0 and high == 0: + return low + let mid = (low + high) div 2 + debug "[binarySearchBlockNumberForEpoch]:", low = low, mid = mid, high = high let (midBlockNumber, midBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(mid)) if midBlockTimestamp < epochTime: - low = mid + 1.u256 + low = mid + 1 elif midBlockTimestamp > epochTime: - high = mid - 1.u256 + high = mid - 1 else: return midBlockNumber # NOTICE that by how the binaty search is implemented, when it finishes @@ -536,10 +536,11 @@ proc binarySearchBlockNumberForEpoch*(provider: Provider, await provider.binarySearchFindClosestBlock( epochTime.truncate(int), low=high, high=low) -proc blockNumberForEpoch*(provider: Provider, epochTime: int64): Future[UInt256] - {.async.} = +proc blockNumberForEpoch*(provider: Provider, + epochTime: SecondsSince1970): Future[UInt256] {.async.} = let avgBlockTime = await provider.estimateAverageBlockTime() debug "[blockNumberForEpoch]:", avgBlockTime = avgBlockTime + debug "[blockNumberForEpoch]:", epochTime = epochTime let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) @@ -554,6 +555,8 @@ proc blockNumberForEpoch*(provider: Provider, epochTime: int64): Future[UInt256] let timeDiff = latestBlockTimestamp - epochTimeUInt256 let blockDiff = timeDiff div avgBlockTime + debug "[blockNumberForEpoch]:", timeDiff = timeDiff, blockDiff = blockDiff + if blockDiff >= latestBlockNumber - earliestBlockNumber: return earliestBlockNumber @@ -581,7 +584,7 @@ method queryPastSlotFilledEvents*( method queryPastSlotFilledEvents*( market: OnChainMarket, - fromTime: int64): Future[seq[SlotFilled]] {.async.} = + fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.async.} = convertEthersError: let fromBlock = diff --git a/codex/market.nim b/codex/market.nim index 0fbcbff78..3b2c1c406 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -268,5 +268,5 @@ method queryPastStorageRequestedEvents*( method queryPastSlotFilledEvents*( market: Market, - fromTime: int64): Future[seq[SlotFilled]] {.base, async.} = + fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.base, async.} = raiseAssert("not implemented") diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 4de6f4d0c..f8dea38aa 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -469,7 +469,7 @@ ethersuite "On-Chain Market": await market.fillSlot(request.id, 2.u256, proof, request.ask.collateral) let events = await market.queryPastSlotFilledEvents( - fromTime = fromTime.truncate(int64)) + fromTime = fromTime.truncate(SecondsSince1970)) check events == @[ SlotFilled(requestId: request.id, slotIndex: 1.u256), @@ -489,7 +489,7 @@ ethersuite "On-Chain Market": await ethProvider.blockNumberAndTimestamp(BlockTag.latest) let events = await market.queryPastSlotFilledEvents( - fromTime = fromTime.truncate(int64)) + fromTime = fromTime.truncate(SecondsSince1970)) check events.len == 0 @@ -513,8 +513,10 @@ ethersuite "On-Chain Market": test "blockNumberForEpoch returns the earliest block when retained history " & "is shorter than the given epoch time": - # create predictable conditions for computing average block time - let averageBlockTime = 10.u256 + # create predictable conditions + # we keep minimal resultion of 1s so that we are sure that + # we will land before the earliest (genesis in our case) block + let averageBlockTime = 1.u256 await ethProvider.mineNBlocks(1) await ethProvider.advanceTime(averageBlockTime) let (earliestBlockNumber, earliestTimestamp) = @@ -523,12 +525,8 @@ ethersuite "On-Chain Market": let fromTime = earliestTimestamp - 1 let actual = await ethProvider.blockNumberForEpoch( - fromTime.truncate(int64)) + fromTime.truncate(SecondsSince1970)) - # Notice this could fail in a network where "earliest" block is - # not the genesis block - we run the tests agains local network - # so we know the earliest block is the same as genesis block - # earliestBlockNumber is 0.u256 in our case. check actual == earliestBlockNumber test "blockNumberForEpoch finds closest blockNumber for given epoch time": @@ -614,7 +612,7 @@ ethersuite "On-Chain Market": debug "Validating", epochTime = epochTime, expectedBlockNumber = expectedBlockNumber let actualBlockNumber = await ethProvider.blockNumberForEpoch( - epochTime.truncate(int64)) + epochTime.truncate(SecondsSince1970)) check actualBlockNumber == expectedBlockNumber test "past event query can specify negative `blocksAgo` parameter": From 62d6935613e345f009dafd87eafbb3a4b210d2ef Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 10 Oct 2024 06:26:32 +0200 Subject: [PATCH 09/71] adds validation tests for historical state restoration --- codex/validation.nim | 11 +-- tests/codex/helpers/mockmarket.nim | 29 +++++-- tests/codex/testvalidation.nim | 120 +++++++++++++++++++++++------ 3 files changed, 126 insertions(+), 34 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index b245ebc28..6468e6a83 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -123,13 +123,13 @@ proc run(validation: Validation) {.async.} = except CatchableError as e: error "Validation failed", msg = e.msg -proc epochForDurationBackFromNow(duration: times.Duration): int64 = - let now = getTime().toUnix - return now - duration.inSeconds +proc epochForDurationBackFromNow(validation: Validation, + duration: times.Duration): SecondsSince1970 = + return validation.clock.now - duration.inSeconds proc restoreHistoricalState(validation: Validation) {.async} = trace "Restoring historical state..." - let startTimeEpoch = epochForDurationBackFromNow(MaxStorageRequestDuration) + let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) trace "Found slot filled events", numberOfSlots = slotFilledEvents.len @@ -150,7 +150,8 @@ proc start*(validation: Validation) {.async.} = validation.running = validation.run() proc stop*(validation: Validation) {.async.} = - await validation.running.cancelAndWait() + if not isNil(validation.running): + await validation.running.cancelAndWait() while validation.subscriptions.len > 0: let subscription = validation.subscriptions.pop() await subscription.unsubscribe() diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 6f393ca3e..25a021d83 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -10,12 +10,16 @@ import pkg/codex/contracts/proofs import pkg/codex/contracts/config from pkg/ethers import BlockTag +import codex/clock import ../examples export market export tables +logScope: + topics = "mockMarket" + type MockMarket* = ref object of Market periodicity: Periodicity @@ -43,6 +47,7 @@ type config*: MarketplaceConfig canReserveSlot*: bool reserveSlotThrowError*: ?(ref MarketError) + clock: ?Clock Fulfillment* = object requestId*: RequestId proof*: Groth16Proof @@ -52,6 +57,7 @@ type host*: Address slotIndex*: UInt256 proof*: Groth16Proof + timestamp: ?SecondsSince1970 Subscriptions = object onRequest: seq[RequestSubscription] onFulfillment: seq[FulfillmentSubscription] @@ -97,7 +103,7 @@ proc hash*(address: Address): Hash = proc hash*(requestId: RequestId): Hash = hash(requestId.toArray) -proc new*(_: type MockMarket): MockMarket = +proc new*(_: type MockMarket, clock: ?Clock = Clock.none): MockMarket = ## Create a new mocked Market instance ## let config = MarketplaceConfig( @@ -114,7 +120,8 @@ proc new*(_: type MockMarket): MockMarket = downtimeProduct: 67.uint8 ) ) - MockMarket(signer: Address.example, config: config, canReserveSlot: true) + MockMarket(signer: Address.example, config: config, + canReserveSlot: true, clock: clock) method getSigner*(market: MockMarket): Future[Address] {.async.} = return market.signer @@ -251,7 +258,8 @@ proc fillSlot*(market: MockMarket, requestId: requestId, slotIndex: slotIndex, proof: proof, - host: host + host: host, + timestamp: market.clock.?now ) market.filled.add(slot) market.slotState[slotId(slot.requestId, slot.slotIndex)] = SlotState.Filled @@ -509,8 +517,19 @@ method queryPastSlotFilledEvents*( method queryPastSlotFilledEvents*( market: MockMarket, - fromTime: int64): Future[seq[SlotFilled]] {.async.} = - return market.filled.map(slot => + fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.async.} = + debug "queryPastSlotFilledEvents:market.filled", + numOfFilledSlots = market.filled.len + let filtered = market.filled.filter( + proc (slot: MockSlot): bool = + debug "queryPastSlotFilledEvents:fromTime", timestamp = slot.timestamp, + fromTime = fromTime + if timestamp =? slot.timestamp: + return timestamp >= fromTime + else: + true + ) + return filtered.map(slot => SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) ) diff --git a/tests/codex/testvalidation.nim b/tests/codex/testvalidation.nim index 7988269ba..87671fa51 100644 --- a/tests/codex/testvalidation.nim +++ b/tests/codex/testvalidation.nim @@ -1,9 +1,10 @@ import pkg/chronos import std/strformat -import std/random +import std/times import codex/validation import codex/periods +import codex/clock import ../asynctest import ./helpers/mockmarket @@ -11,6 +12,9 @@ import ./helpers/mockclock import ./examples import ./helpers +logScope: + topics = "testValidation" + asyncchecksuite "validation": let period = 10 let timeout = 5 @@ -20,10 +24,10 @@ asyncchecksuite "validation": let proof = Groth16Proof.example let collateral = slot.request.ask.collateral - var validation: Validation var market: MockMarket var clock: MockClock var groupIndex: uint16 + var validation: Validation proc initValidationConfig(maxSlots: MaxSlots, validationGroups: ?ValidationGroups, @@ -32,19 +36,27 @@ asyncchecksuite "validation": maxSlots, groups=validationGroups, groupIndex), error: raiseAssert fmt"Creating ValidationConfig failed! Error msg: {error.msg}" validationConfig + + proc newValidation(clock: Clock, + market: Market, + maxSlots: MaxSlots, + validationGroups: ?ValidationGroups, + groupIndex: uint16 = 0): Validation = + let validationConfig = initValidationConfig( + maxSlots, validationGroups, groupIndex) + Validation.new(clock, market, validationConfig) setup: groupIndex = groupIndexForSlotId(slot.id, !validationGroups) - market = MockMarket.new() clock = MockClock.new() - let validationConfig = initValidationConfig( - maxSlots, validationGroups, groupIndex) - validation = Validation.new(clock, market, validationConfig) + market = MockMarket.new(clock = Clock(clock).some) market.config.proofs.period = period.u256 market.config.proofs.timeout = timeout.u256 - await validation.start() + validation = newValidation( + clock, market, maxSlots, validationGroups, groupIndex) teardown: + # calling stop on validation that did not start is harmless await validation.stop() proc advanceToNextPeriod = @@ -79,6 +91,7 @@ asyncchecksuite "validation": test "initializing ValidationConfig fails when maxSlots is negative " & "(validationGroups set)": let maxSlots = -1 + let groupIndex = 0'u16 let validationConfig = ValidationConfig.init( maxSlots = maxSlots, groups = validationGroups, groupIndex) check validationConfig.isFailure == true @@ -86,45 +99,41 @@ asyncchecksuite "validation": fmt"be greater than or equal to 0! (got: {maxSlots})" test "slot is not observed if it is not in the validation group": - let validationConfig = initValidationConfig(maxSlots, validationGroups, - (groupIndex + 1) mod uint16(!validationGroups)) - let validation = Validation.new(clock, market, validationConfig) + validation = newValidation(clock, market, maxSlots, validationGroups, + (groupIndex + 1) mod uint16(!validationGroups)) await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - await validation.stop() check validation.slots.len == 0 test "when a slot is filled on chain, it is added to the list": + await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) check validation.slots == @[slot.id] test "slot should be observed if maxSlots is set to 0": - let validationConfig = initValidationConfig( - maxSlots = 0, ValidationGroups.none) - let validation = Validation.new(clock, market, validationConfig) + validation = newValidation(clock, market, maxSlots = 0, ValidationGroups.none) await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - await validation.stop() check validation.slots == @[slot.id] test "slot should be observed if validation group is not set (and " & "maxSlots is not 0)": - let validationConfig = initValidationConfig( - maxSlots, ValidationGroups.none) - let validation = Validation.new(clock, market, validationConfig) + validation = newValidation(clock, market, maxSlots, ValidationGroups.none) await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - await validation.stop() check validation.slots == @[slot.id] for state in [SlotState.Finished, SlotState.Failed]: test fmt"when slot state changes to {state}, it is removed from the list": + validation = newValidation(clock, market, maxSlots, validationGroups) + await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) market.slotState[slot.id] = state advanceToNextPeriod() check eventually validation.slots.len == 0 test "when a proof is missed, it is marked as missing": + await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) market.setCanProofBeMarkedAsMissing(slot.id, true) advanceToNextPeriod() @@ -132,6 +141,7 @@ asyncchecksuite "validation": check market.markedAsMissingProofs.contains(slot.id) test "when a proof can not be marked as missing, it will not be marked": + await validation.start() await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) market.setCanProofBeMarkedAsMissing(slot.id, false) advanceToNextPeriod() @@ -139,13 +149,75 @@ asyncchecksuite "validation": check market.markedAsMissingProofs.len == 0 test "it does not monitor more than the maximum number of slots": - let validationGroups = ValidationGroups.none - let validationConfig = initValidationConfig( - maxSlots, validationGroups) - let validation = Validation.new(clock, market, validationConfig) + validation = newValidation(clock, market, maxSlots, ValidationGroups.none) await validation.start() for _ in 0.. Date: Fri, 11 Oct 2024 04:04:16 +0200 Subject: [PATCH 10/71] adds mockprovider to simplify and improve testing of the edge conditions --- codex/contracts/market.nim | 59 ++++++++++++------ tests/contracts/helpers/mockprovider.nim | 79 ++++++++++++++++++++++++ tests/contracts/testMarket.nim | 76 +++++++++++++++-------- 3 files changed, 169 insertions(+), 45 deletions(-) create mode 100644 tests/contracts/helpers/mockprovider.nim diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 50a9b8cb2..63a37c034 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,4 +1,5 @@ import std/strutils +import std/times import pkg/ethers import pkg/upraises import pkg/questionable @@ -480,17 +481,6 @@ proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag): (latestBlockNumber, latestBlock.timestamp) -proc estimateAverageBlockTime*(provider: Provider): Future[UInt256] {.async.} = - let (latestBlockNumber, latestBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.latest) - let (_, previousBlockTimestamp) = - await provider.blockNumberAndTimestamp( - BlockTag.init(latestBlockNumber - 1.u256)) - debug "[estimateAverageBlockTime]:", latestBlockNumber = latestBlockNumber, - latestBlockTimestamp = latestBlockTimestamp, - previousBlockTimestamp = previousBlockTimestamp - return latestBlockTimestamp - previousBlockTimestamp - proc binarySearchFindClosestBlock*(provider: Provider, epochTime: int, low: UInt256, @@ -538,8 +528,6 @@ proc binarySearchBlockNumberForEpoch*(provider: Provider, proc blockNumberForEpoch*(provider: Provider, epochTime: SecondsSince1970): Future[UInt256] {.async.} = - let avgBlockTime = await provider.estimateAverageBlockTime() - debug "[blockNumberForEpoch]:", avgBlockTime = avgBlockTime debug "[blockNumberForEpoch]:", epochTime = epochTime let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = @@ -552,12 +540,45 @@ proc blockNumberForEpoch*(provider: Provider, debug "[blockNumberForEpoch]:", earliestBlockNumber = earliestBlockNumber, earliestBlockTimestamp = earliestBlockTimestamp - let timeDiff = latestBlockTimestamp - epochTimeUInt256 - let blockDiff = timeDiff div avgBlockTime - - debug "[blockNumberForEpoch]:", timeDiff = timeDiff, blockDiff = blockDiff - - if blockDiff >= latestBlockNumber - earliestBlockNumber: + # Initially we used the average block time to predict + # the number of blocks we need to look back in order to find + # the block number corresponding to the given epoch time. + # This estimation can be highly inaccurate if block time + # was changing in the past or is fluctuating and therefore + # we used that information initially only to find out + # if the available history is long enough to perform effective search. + # It turns out we do not have to do that. There is an easier way. + # + # First we check if the given epoch time equals the timestamp of either + # the earliest or the latest block. If it does, we just return the + # block number of that block. + # + # Otherwise, if the earliest available block is not the genesis block, + # we should check the timestamp of that earliest block and if it is greater + # than the epoch time, we should issue a warning and return + # that earliest block number. + # In all other cases, thus when the earliest block is not the genesis + # block but its timestamp is not greater than the requested epoch time, or + # if the earliest available block is the genesis block, + # (which means we have the whole history available), we should proceed with + # the binary search. + # + # Additional benefit of this method is that we do not have to rely + # on the average block time, which not only makes the whole thing + # more reliable, but also easier to test. + + # Are lucky today? + if earliestBlockTimestamp == epochTimeUInt256: + return earliestBlockNumber + if latestBlockTimestamp == epochTimeUInt256: + return latestBlockNumber + + if earliestBlockNumber > 0 and earliestBlockTimestamp > epochTimeUInt256: + let availableHistoryInDays = + (latestBlockTimestamp - earliestBlockTimestamp) div + initDuration(days = 1).inSeconds.u256 + warn "Short block history detected.", earliestBlockTimestamp = + earliestBlockTimestamp, days = availableHistoryInDays return earliestBlockNumber return await provider.binarySearchBlockNumberForEpoch( diff --git a/tests/contracts/helpers/mockprovider.nim b/tests/contracts/helpers/mockprovider.nim new file mode 100644 index 000000000..1ef826353 --- /dev/null +++ b/tests/contracts/helpers/mockprovider.nim @@ -0,0 +1,79 @@ +import std/strutils +import std/tables + +import pkg/ethers/provider +from codex/clock import SecondsSince1970 + +export provider.Block + +type MockProvider* = ref object of Provider + blocks: OrderedTableRef[int, Block] + earliest: ?int + latest: ?int + +method getBlock*( + provider: MockProvider, + tag: BlockTag +): Future[?Block] {.async.} = + if $tag == "latest": + if latestBlock =? provider.latest: + if provider.blocks.hasKey(latestBlock): + return provider.blocks[latestBlock].some + elif $tag == "earliest": + if earliestBlock =? provider.earliest: + if provider.blocks.hasKey(earliestBlock): + return provider.blocks[earliestBlock].some + else: + let blockNumber = parseHexInt($tag) + if provider.blocks.hasKey(blockNumber): + return provider.blocks[blockNumber].some + return Block.none + +proc updateEarliestAndLatest(provider: MockProvider, blockNumber: int) = + if provider.earliest.isNone: + provider.earliest = blockNumber.some + provider.latest = blockNumber.some + +proc addBlocks*(provider: MockProvider, blocks: OrderedTableRef[int, Block]) = + for number, blk in blocks.pairs: + if provider.blocks.hasKey(number): + continue + provider.updateEarliestAndLatest(number) + provider.blocks[number] = blk + +proc addBlock*(provider: MockProvider, number: int, blk: Block) = + if not provider.blocks.hasKey(number): + provider.updateEarliestAndLatest(number) + provider.blocks[number] = blk + +proc newMockProvider*(): MockProvider = + MockProvider( + blocks: newOrderedTable[int, Block](), + earliest: int.none, + latest: int.none + ) + +proc newMockProvider*(blocks: OrderedTableRef[int, Block]): MockProvider = + let provider = newMockProvider() + provider.addBlocks(blocks) + provider + +proc newMockProvider*( + numberOfBlocks: int, + earliestBlockNumber: int, + earliestBlockTimestamp: SecondsSince1970, + timeIntervalBetweenBlocks: SecondsSince1970 +): MockProvider = + var blocks = newOrderedTable[int, provider.Block]() + var blockNumber = earliestBlockNumber + var blockTime = earliestBlockTimestamp + for i in 0.. Date: Fri, 11 Oct 2024 04:19:33 +0200 Subject: [PATCH 11/71] adds slot reservation to the new tests after rebasing --- tests/contracts/testMarket.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 7aff1a1cb..987786b36 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -453,6 +453,7 @@ ethersuite "On-Chain Market": test "can query past SlotFilled events since given timestamp": await market.requestStorage(request) + await market.reserveSlot(request.id, 0.u256) await market.fillSlot(request.id, 0.u256, proof, request.ask.collateral) # The SlotFilled event will be included in the same block as @@ -466,6 +467,8 @@ ethersuite "On-Chain Market": let (_, fromTime) = await ethProvider.blockNumberAndTimestamp(BlockTag.latest) + await market.reserveSlot(request.id, 1.u256) + await market.reserveSlot(request.id, 2.u256) await market.fillSlot(request.id, 1.u256, proof, request.ask.collateral) await market.fillSlot(request.id, 2.u256, proof, request.ask.collateral) @@ -480,6 +483,9 @@ ethersuite "On-Chain Market": test "queryPastSlotFilledEvents returns empty sequence of events when " & "no SlotFilled events have occurred since given timestamp": await market.requestStorage(request) + await market.reserveSlot(request.id, 0.u256) + await market.reserveSlot(request.id, 1.u256) + await market.reserveSlot(request.id, 2.u256) await market.fillSlot(request.id, 0.u256, proof, request.ask.collateral) await market.fillSlot(request.id, 1.u256, proof, request.ask.collateral) await market.fillSlot(request.id, 2.u256, proof, request.ask.collateral) From 7bd76aca36ce4cd6b408aa01b17fb8244a7a6de1 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 01:49:39 +0200 Subject: [PATCH 12/71] adds validation groups and group index in logs of validator --- codex/validation.nim | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index 6468e6a83..ae9a27370 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -47,7 +47,8 @@ proc getCurrentPeriod(validation: Validation): UInt256 = proc waitUntilNextPeriod(validation: Validation) {.async.} = let period = validation.getCurrentPeriod() let periodEnd = validation.periodicity.periodEnd(period) - trace "Waiting until next period", currentPeriod = period + trace "Waiting until next period", currentPeriod = period, groups = validation.config.groups, + groupIndex = validation.config.groupIndex await validation.clock.waitUntil(periodEnd.truncate(int64) + 1) func groupIndexForSlotId*(slotId: SlotId, @@ -71,7 +72,8 @@ proc subscribeSlotFilled(validation: Validation) {.async.} = proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) = let slotId = slotId(requestId, slotIndex) if validation.shouldValidateSlot(slotId): - trace "Adding slot", slotId + trace "Adding slot", slotId, groups = validation.config.groups, + groupIndex = validation.config.groupIndex validation.slots.incl(slotId) let subscription = await validation.market.subscribeSlotFilled(onSlotFilled) validation.subscriptions.add(subscription) @@ -82,7 +84,8 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = for slotId in slots: let state = await validation.market.slotState(slotId) if state != SlotState.Filled: - trace "Removing slot", slotId + trace "Removing slot", slotId, groups = validation.config.groups, + groupIndex = validation.config.groupIndex ended.incl(slotId) validation.slots.excl(ended) @@ -94,11 +97,15 @@ proc markProofAsMissing(validation: Validation, try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): - trace "Marking proof as missing", slotId, periodProofMissed = period + trace "Marking proof as missing", slotId, periodProofMissed = period, + groups = validation.config.groups, + groupIndex = validation.config.groupIndex await validation.market.markProofAsMissing(slotId, period) else: let inDowntime {.used.} = await validation.market.inDowntime(slotId) - trace "Proof not missing", checkedPeriod = period, inDowntime + trace "Proof not missing", checkedPeriod = period, inDowntime, + groups = validation.config.groups, + groupIndex = validation.config.groupIndex except CancelledError: raise except CatchableError as e: @@ -111,36 +118,47 @@ proc markProofsAsMissing(validation: Validation) {.async.} = await validation.markProofAsMissing(slotId, previousPeriod) proc run(validation: Validation) {.async.} = - trace "Validation started" + trace "Validation started", groups = validation.config.groups, + groupIndex = validation.config.groupIndex try: while true: await validation.waitUntilNextPeriod() await validation.removeSlotsThatHaveEnded() await validation.markProofsAsMissing() except CancelledError: - trace "Validation stopped" + trace "Validation stopped", groups = validation.config.groups, + groupIndex = validation.config.groupIndex discard except CatchableError as e: - error "Validation failed", msg = e.msg + error "Validation failed", msg = e.msg, groups = validation.config.groups, + groupIndex = validation.config.groupIndex proc epochForDurationBackFromNow(validation: Validation, duration: times.Duration): SecondsSince1970 = return validation.clock.now - duration.inSeconds proc restoreHistoricalState(validation: Validation) {.async} = - trace "Restoring historical state..." + trace "Restoring historical state...", groups = validation.config.groups, + groupIndex = validation.config.groupIndex let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) - trace "Found slot filled events", numberOfSlots = slotFilledEvents.len + trace "Found slot filled events", numberOfSlots = slotFilledEvents.len, + groups = validation.config.groups, + groupIndex = validation.config.groupIndex for event in slotFilledEvents: let slotId = slotId(event.requestId, event.slotIndex) if validation.shouldValidateSlot(slotId): - trace "Adding slot [historical]", slotId + trace "Adding slot [historical]", slotId, + groups = validation.config.groups, + groupIndex = validation.config.groupIndex validation.slots.incl(slotId) - trace "Removing slots that have ended..." + trace "Removing slots that have ended...", groups = validation.config.groups, + groupIndex = validation.config.groupIndex await removeSlotsThatHaveEnded(validation) - trace "Historical state restored", numberOfSlots = validation.slots.len + trace "Historical state restored", numberOfSlots = validation.slots.len, + groups = validation.config.groups, + groupIndex = validation.config.groupIndex proc start*(validation: Validation) {.async.} = validation.periodicity = await validation.market.periodicity() From d369ebc963290d57153538573599b2d9c2554951 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 01:50:56 +0200 Subject: [PATCH 13/71] adds integration test with two validators --- tests/integration/codexconfig.nim | 22 ++++++++ tests/integration/testvalidator.nim | 79 +++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/integration/testvalidator.nim diff --git a/tests/integration/codexconfig.nim b/tests/integration/codexconfig.nim index 5f15331b0..d4c917740 100644 --- a/tests/integration/codexconfig.nim +++ b/tests/integration/codexconfig.nim @@ -239,6 +239,28 @@ proc withSimulateProofFailures*( StartUpCmd.persistence, "--simulate-proof-failures", $failEveryNProofs) return startConfig +proc withValidationGroups*( + self: CodexConfigs, + groups: ValidationGroups): CodexConfigs {.raises: [CodexConfigError].} = + + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption( + StartUpCmd.persistence, "--validator-groups", $(groups)) + return startConfig + +proc withValidationGroupIndex*( + self: CodexConfigs, + idx: int, + groupIndex: uint16): CodexConfigs {.raises: [CodexConfigError].} = + + self.checkBounds idx + + var startConfig = self + startConfig.configs[idx].addCliOption( + StartUpCmd.persistence, "--validator-group-index", $groupIndex) + return startConfig + proc logLevelWithTopics( config: CodexConfig, topics: varargs[string]): string {.raises: [CodexConfigError].} = diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim new file mode 100644 index 000000000..62f551603 --- /dev/null +++ b/tests/integration/testvalidator.nim @@ -0,0 +1,79 @@ +from std/times import inMilliseconds +import pkg/codex/logutils +import ../contracts/time +import ../contracts/deployment +import ../codex/helpers +import ../examples +import ./marketplacesuite +import ./nodeconfigs + +export logutils + +logScope: + topics = "integration test validation" + + +marketplacesuite "Validator marks proofs as missed when using validation groups": + + test "slot is freed after too many invalid proofs submitted", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some, + + validators: + CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("validator", "integration") + # .withLogTopics("validator", "integration", "ethers", "clock") + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration = expiry + 10.periods + + let data = await RandomChunker.example(blocks=8) + createAvailabilities(data.len * 2, duration) # TODO: better value for data.len + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=3, + tolerance=1, + proofProbability=1 + ) + let requestId = client0.requestId(purchaseId).get + + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) + + var slotWasFreed = false + proc onSlotFreed(event: SlotFreed) = + if event.requestId == requestId: + slotWasFreed = true + + let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + + check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000) + + await subscription.unsubscribe() From 6a61f0c604e290093aeb9c264508d9de6ebb34b5 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 02:34:09 +0200 Subject: [PATCH 14/71] adds comment on how to enable logging in integration test executable itself --- build.nims | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.nims b/build.nims index 3d1a3cac4..a9a0e5534 100644 --- a/build.nims +++ b/build.nims @@ -41,6 +41,9 @@ task testContracts, "Build & run Codex Contract tests": task testIntegration, "Run integration tests": buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true" test "testIntegration" + # use params to enable logging from the integration test executable + # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & + # "-d:chronicles_enabled_topics:integration:TRACE" task build, "build codex binary": codexTask() From 4d1efa72de4bddc4a4861fa5f7054c2340d86230 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 02:35:22 +0200 Subject: [PATCH 15/71] testIntegration: makes list is running nodes injected and available in the body of the test --- tests/integration/multinodes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 55d672f7f..983466ae0 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -62,7 +62,7 @@ template multinodesuite*(name: string, body: untyped) = asyncchecksuite name: - var running: seq[RunningNode] + var running {.inject, used.}: seq[RunningNode] var bootstrap: string let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") var currentTestName = "" From 5c48d0fbf3ef6233c0e30c53075c1c7f6aeec117 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 02:36:00 +0200 Subject: [PATCH 16/71] validation: adds integration test for historical state --- tests/integration/testvalidator.nim | 74 +++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 62f551603..8abd70fe6 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -12,10 +12,78 @@ export logutils logScope: topics = "integration test validation" +marketplacesuite "Validaton": -marketplacesuite "Validator marks proofs as missed when using validation groups": + test "validator uses historical state to mark missing proofs", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration = expiry + 10.periods + + let data = await RandomChunker.example(blocks=8) + createAvailabilities(data.len * 2, duration) # TODO: better value for data.len + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=3, + tolerance=1, + proofProbability=1 + ) + let requestId = client0.requestId(purchaseId).get + + check eventually(client0.purchaseStateIs(purchaseId, "started"), + timeout = expiry.int * 1000) + + var validators = CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument + + failAndTeardownOnError "failed to start validator nodes": + for config in validators.configs.mitems: + let node = await startValidatorNode(config) + running.add RunningNode( + role: Role.Validator, + node: node + ) + + var slotWasFreed = false + proc onSlotFreed(event: SlotFreed) = + if event.requestId == requestId: + slotWasFreed = true + + let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + + check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000) + + await subscription.unsubscribe() - test "slot is freed after too many invalid proofs submitted", NodeConfigs( + test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: HardhatConfig.none, @@ -42,7 +110,7 @@ marketplacesuite "Validator marks proofs as missed when using validation groups" .withValidationGroupIndex(idx = 1, groupIndex = 1) .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator", "integration") + .withLogTopics("validator") # .withLogTopics("validator", "integration", "ethers", "clock") .some ): From da88109dc672984f688deaadd3ba36ddc48e619a Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 06:31:07 +0200 Subject: [PATCH 17/71] adds more logging to validator --- codex/validation.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codex/validation.nim b/codex/validation.nim index ae9a27370..166de8e8b 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -119,7 +119,9 @@ proc markProofsAsMissing(validation: Validation) {.async.} = proc run(validation: Validation) {.async.} = trace "Validation started", groups = validation.config.groups, - groupIndex = validation.config.groupIndex + groupIndex = validation.config.groupIndex, + currentTime = validation.clock.now, + currentTime = validation.clock.now.fromUnix try: while true: await validation.waitUntilNextPeriod() From 11d6a76c5e017bdbe3eec855c432b2713bfea275 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 14 Oct 2024 06:32:53 +0200 Subject: [PATCH 18/71] integration test: validator only looks 30 days back for historical state --- tests/integration/testvalidator.nim | 114 +++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 8abd70fe6..d811c4229 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,4 +1,4 @@ -from std/times import inMilliseconds +from std/times import inMilliseconds, initDuration, inSeconds, fromUnix import pkg/codex/logutils import ../contracts/time import ../contracts/deployment @@ -82,6 +82,118 @@ marketplacesuite "Validaton": check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000) await subscription.unsubscribe() + + test "validator only looks 30 days back for historical state", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration30days = initDuration(days = 30) + let duration = expiry + duration30days.inSeconds.uint64 + 10.periods + + let data = await RandomChunker.example(blocks=8) + createAvailabilities(data.len * 2, duration) # TODO: better value for data.len + + let cid = client0.upload(data).get + + var currentTime = await ethProvider.currentTime() + + let expiryEndTime = currentTime.truncate(uint64) + expiry + let requestEndTime = currentTime.truncate(uint64) + duration + debug "test validator: ", currentTime = currentTime.truncate(uint64), + requestEndTime = requestEndTime, expiryEndTime = expiryEndTime + debug "test validator: ", currentTime = currentTime.truncate(int64).fromUnix, + requestEndTime = requestEndTime.int64.fromUnix, + expiryEndTime = expiryEndTime.int64.fromUnix + + proc onSlotFilled(event: SlotFilled) = + let slotId = slotId(event.requestId, event.slotIndex) + debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex, + slotId = slotId + + let subscriptionOnSlotFilled = await marketplace.subscribe(SlotFilled, onSlotFilled) + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=3, + tolerance=1, + reward=1.u256, + proofProbability=1 + ) + let requestId = client0.requestId(purchaseId).get + + check eventually(client0.purchaseStateIs(purchaseId, "started"), + timeout = expiry.int * 1000) + + currentTime = await ethProvider.currentTime() + var waitTime = (expiryEndTime - currentTime.truncate(uint64)).int.seconds + debug "test validation - waiting till end of expiry", waitTime = waitTime + await sleepAsync(waitTime) + + discard await ethProvider.send("evm_mine") + + await ethProvider.advanceTimeTo( + expiryEndTime.u256 + duration30days.inSeconds.u256) + debug "test validator[after advance]: ", currentTime = currentTime.truncate(SecondsSince1970) + debug "test validator[after advance]: ", currentTime = + currentTime.truncate(SecondsSince1970).fromUnix + + var validators = CodexConfigs.init(nodes=1) + .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("validator", "clock", "market") # each topic as a separate string argument + + failAndTeardownOnError "failed to start validator nodes": + for config in validators.configs.mitems: + let node = await startValidatorNode(config) + running.add RunningNode( + role: Role.Validator, + node: node + ) + + var slotWasFreed = false + proc onSlotFreed(event: SlotFreed) = + if event.requestId == requestId: + slotWasFreed = true + + let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + + # check not freed + currentTime = await ethProvider.currentTime() + if requestEndTime > currentTime.truncate(uint64): + waitTime = (requestEndTime - currentTime.truncate(uint64)).int.seconds + debug "test validation - waiting for request end", waitTime = waitTime + await sleepAsync(waitTime) + + debug "test validation - request ended" + + check not slotWasFreed + + # check eventually(client0.purchaseStateIs(purchaseId, "finished"), + # timeout = 60 * 1000) + + await subscription.unsubscribe() + await subscriptionOnSlotFilled.unsubscribe() + debug "test validation - unsubscribed" test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally From babf1fe8da6ef54058cd36722c8a2c9f37858f23 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 15 Oct 2024 00:27:00 +0200 Subject: [PATCH 19/71] adds logging of the slotState when removing slots during validation --- codex/validation.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex/validation.nim b/codex/validation.nim index 166de8e8b..eda92dab3 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -85,7 +85,7 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = let state = await validation.market.slotState(slotId) if state != SlotState.Filled: trace "Removing slot", slotId, groups = validation.config.groups, - groupIndex = validation.config.groupIndex + groupIndex = validation.config.groupIndex, slotState = state ended.incl(slotId) validation.slots.excl(ended) From 3ae86858d41f852ccc23ad939bbae6492776686d Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 15 Oct 2024 00:27:34 +0200 Subject: [PATCH 20/71] review and refactor validator integration tests --- tests/integration/testvalidator.nim | 262 ++++++++++++---------------- 1 file changed, 109 insertions(+), 153 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index d811c4229..20ced51cc 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,4 +1,5 @@ from std/times import inMilliseconds, initDuration, inSeconds, fromUnix +import std/sets import pkg/codex/logutils import ../contracts/time import ../contracts/deployment @@ -13,8 +14,51 @@ logScope: topics = "integration test validation" marketplacesuite "Validaton": + let nodes = 3 + let tolerance = 1 + var slotsFilled: seq[SlotId] + var slotsFreed: seq[SlotId] + + proc trackSlotsFilled(marketplace: Marketplace): + Future[provider.Subscription] {.async.} = + slotsFilled = newSeq[SlotId]() + proc onSlotFilled(event: SlotFilled) = + let slotId = slotId(event.requestId, event.slotIndex) + slotsFilled.add(slotId) + debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex, + slotId = slotId - test "validator uses historical state to mark missing proofs", NodeConfigs( + let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) + subscription + + proc trackSlotsFreed(requestId: RequestId, marketplace: Marketplace): + Future[provider.Subscription] {.async.} = + slotsFreed = newSeq[SlotId]() + proc onSlotFreed(event: SlotFreed) = + if event.requestId == requestId: + let slotId = slotId(event.requestId, event.slotIndex) + slotsFreed.add(slotId) + debug "onSlotFreed", requestId = requestId, slotIndex = event.slotIndex, + slotId = slotId, slotsFreed = slotsFreed.len + + let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + subscription + + proc checkSlotsFailed(slotsFilled: seq[SlotId], + slotsFreed: seq[SlotId], marketplace: Marketplace) {.async.} = + let slotsNotFreed = slotsFilled.filter( + slotId => not slotsFreed.contains(slotId) + ).toHashSet + var slotsFailed = initHashSet[SlotId]() + for slotId in slotsFilled: + let state = await marketplace.slotState(slotId) + if state == SlotState.Failed: + slotsFailed.incl(slotId) + + debug "slots failed", slotsFailed = slotsFailed, slotsNotFreed = slotsNotFreed + check slotsNotFreed == slotsFailed + + test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: HardhatConfig.none, @@ -32,6 +76,17 @@ marketplacesuite "Validaton": # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some, + + validators: + CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("validator") + # .withLogTopics("validator", "integration", "ethers", "clock") .some ): let client0 = clients()[0].client @@ -39,7 +94,12 @@ marketplacesuite "Validaton": let duration = expiry + 10.periods let data = await RandomChunker.example(blocks=8) - createAvailabilities(data.len * 2, duration) # TODO: better value for data.len + + # TODO: better value for data.len below. This TODO is also present in + # testproofs.nim - we may want to address it or remove the comment. + createAvailabilities(data.len * 2, duration) + + let slotFilledSubscription = await trackSlotsFilled(marketplace) let cid = client0.upload(data).get @@ -47,43 +107,31 @@ marketplacesuite "Validaton": cid, expiry=expiry, duration=duration, - nodes=3, - tolerance=1, + nodes=nodes, + tolerance=tolerance, proofProbability=1 ) let requestId = client0.requestId(purchaseId).get - check eventually(client0.purchaseStateIs(purchaseId, "started"), + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) + + let slotFreedSubscription = + await trackSlotsFreed(requestId, marketplace) - var validators = CodexConfigs.init(nodes=2) - .withValidationGroups(groups = 2) - .withValidationGroupIndex(idx = 0, groupIndex = 0) - .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator") # each topic as a separate string argument + let expectedSlotsFreed = nodes - tolerance + check eventually((slotsFreed.len == expectedSlotsFreed), + timeout=(duration - expiry).int * 1000) - failAndTeardownOnError "failed to start validator nodes": - for config in validators.configs.mitems: - let node = await startValidatorNode(config) - running.add RunningNode( - role: Role.Validator, - node: node - ) + # Because of erasure coding, if e.g. 2 out of 3 nodes are freed, the last + # node will not be freed but marked as "Failed" because the whole request + # will fail. For this reason we need an extra check: + await checkSlotsFailed(slotsFilled, slotsFreed, marketplace) - var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: - slotWasFreed = true - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000) - - await subscription.unsubscribe() + await slotFilledSubscription.unsubscribe() + await slotFreedSubscription.unsubscribe() - test "validator only looks 30 days back for historical state", NodeConfigs( + test "validator uses historical state to mark missing proofs", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: HardhatConfig.none, @@ -105,62 +153,43 @@ marketplacesuite "Validaton": ): let client0 = clients()[0].client let expiry = 5.periods - let duration30days = initDuration(days = 30) - let duration = expiry + duration30days.inSeconds.uint64 + 10.periods + let duration = expiry + 10.periods let data = await RandomChunker.example(blocks=8) - createAvailabilities(data.len * 2, duration) # TODO: better value for data.len - let cid = client0.upload(data).get + # TODO: better value for data.len below. This TODO is also present in + # testproofs.nim - we may want to address it or remove the comment. + createAvailabilities(data.len * 2, duration) - var currentTime = await ethProvider.currentTime() + let slotFilledSubscription = await trackSlotsFilled(marketplace) - let expiryEndTime = currentTime.truncate(uint64) + expiry - let requestEndTime = currentTime.truncate(uint64) + duration - debug "test validator: ", currentTime = currentTime.truncate(uint64), - requestEndTime = requestEndTime, expiryEndTime = expiryEndTime - debug "test validator: ", currentTime = currentTime.truncate(int64).fromUnix, - requestEndTime = requestEndTime.int64.fromUnix, - expiryEndTime = expiryEndTime.int64.fromUnix - - proc onSlotFilled(event: SlotFilled) = - let slotId = slotId(event.requestId, event.slotIndex) - debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex, - slotId = slotId + let cid = client0.upload(data).get - let subscriptionOnSlotFilled = await marketplace.subscribe(SlotFilled, onSlotFilled) - let purchaseId = await client0.requestStorage( cid, expiry=expiry, duration=duration, - nodes=3, - tolerance=1, - reward=1.u256, + nodes=nodes, + tolerance=tolerance, proofProbability=1 ) let requestId = client0.requestId(purchaseId).get check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) - - currentTime = await ethProvider.currentTime() - var waitTime = (expiryEndTime - currentTime.truncate(uint64)).int.seconds - debug "test validation - waiting till end of expiry", waitTime = waitTime - await sleepAsync(waitTime) - + + # just to make sure we have a mined block that separates us + # from the block containing the last SlotFilled event discard await ethProvider.send("evm_mine") - await ethProvider.advanceTimeTo( - expiryEndTime.u256 + duration30days.inSeconds.u256) - debug "test validator[after advance]: ", currentTime = currentTime.truncate(SecondsSince1970) - debug "test validator[after advance]: ", currentTime = - currentTime.truncate(SecondsSince1970).fromUnix - - var validators = CodexConfigs.init(nodes=1) - .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator", "clock", "market") # each topic as a separate string argument + var validators = CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to: + # tests/integration/logs/ //_.log + # .withLogTopics("validator") # each topic as a separate string argument failAndTeardownOnError "failed to start validator nodes": for config in validators.configs.mitems: @@ -170,90 +199,17 @@ marketplacesuite "Validaton": node: node ) - var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: - slotWasFreed = true - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + let slotFreedSubscription = + await trackSlotsFreed(requestId, marketplace) - # check not freed - currentTime = await ethProvider.currentTime() - if requestEndTime > currentTime.truncate(uint64): - waitTime = (requestEndTime - currentTime.truncate(uint64)).int.seconds - debug "test validation - waiting for request end", waitTime = waitTime - await sleepAsync(waitTime) + let expectedSlotsFreed = nodes - tolerance + check eventually((slotsFreed.len == expectedSlotsFreed), + timeout=(duration - expiry).int * 1000) - debug "test validation - request ended" - - check not slotWasFreed - - # check eventually(client0.purchaseStateIs(purchaseId, "finished"), - # timeout = 60 * 1000) - - await subscription.unsubscribe() - await subscriptionOnSlotFilled.unsubscribe() - debug "test validation - unsubscribed" - - test "validator marks proofs as missing when using validation groups", NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: - HardhatConfig.none, - - clients: - CodexConfigs.init(nodes=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - .some, - - providers: - CodexConfigs.init(nodes=1) - .withSimulateProofFailures(idx=0, failEveryNProofs=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") - .some, - - validators: - CodexConfigs.init(nodes=2) - .withValidationGroups(groups = 2) - .withValidationGroupIndex(idx = 0, groupIndex = 0) - .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator") - # .withLogTopics("validator", "integration", "ethers", "clock") - .some - ): - let client0 = clients()[0].client - let expiry = 5.periods - let duration = expiry + 10.periods - - let data = await RandomChunker.example(blocks=8) - createAvailabilities(data.len * 2, duration) # TODO: better value for data.len - - let cid = client0.upload(data).get - - let purchaseId = await client0.requestStorage( - cid, - expiry=expiry, - duration=duration, - nodes=3, - tolerance=1, - proofProbability=1 - ) - let requestId = client0.requestId(purchaseId).get - - check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) - - var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: - slotWasFreed = true - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000) - - await subscription.unsubscribe() + # Because of erasure coding, if e.g. 2 out of 3 nodes are freed, the last + # node will not be freed but marked as "Failed" because the whole request + # will fail. For this reason we need an extra check: + await checkSlotsFailed(slotsFilled, slotsFreed, marketplace) + + await slotFilledSubscription.unsubscribe() + await slotFreedSubscription.unsubscribe() From a0a43f1f208d3b7b8176165ad7508ce7eff2fbc4 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 15 Oct 2024 00:58:56 +0200 Subject: [PATCH 21/71] adds validation to the set of integration tests --- tests/testIntegration.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index b1f81ef40..f0a59ee45 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -6,6 +6,7 @@ import ./integration/testpurchasing import ./integration/testblockexpiration import ./integration/testmarketplace import ./integration/testproofs +import ./integration/testvalidator import ./integration/testecbug {.warning[UnusedImport]:off.} From 509af579a4b3d072f5d040154a77382e386183e9 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 15 Oct 2024 01:39:20 +0200 Subject: [PATCH 22/71] Fixes mistyped name of the mock provider module in testMarket --- tests/contracts/testMarket.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 987786b36..509a11e8a 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -7,7 +7,7 @@ import ../ethertest import ./examples import ./time import ./deployment -import ./helpers/mockProvider +import ./helpers/mockprovider privateAccess(OnChainMarket) # enable access to private fields From 9ab5c1c37409517fcf859d73e66e77e6a17d738d Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 15 Oct 2024 03:11:46 +0200 Subject: [PATCH 23/71] Fixes a typo in the name of the validation suite in integration tests --- tests/integration/testvalidator.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 20ced51cc..c5e6c2558 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -13,7 +13,7 @@ export logutils logScope: topics = "integration test validation" -marketplacesuite "Validaton": +marketplacesuite "Validation": let nodes = 3 let tolerance = 1 var slotsFilled: seq[SlotId] From 4fccdc407a2e2dc2070ffb4e57bc430c702a379f Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 15 Oct 2024 03:13:58 +0200 Subject: [PATCH 24/71] Makes validation unit test a bit easier to follow --- tests/codex/testvalidation.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/codex/testvalidation.nim b/tests/codex/testvalidation.nim index 87671fa51..4cfcd60c9 100644 --- a/tests/codex/testvalidation.nim +++ b/tests/codex/testvalidation.nim @@ -162,9 +162,9 @@ asyncchecksuite "validation": await market.fillSlot(earlySlot.request.id, earlySlot.slotIndex, proof, collateral) let fromTime = clock.now() clock.set(fromTime + 1) - let duration: times.Duration = initDuration(days = 30) await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) + let duration: times.Duration = initDuration(days = 30) clock.set(fromTime + duration.inSeconds + 1) validation = newValidation(clock, market, maxSlots = 0, From c31943e008066ce1036457661d07981b33c27d32 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 00:51:57 +0200 Subject: [PATCH 25/71] better use of logScopes to reduce duplication --- codex/validation.nim | 47 ++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index eda92dab3..2f39dbed8 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -79,13 +79,15 @@ proc subscribeSlotFilled(validation: Validation) {.async.} = validation.subscriptions.add(subscription) proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = + logScope: + groups = validation.config.groups + groupIndex = validation.config.groupIndex var ended: HashSet[SlotId] let slots = validation.slots for slotId in slots: let state = await validation.market.slotState(slotId) if state != SlotState.Filled: - trace "Removing slot", slotId, groups = validation.config.groups, - groupIndex = validation.config.groupIndex, slotState = state + trace "Removing slot", slotId, slotState = state ended.incl(slotId) validation.slots.excl(ended) @@ -94,18 +96,16 @@ proc markProofAsMissing(validation: Validation, period: Period) {.async.} = logScope: currentPeriod = validation.getCurrentPeriod() + groups = validation.config.groups + groupIndex = validation.config.groupIndex try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): - trace "Marking proof as missing", slotId, periodProofMissed = period, - groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Marking proof as missing", slotId, periodProofMissed = period await validation.market.markProofAsMissing(slotId, period) else: let inDowntime {.used.} = await validation.market.inDowntime(slotId) - trace "Proof not missing", checkedPeriod = period, inDowntime, - groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Proof not missing", checkedPeriod = period, inDowntime except CancelledError: raise except CatchableError as e: @@ -118,18 +118,18 @@ proc markProofsAsMissing(validation: Validation) {.async.} = await validation.markProofAsMissing(slotId, previousPeriod) proc run(validation: Validation) {.async.} = - trace "Validation started", groups = validation.config.groups, - groupIndex = validation.config.groupIndex, - currentTime = validation.clock.now, - currentTime = validation.clock.now.fromUnix + logScope: + groups = validation.config.groups + groupIndex = validation.config.groupIndex + trace "Validation started", currentTime = validation.clock.now, + currentTime = validation.clock.now.fromUnix try: while true: await validation.waitUntilNextPeriod() await validation.removeSlotsThatHaveEnded() await validation.markProofsAsMissing() except CancelledError: - trace "Validation stopped", groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Validation stopped" discard except CatchableError as e: error "Validation failed", msg = e.msg, groups = validation.config.groups, @@ -140,27 +140,22 @@ proc epochForDurationBackFromNow(validation: Validation, return validation.clock.now - duration.inSeconds proc restoreHistoricalState(validation: Validation) {.async} = - trace "Restoring historical state...", groups = validation.config.groups, + logScope: + groups = validation.config.groups groupIndex = validation.config.groupIndex + trace "Restoring historical state..." let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) - trace "Found slot filled events", numberOfSlots = slotFilledEvents.len, - groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Found slot filled events", numberOfSlots = slotFilledEvents.len for event in slotFilledEvents: let slotId = slotId(event.requestId, event.slotIndex) if validation.shouldValidateSlot(slotId): - trace "Adding slot [historical]", slotId, - groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Adding slot [historical]", slotId validation.slots.incl(slotId) - trace "Removing slots that have ended...", groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Removing slots that have ended..." await removeSlotsThatHaveEnded(validation) - trace "Historical state restored", numberOfSlots = validation.slots.len, - groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Historical state restored", numberOfSlots = validation.slots.len proc start*(validation: Validation) {.async.} = validation.periodicity = await validation.market.periodicity() From afb444cac8bc268c23298fde01c79f76eb4c0891 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 01:09:20 +0200 Subject: [PATCH 26/71] improves timing and clarifies the test conditions --- tests/integration/testvalidator.nim | 59 +++++++++++++++++++---------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index c5e6c2558..9c11da353 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -16,6 +16,7 @@ logScope: marketplacesuite "Validation": let nodes = 3 let tolerance = 1 + let proofProbability = 1 var slotsFilled: seq[SlotId] var slotsFreed: seq[SlotId] @@ -31,14 +32,14 @@ marketplacesuite "Validation": let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) subscription - proc trackSlotsFreed(requestId: RequestId, marketplace: Marketplace): + proc trackSlotsFreed(marketplace: Marketplace, requestId: RequestId): Future[provider.Subscription] {.async.} = slotsFreed = newSeq[SlotId]() proc onSlotFreed(event: SlotFreed) = if event.requestId == requestId: let slotId = slotId(event.requestId, event.slotIndex) slotsFreed.add(slotId) - debug "onSlotFreed", requestId = requestId, slotIndex = event.slotIndex, + debug "SlotFreed", requestId = requestId, slotIndex = event.slotIndex, slotId = slotId, slotsFreed = slotsFreed.len let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) @@ -85,7 +86,7 @@ marketplacesuite "Validation": .withValidationGroupIndex(idx = 1, groupIndex = 1) # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("validator") + # .withLogTopics("validator") # each topic as a separate string argument # .withLogTopics("validator", "integration", "ethers", "clock") .some ): @@ -93,13 +94,16 @@ marketplacesuite "Validation": let expiry = 5.periods let duration = expiry + 10.periods + var currentTime = await ethProvider.currentTime() + let requestEndTime = currentTime.truncate(uint64) + duration + let data = await RandomChunker.example(blocks=8) # TODO: better value for data.len below. This TODO is also present in # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) - let slotFilledSubscription = await trackSlotsFilled(marketplace) + let slotFilledSubscription = await marketplace.trackSlotsFilled() let cid = client0.upload(data).get @@ -109,23 +113,30 @@ marketplacesuite "Validation": duration=duration, nodes=nodes, tolerance=tolerance, - proofProbability=1 + proofProbability=proofProbability ) let requestId = client0.requestId(purchaseId).get + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) + currentTime = await ethProvider.currentTime() + let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int + + debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds + let slotFreedSubscription = - await trackSlotsFreed(requestId, marketplace) + await marketplace.trackSlotsFreed(requestId) - let expectedSlotsFreed = nodes - tolerance + let expectedSlotsFreed = tolerance + 1 check eventually((slotsFreed.len == expectedSlotsFreed), - timeout=(duration - expiry).int * 1000) + timeout=(secondsTillRequestEnd + 60) * 1000) - # Because of erasure coding, if e.g. 2 out of 3 nodes are freed, the last - # node will not be freed but marked as "Failed" because the whole request - # will fail. For this reason we need an extra check: + # Because of erasure coding, after (tolerance + 1) slots are freed, the + # remaining nodes are be freed but marked as "Failed" as the whole + # request fails. To capture this we need an extra check: await checkSlotsFailed(slotsFilled, slotsFreed, marketplace) await slotFilledSubscription.unsubscribe() @@ -155,13 +166,16 @@ marketplacesuite "Validation": let expiry = 5.periods let duration = expiry + 10.periods + var currentTime = await ethProvider.currentTime() + let requestEndTime = currentTime.truncate(uint64) + duration + let data = await RandomChunker.example(blocks=8) # TODO: better value for data.len below. This TODO is also present in # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) - let slotFilledSubscription = await trackSlotsFilled(marketplace) + let slotFilledSubscription = await marketplace.trackSlotsFilled() let cid = client0.upload(data).get @@ -171,10 +185,12 @@ marketplacesuite "Validation": duration=duration, nodes=nodes, tolerance=tolerance, - proofProbability=1 + proofProbability=proofProbability ) let requestId = client0.requestId(purchaseId).get + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) @@ -199,16 +215,21 @@ marketplacesuite "Validation": node: node ) + currentTime = await ethProvider.currentTime() + let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int + + debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds + let slotFreedSubscription = - await trackSlotsFreed(requestId, marketplace) + await marketplace.trackSlotsFreed(requestId) - let expectedSlotsFreed = nodes - tolerance + let expectedSlotsFreed = tolerance + 1 check eventually((slotsFreed.len == expectedSlotsFreed), - timeout=(duration - expiry).int * 1000) + timeout=(secondsTillRequestEnd + 60) * 1000) - # Because of erasure coding, if e.g. 2 out of 3 nodes are freed, the last - # node will not be freed but marked as "Failed" because the whole request - # will fail. For this reason we need an extra check: + # Because of erasure coding, after (tolerance + 1) slots are freed, the + # remaining nodes are be freed but marked as "Failed" as the whole + # request fails. To capture this we need an extra check: await checkSlotsFailed(slotsFilled, slotsFreed, marketplace) await slotFilledSubscription.unsubscribe() From c32eac1fa8c010b6412c3385cc5d65dacd3973b2 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 04:13:58 +0200 Subject: [PATCH 27/71] uses http as default RPC provider for nodes running in integration tests as a workaround for dropped subscriptions --- tests/integration/multinodes.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 983466ae0..ddb875df8 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -67,6 +67,10 @@ template multinodesuite*(name: string, body: untyped) = let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") var currentTestName = "" var nodeConfigs: NodeConfigs + # Workaround for https://github.com/NomicFoundation/hardhat/issues/2053 + # Do not use websockets, but use http and polling to stop subscriptions + # from being removed after 5 minutes + let defaultProviderUrl = "http://127.0.0.1:8545" var ethProvider {.inject, used.}: JsonRpcProvider var accounts {.inject, used.}: seq[Address] var snapshot: JsonNode @@ -196,7 +200,7 @@ template multinodesuite*(name: string, body: untyped) = proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = let clientIdx = clients().len var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", "http://127.0.0.1:8545") + config.addCliOption(StartUpCmd.persistence, "--eth-provider", defaultProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) return await newCodexProcess(clientIdx, config, Role.Client) @@ -204,7 +208,7 @@ template multinodesuite*(name: string, body: untyped) = let providerIdx = providers().len var config = conf config.addCliOption("--bootstrap-node", bootstrap) - config.addCliOption(StartUpCmd.persistence, "--eth-provider", "http://127.0.0.1:8545") + config.addCliOption(StartUpCmd.persistence, "--eth-provider", defaultProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) config.addCliOption(PersistenceCmd.prover, "--circom-r1cs", "vendor/codex-contracts-eth/verifier/networks/hardhat/proof_main.r1cs") @@ -219,7 +223,7 @@ template multinodesuite*(name: string, body: untyped) = let validatorIdx = validators().len var config = conf config.addCliOption("--bootstrap-node", bootstrap) - config.addCliOption(StartUpCmd.persistence, "--eth-provider", "http://127.0.0.1:8545") + config.addCliOption(StartUpCmd.persistence, "--eth-provider", defaultProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) config.addCliOption(StartUpCmd.persistence, "--validator") @@ -268,7 +272,7 @@ template multinodesuite*(name: string, body: untyped) = # Do not use websockets, but use http and polling to stop subscriptions # from being removed after 5 minutes ethProvider = JsonRpcProvider.new( - "http://127.0.0.1:8545", + defaultProviderUrl, pollingInterval = chronos.milliseconds(100) ) # if hardhat was NOT started by the test, take a snapshot so it can be From 0534380ab2e45758a3542531223994cea3cdd11b Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 04:15:10 +0200 Subject: [PATCH 28/71] simplifies the validation integration tests by waiting for failed request instead of tracking slots --- tests/integration/testvalidator.nim | 95 ++++++----------------------- 1 file changed, 20 insertions(+), 75 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 9c11da353..2c658029f 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,5 +1,4 @@ from std/times import inMilliseconds, initDuration, inSeconds, fromUnix -import std/sets import pkg/codex/logutils import ../contracts/time import ../contracts/deployment @@ -17,47 +16,6 @@ marketplacesuite "Validation": let nodes = 3 let tolerance = 1 let proofProbability = 1 - var slotsFilled: seq[SlotId] - var slotsFreed: seq[SlotId] - - proc trackSlotsFilled(marketplace: Marketplace): - Future[provider.Subscription] {.async.} = - slotsFilled = newSeq[SlotId]() - proc onSlotFilled(event: SlotFilled) = - let slotId = slotId(event.requestId, event.slotIndex) - slotsFilled.add(slotId) - debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex, - slotId = slotId - - let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - subscription - - proc trackSlotsFreed(marketplace: Marketplace, requestId: RequestId): - Future[provider.Subscription] {.async.} = - slotsFreed = newSeq[SlotId]() - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: - let slotId = slotId(event.requestId, event.slotIndex) - slotsFreed.add(slotId) - debug "SlotFreed", requestId = requestId, slotIndex = event.slotIndex, - slotId = slotId, slotsFreed = slotsFreed.len - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - subscription - - proc checkSlotsFailed(slotsFilled: seq[SlotId], - slotsFreed: seq[SlotId], marketplace: Marketplace) {.async.} = - let slotsNotFreed = slotsFilled.filter( - slotId => not slotsFreed.contains(slotId) - ).toHashSet - var slotsFailed = initHashSet[SlotId]() - for slotId in slotsFilled: - let state = await marketplace.slotState(slotId) - if state == SlotState.Failed: - slotsFailed.incl(slotId) - - debug "slots failed", slotsFailed = slotsFailed, slotsNotFreed = slotsNotFreed - check slotsNotFreed == slotsFailed test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally @@ -69,6 +27,7 @@ marketplacesuite "Validation": # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("node", "marketplace", "clock") + # .withLogTopics("node", "purchases", "slotqueue", "market") .some, providers: @@ -87,7 +46,6 @@ marketplacesuite "Validation": # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("validator") # each topic as a separate string argument - # .withLogTopics("validator", "integration", "ethers", "clock") .some ): let client0 = clients()[0].client @@ -103,8 +61,6 @@ marketplacesuite "Validation": # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) - let slotFilledSubscription = await marketplace.trackSlotsFilled() - let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( @@ -127,20 +83,15 @@ marketplacesuite "Validation": debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - let slotFreedSubscription = - await marketplace.trackSlotsFreed(requestId) - - let expectedSlotsFreed = tolerance + 1 - check eventually((slotsFreed.len == expectedSlotsFreed), - timeout=(secondsTillRequestEnd + 60) * 1000) - - # Because of erasure coding, after (tolerance + 1) slots are freed, the - # remaining nodes are be freed but marked as "Failed" as the whole - # request fails. To capture this we need an extra check: - await checkSlotsFailed(slotsFilled, slotsFreed, marketplace) - - await slotFilledSubscription.unsubscribe() - await slotFreedSubscription.unsubscribe() + # Because of Erasure Coding, the expected number of slots being freed + # is tolerance + 1. When more than tolerance slots are freed, the whole + # request will fail. Thus, awaiting for a failing state should + # be sufficient to conclude that validators did their job correctly. + # NOTICE: We actually have to wait for the "errored" state, because + # immediately after withdrawing the funds the purchasing state machine + # transitions to the "errored" state. + check eventually(client0.purchaseStateIs(purchaseId, "errored"), + timeout = (secondsTillRequestEnd + 60) * 1000) test "validator uses historical state to mark missing proofs", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally @@ -152,6 +103,7 @@ marketplacesuite "Validation": # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("node", "marketplace", "clock") + # .withLogTopics("node", "purchases", "slotqueue", "market") .some, providers: @@ -175,8 +127,6 @@ marketplacesuite "Validation": # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) - let slotFilledSubscription = await marketplace.trackSlotsFilled() - let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( @@ -219,18 +169,13 @@ marketplacesuite "Validation": let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - let slotFreedSubscription = - await marketplace.trackSlotsFreed(requestId) - - let expectedSlotsFreed = tolerance + 1 - check eventually((slotsFreed.len == expectedSlotsFreed), - timeout=(secondsTillRequestEnd + 60) * 1000) - - # Because of erasure coding, after (tolerance + 1) slots are freed, the - # remaining nodes are be freed but marked as "Failed" as the whole - # request fails. To capture this we need an extra check: - await checkSlotsFailed(slotsFilled, slotsFreed, marketplace) - await slotFilledSubscription.unsubscribe() - await slotFreedSubscription.unsubscribe() + # Because of Erasure Coding, the expected number of slots being freed + # is tolerance + 1. When more than tolerance slots are freed, the whole + # request will fail. Thus, awaiting for a failing state should + # be sufficient to conclude that validators did their job correctly. + # NOTICE: We actually have to wait for the "errored" state, because + # immediately after withdrawing the funds the purchasing state machine + # transitions to the "errored" state. + check eventually(client0.purchaseStateIs(purchaseId, "errored"), + timeout = (secondsTillRequestEnd + 60) * 1000) From 375b65b50c7fdc5804652608d03045e839f44d55 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 22:23:59 +0200 Subject: [PATCH 29/71] adds config option allowing selectively to set different provider url --- tests/integration/codexconfig.nim | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/integration/codexconfig.nim b/tests/integration/codexconfig.nim index d4c917740..f321364fa 100644 --- a/tests/integration/codexconfig.nim +++ b/tests/integration/codexconfig.nim @@ -261,6 +261,29 @@ proc withValidationGroupIndex*( StartUpCmd.persistence, "--validator-group-index", $groupIndex) return startConfig +proc withEthProvider*( + self: CodexConfigs, + idx: int, + ethProvider: string +): CodexConfigs {.raises: [CodexConfigError].} = + + self.checkBounds idx + + var startConfig = self + startConfig.configs[idx].addCliOption(StartUpCmd.persistence, + "--eth-provider", ethProvider) + return startConfig + +proc withEthProvider*( + self: CodexConfigs, + ethProvider: string): CodexConfigs {.raises: [CodexConfigError].} = + + var startConfig = self + for config in startConfig.configs.mitems: + config.addCliOption(StartUpCmd.persistence, + "--eth-provider", ethProvider) + return startConfig + proc logLevelWithTopics( config: CodexConfig, topics: varargs[string]): string {.raises: [CodexConfigError].} = From 00e7d8cdacd0c678505f27947ca46cc06454faa6 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 22:33:52 +0200 Subject: [PATCH 30/71] Brings back the default settings for RPC provider in integration tests --- tests/integration/multinodes.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index ddb875df8..f29537030 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -200,7 +200,6 @@ template multinodesuite*(name: string, body: untyped) = proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = let clientIdx = clients().len var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", defaultProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) return await newCodexProcess(clientIdx, config, Role.Client) @@ -208,7 +207,6 @@ template multinodesuite*(name: string, body: untyped) = let providerIdx = providers().len var config = conf config.addCliOption("--bootstrap-node", bootstrap) - config.addCliOption(StartUpCmd.persistence, "--eth-provider", defaultProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) config.addCliOption(PersistenceCmd.prover, "--circom-r1cs", "vendor/codex-contracts-eth/verifier/networks/hardhat/proof_main.r1cs") @@ -223,7 +221,6 @@ template multinodesuite*(name: string, body: untyped) = let validatorIdx = validators().len var config = conf config.addCliOption("--bootstrap-node", bootstrap) - config.addCliOption(StartUpCmd.persistence, "--eth-provider", defaultProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) config.addCliOption(StartUpCmd.persistence, "--validator") From 7c518461bba3242be68a5a71162fe530db496fa2 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 22:34:45 +0200 Subject: [PATCH 31/71] use http RPC provider for clients in validation integration tests --- tests/integration/testvalidator.nim | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 2c658029f..04be5bf53 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -24,18 +24,19 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - # .debug() # uncomment to enable console log output + .withEthProvider("http://localhost:8545") + .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("node", "marketplace", "clock") - # .withLogTopics("node", "purchases", "slotqueue", "market") + .withLogTopics("node", "purchases", "slotqueue", "market") .some, providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") .some, validators: @@ -43,9 +44,9 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("validator") # each topic as a separate string argument + .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument .some ): let client0 = clients()[0].client @@ -100,18 +101,18 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - # .withLogTopics("node", "purchases", "slotqueue", "market") + .withEthProvider("http://localhost:8545") + .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("node", "purchases", "slotqueue", "market") .some, providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") .some ): let client0 = clients()[0].client @@ -152,10 +153,9 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to: - # tests/integration/logs/ //_.log - # .withLogTopics("validator") # each topic as a separate string argument + .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to: # tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument failAndTeardownOnError "failed to start validator nodes": for config in validators.configs.mitems: From 64077141ca40ded2af462a0ff86c201c372ad0aa Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 22:35:57 +0200 Subject: [PATCH 32/71] fine-tune the tests --- .github/workflows/ci.yml | 35 +++-- tests/integration/nodeprocess.nim | 3 +- tests/integration/testmarketplace.nim | 2 - tests/integration/testvalidator.nim | 29 +++-- tests/integration/testvalidator2.nim | 181 ++++++++++++++++++++++++++ 5 files changed, 215 insertions(+), 35 deletions(-) create mode 100644 tests/integration/testvalidator2.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbe18e64b..0d27077a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,6 @@ env: cache_nonce: 0 # Allows for easily busting actions/cache caches nim_version: pinned - concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: true @@ -23,23 +22,23 @@ jobs: matrix: ${{ steps.matrix.outputs.matrix }} cache_nonce: ${{ env.cache_nonce }} steps: - - name: Compute matrix - id: matrix - uses: fabiocaccamo/create-matrix-action@v4 - with: - matrix: | - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} + - name: Compute matrix + id: matrix + uses: fabiocaccamo/create-matrix-action@v4 + with: + matrix: | + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} build: needs: matrix diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index 61947c20c..4f5cd1263 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -156,7 +156,8 @@ proc waitUntilStarted*(node: NodeProcess) {.async.} = let started = newFuture[void]() try: discard node.captureOutput(node.startedOutput, started).track(node) - await started.wait(35.seconds) # allow enough time for proof generation + await started.wait(60.seconds) # allow enough time for proof generation + trace "node started" except AsyncTimeoutError: # attempt graceful shutdown in case node was partially started, prevent # zombies diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index 0c18ff1a9..b5ae4796f 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -1,5 +1,3 @@ -import pkg/stew/byteutils -import pkg/codex/units import ../examples import ../contracts/time import ../contracts/deployment diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 04be5bf53..f87766d62 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -25,18 +25,18 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) .withEthProvider("http://localhost:8545") - .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - .withLogTopics("node", "purchases", "slotqueue", "market") + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("purchases", "onchain") .some, providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .withEthProvider("http://localhost:8545") + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("sales", "onchain") .some, validators: @@ -44,7 +44,7 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument .some @@ -102,17 +102,18 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) .withEthProvider("http://localhost:8545") - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node", "purchases", "slotqueue", "market") + .withLogTopics("purchases", "onchain") .some, providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .withEthProvider("http://localhost:8545") + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("sales", "onchain") .some ): let client0 = clients()[0].client @@ -153,7 +154,7 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to: # tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument diff --git a/tests/integration/testvalidator2.nim b/tests/integration/testvalidator2.nim new file mode 100644 index 000000000..b0e921884 --- /dev/null +++ b/tests/integration/testvalidator2.nim @@ -0,0 +1,181 @@ +from std/times import inMilliseconds, initDuration, inSeconds, fromUnix +import pkg/codex/logutils +import ../contracts/time +import ../contracts/deployment +import ../codex/helpers +import ../examples +import ./marketplacesuite +import ./nodeconfigs + +export logutils + +logScope: + topics = "integration test validation" + +marketplacesuite "Validation": + let nodes = 3 + let tolerance = 1 + let proofProbability = 1 + + test "validator marks proofs as missing when using validation groups", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + .withLogTopics("node", "purchases", "slotqueue", "market") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some, + + validators: + CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration = expiry + 10.periods + + var currentTime = await ethProvider.currentTime() + let requestEndTime = currentTime.truncate(uint64) + duration + + let data = await RandomChunker.example(blocks=8) + + # TODO: better value for data.len below. This TODO is also present in + # testproofs.nim - we may want to address it or remove the comment. + createAvailabilities(data.len * 2, duration) + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=nodes, + tolerance=tolerance, + proofProbability=proofProbability + ) + let requestId = client0.requestId(purchaseId).get + + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId + + check eventually(client0.purchaseStateIs(purchaseId, "started"), + timeout = expiry.int * 1000) + + currentTime = await ethProvider.currentTime() + let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int + + debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds + + # Because of Erasure Coding, the expected number of slots being freed + # is tolerance + 1. When more than tolerance slots are freed, the whole + # request will fail. Thus, awaiting for a failing state should + # be sufficient to conclude that validators did their job correctly. + # NOTICE: We actually have to wait for the "errored" state, because + # immediately after withdrawing the funds the purchasing state machine + # transitions to the "errored" state. + check eventually(client0.purchaseStateIs(purchaseId, "errored"), + timeout = (secondsTillRequestEnd + 60) * 1000) + + test "validator uses historical state to mark missing proofs", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + # .withLogTopics("node", "purchases", "slotqueue", "market") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration = expiry + 10.periods + + var currentTime = await ethProvider.currentTime() + let requestEndTime = currentTime.truncate(uint64) + duration + + let data = await RandomChunker.example(blocks=8) + + # TODO: better value for data.len below. This TODO is also present in + # testproofs.nim - we may want to address it or remove the comment. + createAvailabilities(data.len * 2, duration) + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=nodes, + tolerance=tolerance, + proofProbability=proofProbability + ) + let requestId = client0.requestId(purchaseId).get + + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId + + check eventually(client0.purchaseStateIs(purchaseId, "started"), + timeout = expiry.int * 1000) + + # just to make sure we have a mined block that separates us + # from the block containing the last SlotFilled event + discard await ethProvider.send("evm_mine") + + var validators = CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to: + # tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument + + failAndTeardownOnError "failed to start validator nodes": + for config in validators.configs.mitems: + let node = await startValidatorNode(config) + running.add RunningNode( + role: Role.Validator, + node: node + ) + + currentTime = await ethProvider.currentTime() + let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int + + debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds + + # Because of Erasure Coding, the expected number of slots being freed + # is tolerance + 1. When more than tolerance slots are freed, the whole + # request will fail. Thus, awaiting for a failing state should + # be sufficient to conclude that validators did their job correctly. + # NOTICE: We actually have to wait for the "errored" state, because + # immediately after withdrawing the funds the purchasing state machine + # transitions to the "errored" state. + check eventually(client0.purchaseStateIs(purchaseId, "errored"), + timeout = (secondsTillRequestEnd + 60) * 1000) From ad0b8b6862f6607db1caeda88bec84680d31d72e Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 17 Oct 2024 06:42:13 +0200 Subject: [PATCH 33/71] Makes validator integration test more robust - adds extra tracking --- tests/integration/testvalidator.nim | 84 ++++++++++++++++++++++++----- tests/testIntegration.nim | 18 +++---- 2 files changed, 81 insertions(+), 21 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index f87766d62..07f95b045 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,5 +1,7 @@ from std/times import inMilliseconds, initDuration, inSeconds, fromUnix +import std/strformat import pkg/codex/logutils +import pkg/questionable/results import ../contracts/time import ../contracts/deployment import ../codex/helpers @@ -12,10 +14,30 @@ export logutils logScope: topics = "integration test validation" +template eventuallyS*(expression: untyped, timeout=10, step = 5): bool = + bind Moment, now, seconds + + proc eventuallyS: Future[bool] {.async.} = + let endTime = Moment.now() + timeout.seconds + var i = 0 + while not expression: + inc i + echo (i*step).seconds + if endTime < Moment.now(): + return false + await sleepAsync(step.seconds) + return true + + await eventuallyS() + marketplacesuite "Validation": let nodes = 3 let tolerance = 1 let proofProbability = 1 + when defined(windows): + let providerUrl = "ws://localhost:8545" + else: + let providerUrl = "http://localhost:8545" test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally @@ -24,7 +46,7 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - .withEthProvider("http://localhost:8545") + .withEthProvider(providerUrl) # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("purchases", "onchain") @@ -33,7 +55,7 @@ marketplacesuite "Validation": providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .withEthProvider("http://localhost:8545") + .withEthProvider(providerUrl) # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("sales", "onchain") @@ -41,6 +63,7 @@ marketplacesuite "Validation": validators: CodexConfigs.init(nodes=2) + .withEthProvider(providerUrl) .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) @@ -53,6 +76,11 @@ marketplacesuite "Validation": let expiry = 5.periods let duration = expiry + 10.periods + echo fmt"{providerUrl = }" + + # for a good start + discard await ethProvider.send("evm_mine") + var currentTime = await ethProvider.currentTime() let requestEndTime = currentTime.truncate(uint64) + duration @@ -76,8 +104,22 @@ marketplacesuite "Validation": debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - check eventually(client0.purchaseStateIs(purchaseId, "started"), - timeout = expiry.int * 1000) + echo fmt"expiry = {(expiry + 60).int.seconds}" + + check eventuallyS(client0.purchaseStateIs(purchaseId, "started"), + timeout = (expiry + 60).int, step = 5) + + # if purchase state is not "started", it does not make sense to continue + without purchaseState =? client0.getPurchase(purchaseId).?state: + fail() + + debug "validation suite", purchaseState = purchaseState + echo fmt"{purchaseState = }" + + if purchaseState != "started": + fail() + + discard await ethProvider.send("evm_mine") currentTime = await ethProvider.currentTime() let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int @@ -91,8 +133,8 @@ marketplacesuite "Validation": # NOTICE: We actually have to wait for the "errored" state, because # immediately after withdrawing the funds the purchasing state machine # transitions to the "errored" state. - check eventually(client0.purchaseStateIs(purchaseId, "errored"), - timeout = (secondsTillRequestEnd + 60) * 1000) + check eventuallyS(client0.purchaseStateIs(purchaseId, "errored"), + timeout = secondsTillRequestEnd + 60, step = 5) test "validator uses historical state to mark missing proofs", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally @@ -101,7 +143,7 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - .withEthProvider("http://localhost:8545") + .withEthProvider(providerUrl) # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("purchases", "onchain") @@ -109,8 +151,8 @@ marketplacesuite "Validation": providers: CodexConfigs.init(nodes=1) + .withEthProvider(providerUrl) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .withEthProvider("http://localhost:8545") # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("sales", "onchain") @@ -120,6 +162,11 @@ marketplacesuite "Validation": let expiry = 5.periods let duration = expiry + 10.periods + echo fmt"{providerUrl = }" + + # for a good start + discard await ethProvider.send("evm_mine") + var currentTime = await ethProvider.currentTime() let requestEndTime = currentTime.truncate(uint64) + duration @@ -143,14 +190,27 @@ marketplacesuite "Validation": debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - check eventually(client0.purchaseStateIs(purchaseId, "started"), - timeout = expiry.int * 1000) + echo fmt"expiry = {(expiry + 60).int.seconds}" + + check eventuallyS(client0.purchaseStateIs(purchaseId, "started"), + timeout = (expiry + 60).int, step = 5) + + # if purchase state is not "started", it does not make sense to continue + without purchaseState =? client0.getPurchase(purchaseId).?state: + fail() + + debug "validation suite", purchaseState = purchaseState + echo fmt"{purchaseState = }" + + if purchaseState != "started": + fail() # just to make sure we have a mined block that separates us # from the block containing the last SlotFilled event discard await ethProvider.send("evm_mine") var validators = CodexConfigs.init(nodes=2) + .withEthProvider(providerUrl) .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) @@ -178,5 +238,5 @@ marketplacesuite "Validation": # NOTICE: We actually have to wait for the "errored" state, because # immediately after withdrawing the funds the purchasing state machine # transitions to the "errored" state. - check eventually(client0.purchaseStateIs(purchaseId, "errored"), - timeout = (secondsTillRequestEnd + 60) * 1000) + check eventuallyS(client0.purchaseStateIs(purchaseId, "errored"), + timeout = secondsTillRequestEnd + 60, step = 5) diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index f0a59ee45..91cc3630b 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -1,12 +1,12 @@ -import ./integration/testcli -import ./integration/testrestapi -import ./integration/testupdownload -import ./integration/testsales -import ./integration/testpurchasing -import ./integration/testblockexpiration -import ./integration/testmarketplace -import ./integration/testproofs +# import ./integration/testcli +# import ./integration/testrestapi +# import ./integration/testupdownload +# import ./integration/testsales +# import ./integration/testpurchasing +# import ./integration/testblockexpiration +# import ./integration/testmarketplace +# import ./integration/testproofs import ./integration/testvalidator -import ./integration/testecbug +# import ./integration/testecbug {.warning[UnusedImport]:off.} From c7fd863178aabd1999c31d54d5dbf380e183de72 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 17 Oct 2024 16:26:31 +0200 Subject: [PATCH 34/71] brings tracking of marketplace event back to validator integration test --- .github/workflows/ci.yml | 1 - build.nims | 6 +- tests/integration/testvalidator.nim | 173 ++++++++++++++++++++-------- 3 files changed, 131 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d27077a3..55d8ba0a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,6 @@ jobs: os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} build: needs: matrix diff --git a/build.nims b/build.nims index a9a0e5534..d26fd413c 100644 --- a/build.nims +++ b/build.nims @@ -40,10 +40,10 @@ task testContracts, "Build & run Codex Contract tests": task testIntegration, "Run integration tests": buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true" - test "testIntegration" + # test "testIntegration" # use params to enable logging from the integration test executable - # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & - # "-d:chronicles_enabled_topics:integration:TRACE" + test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & + "-d:chronicles_enabled_topics:integration:TRACE" task build, "build codex binary": codexTask() diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 07f95b045..1572b79c4 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -14,7 +14,8 @@ export logutils logScope: topics = "integration test validation" -template eventuallyS*(expression: untyped, timeout=10, step = 5): bool = +template eventuallyS*(expression: untyped, timeout=10, step = 5, + cancelWhenExpression: untyped = false): bool = bind Moment, now, seconds proc eventuallyS: Future[bool] {.async.} = @@ -25,6 +26,8 @@ template eventuallyS*(expression: untyped, timeout=10, step = 5): bool = echo (i*step).seconds if endTime < Moment.now(): return false + if cancelWhenExpression: + return false await sleepAsync(step.seconds) return true @@ -34,10 +37,89 @@ marketplacesuite "Validation": let nodes = 3 let tolerance = 1 let proofProbability = 1 - when defined(windows): - let providerUrl = "ws://localhost:8545" - else: - let providerUrl = "http://localhost:8545" + + var slotsFilled: seq[SlotId] + var slotsFreed: seq[SlotId] + var requestsFailed: seq[RequestId] + var requestCancelled = false + + var slotFilledSubscription: provider.Subscription + var requestFailedSubscription: provider.Subscription + var slotFreedSubscription: provider.Subscription + var requestCancelledSubscription: provider.Subscription + + proc trackSlotsFilled(marketplace: Marketplace): + Future[provider.Subscription] {.async.} = + slotsFilled = newSeq[SlotId]() + proc onSlotFilled(event: SlotFilled) = + let slotId = slotId(event.requestId, event.slotIndex) + slotsFilled.add(slotId) + debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex, + slotId = slotId + + let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) + subscription + + proc trackRequestsFailed(marketplace: Marketplace): + Future[provider.Subscription] {.async.} = + requestsFailed = newSeq[RequestId]() + proc onRequestFailed(event: RequestFailed) = + requestsFailed.add(event.requestId) + debug "RequestFailed", requestId = event.requestId + + let subscription = await marketplace.subscribe(RequestFailed, onRequestFailed) + subscription + + proc trackRequestCancelled(marketplace: Marketplace, requestId: RequestId): + Future[provider.Subscription] {.async.} = + requestCancelled = false + proc onRequestCancelled(event: RequestCancelled) = + if requestId == event.requestId: + requestCancelled = true + debug "RequestCancelled", requestId = event.requestId + + let subscription = await marketplace.subscribe(RequestCancelled, onRequestCancelled) + subscription + + proc trackSlotsFreed(marketplace: Marketplace, requestId: RequestId): + Future[provider.Subscription] {.async.} = + slotsFreed = newSeq[SlotId]() + proc onSlotFreed(event: SlotFreed) = + if event.requestId == requestId: + let slotId = slotId(event.requestId, event.slotIndex) + slotsFreed.add(slotId) + debug "SlotFreed", requestId = event.requestId, slotIndex = event.slotIndex, + slotId = slotId, slotsFreed = slotsFreed.len + + let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + subscription + + proc startTrackingEvents(marketplace: Marketplace, requestId: RequestId) {.async.} = + slotFilledSubscription = await marketplace.trackSlotsFilled() + requestFailedSubscription = await marketplace.trackRequestsFailed() + slotFreedSubscription = await marketplace.trackSlotsFreed(requestId) + requestCancelledSubscription = + await marketplace.trackRequestCancelled(requestId) + + proc stopTrackingEvents() {.async.} = + await slotFilledSubscription.unsubscribe() + await slotFreedSubscription.unsubscribe() + await requestFailedSubscription.unsubscribe() + await requestCancelledSubscription.unsubscribe() + + proc checkSlotsFailed(marketplace: Marketplace, slotsFilled: seq[SlotId], + slotsFreed: seq[SlotId]) {.async.} = + let slotsNotFreed = slotsFilled.filter( + slotId => not slotsFreed.contains(slotId) + ).toHashSet + var slotsFailed = initHashSet[SlotId]() + for slotId in slotsFilled: + let state = await marketplace.slotState(slotId) + if state == SlotState.Failed: + slotsFailed.incl(slotId) + + debug "slots failed", slotsFailed = slotsFailed, slotsNotFreed = slotsNotFreed + check slotsNotFreed == slotsFailed test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally @@ -46,8 +128,7 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - .withEthProvider(providerUrl) - # .debug() # uncomment to enable console log output + .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("purchases", "onchain") .some, @@ -55,7 +136,6 @@ marketplacesuite "Validation": providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .withEthProvider(providerUrl) # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .withLogTopics("sales", "onchain") @@ -63,11 +143,10 @@ marketplacesuite "Validation": validators: CodexConfigs.init(nodes=2) - .withEthProvider(providerUrl) .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output + .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument .some @@ -76,9 +155,7 @@ marketplacesuite "Validation": let expiry = 5.periods let duration = expiry + 10.periods - echo fmt"{providerUrl = }" - - # for a good start + # let mine a block to sync the blocktime with the current clock discard await ethProvider.send("evm_mine") var currentTime = await ethProvider.currentTime() @@ -91,7 +168,6 @@ marketplacesuite "Validation": createAvailabilities(data.len * 2, duration) let cid = client0.upload(data).get - let purchaseId = await client0.requestStorage( cid, expiry=expiry, @@ -102,6 +178,8 @@ marketplacesuite "Validation": ) let requestId = client0.requestId(purchaseId).get + await marketplace.startTrackingEvents(requestId) + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId echo fmt"expiry = {(expiry + 60).int.seconds}" @@ -120,22 +198,25 @@ marketplacesuite "Validation": fail() discard await ethProvider.send("evm_mine") - currentTime = await ethProvider.currentTime() let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - # Because of Erasure Coding, the expected number of slots being freed - # is tolerance + 1. When more than tolerance slots are freed, the whole - # request will fail. Thus, awaiting for a failing state should - # be sufficient to conclude that validators did their job correctly. - # NOTICE: We actually have to wait for the "errored" state, because - # immediately after withdrawing the funds the purchasing state machine - # transitions to the "errored" state. - check eventuallyS(client0.purchaseStateIs(purchaseId, "errored"), - timeout = secondsTillRequestEnd + 60, step = 5) - + # Because of erasure coding, after (tolerance + 1) slots are freed, the + # remaining nodes are be freed but marked as "Failed" as the whole + # request fails. A couple of checks to capture this: + let expectedSlotsFreed = tolerance + 1 + check eventuallyS((slotsFreed.len == expectedSlotsFreed and + requestsFailed.contains(requestId)), + timeout = secondsTillRequestEnd + 60, step = 5, + cancelWhenExpression = requestCancelled) + + # extra check + await marketplace.checkSlotsFailed(slotsFilled, slotsFreed) + + await stopTrackingEvents() + test "validator uses historical state to mark missing proofs", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: @@ -143,15 +224,13 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - .withEthProvider(providerUrl) - # .debug() # uncomment to enable console log output + .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("purchases", "onchain") .some, providers: CodexConfigs.init(nodes=1) - .withEthProvider(providerUrl) .withSimulateProofFailures(idx=0, failEveryNProofs=1) # .debug() # uncomment to enable console log output # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log @@ -162,9 +241,7 @@ marketplacesuite "Validation": let expiry = 5.periods let duration = expiry + 10.periods - echo fmt"{providerUrl = }" - - # for a good start + # let mine a block to sync the blocktime with the current clock discard await ethProvider.send("evm_mine") var currentTime = await ethProvider.currentTime() @@ -177,7 +254,6 @@ marketplacesuite "Validation": createAvailabilities(data.len * 2, duration) let cid = client0.upload(data).get - let purchaseId = await client0.requestStorage( cid, expiry=expiry, @@ -188,6 +264,8 @@ marketplacesuite "Validation": ) let requestId = client0.requestId(purchaseId).get + await marketplace.startTrackingEvents(requestId) + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId echo fmt"expiry = {(expiry + 60).int.seconds}" @@ -205,16 +283,15 @@ marketplacesuite "Validation": if purchaseState != "started": fail() - # just to make sure we have a mined block that separates us - # from the block containing the last SlotFilled event + # extra block just to make sure we have one that separates us + # from the block containing the last (past) SlotFilled event discard await ethProvider.send("evm_mine") var validators = CodexConfigs.init(nodes=2) - .withEthProvider(providerUrl) .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output + .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to: # tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument @@ -226,17 +303,23 @@ marketplacesuite "Validation": node: node ) + discard await ethProvider.send("evm_mine") currentTime = await ethProvider.currentTime() let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - # Because of Erasure Coding, the expected number of slots being freed - # is tolerance + 1. When more than tolerance slots are freed, the whole - # request will fail. Thus, awaiting for a failing state should - # be sufficient to conclude that validators did their job correctly. - # NOTICE: We actually have to wait for the "errored" state, because - # immediately after withdrawing the funds the purchasing state machine - # transitions to the "errored" state. - check eventuallyS(client0.purchaseStateIs(purchaseId, "errored"), - timeout = secondsTillRequestEnd + 60, step = 5) + # Because of erasure coding, after (tolerance + 1) slots are freed, the + # remaining nodes are be freed but marked as "Failed" as the whole + # request fails. A couple of checks to capture this: + let expectedSlotsFreed = tolerance + 1 + + check eventuallyS((slotsFreed.len == expectedSlotsFreed and + requestsFailed.contains(requestId)), + timeout = secondsTillRequestEnd + 60, step = 5, + cancelWhenExpression = requestCancelled) + + # extra check + await marketplace.checkSlotsFailed(slotsFilled, slotsFreed) + + await stopTrackingEvents() From 32ebc5efcdf87726de14ec0aab12c12bc0fecb14 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 18 Oct 2024 05:09:03 +0200 Subject: [PATCH 35/71] refactors integration tests --- tests/integration/testvalidator.nim | 147 +++++++++++++--------------- 1 file changed, 67 insertions(+), 80 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 1572b79c4..914646c55 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,5 +1,7 @@ from std/times import inMilliseconds, initDuration, inSeconds, fromUnix import std/strformat +import std/sequtils +import std/sugar import pkg/codex/logutils import pkg/questionable/results import ../contracts/time @@ -15,7 +17,7 @@ logScope: topics = "integration test validation" template eventuallyS*(expression: untyped, timeout=10, step = 5, - cancelWhenExpression: untyped = false): bool = + cancelExpression: untyped = false): bool = bind Moment, now, seconds proc eventuallyS: Future[bool] {.async.} = @@ -26,7 +28,7 @@ template eventuallyS*(expression: untyped, timeout=10, step = 5, echo (i*step).seconds if endTime < Moment.now(): return false - if cancelWhenExpression: + if cancelExpression: return false await sleepAsync(step.seconds) return true @@ -38,77 +40,64 @@ marketplacesuite "Validation": let tolerance = 1 let proofProbability = 1 - var slotsFilled: seq[SlotId] - var slotsFreed: seq[SlotId] - var requestsFailed: seq[RequestId] - var requestCancelled = false - - var slotFilledSubscription: provider.Subscription - var requestFailedSubscription: provider.Subscription - var slotFreedSubscription: provider.Subscription - var requestCancelledSubscription: provider.Subscription - - proc trackSlotsFilled(marketplace: Marketplace): - Future[provider.Subscription] {.async.} = - slotsFilled = newSeq[SlotId]() - proc onSlotFilled(event: SlotFilled) = - let slotId = slotId(event.requestId, event.slotIndex) - slotsFilled.add(slotId) - debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex, - slotId = slotId - - let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - subscription + # var slotsAndRequests = initTable[string, seq[UInt256]]() + # var events = initTable[string, seq[ref MarketplaceEvent]]() + var events = { + $SlotFilled: newSeq[ref MarketplaceEvent](), + $SlotFreed: newSeq[ref MarketplaceEvent](), + $RequestFailed: newSeq[ref MarketplaceEvent](), + $RequestCancelled: newSeq[ref MarketplaceEvent]() + }.toTable + var eventSubscriptions = newSeq[provider.Subscription]() + + proc box[T](x: T): ref T = + new(result); + result[] = x + + proc onMarketplaceEvent[T: MarketplaceEvent](event: T) {.gcsafe, raises:[].} = + try: + debug "onMarketplaceEvent", eventType = $T, event = event + events[$T].add(box(event)) + except KeyError: + discard + + proc startTrackingEvents(marketplace: Marketplace) {.async.} = + eventSubscriptions.add( + await marketplace.subscribe(SlotFilled, onMarketplaceEvent[SlotFilled]) + ) + eventSubscriptions.add( + await marketplace.subscribe(RequestFailed, onMarketplaceEvent[RequestFailed]) + ) + eventSubscriptions.add( + await marketplace.subscribe(SlotFreed, onMarketplaceEvent[SlotFreed]) + ) + eventSubscriptions.add( + await marketplace.subscribe(RequestCancelled, onMarketplaceEvent[RequestCancelled]) + ) - proc trackRequestsFailed(marketplace: Marketplace): - Future[provider.Subscription] {.async.} = - requestsFailed = newSeq[RequestId]() - proc onRequestFailed(event: RequestFailed) = - requestsFailed.add(event.requestId) - debug "RequestFailed", requestId = event.requestId - - let subscription = await marketplace.subscribe(RequestFailed, onRequestFailed) - subscription + proc stopTrackingEvents() {.async.} = + for event in eventSubscriptions: + await event.unsubscribe() - proc trackRequestCancelled(marketplace: Marketplace, requestId: RequestId): - Future[provider.Subscription] {.async.} = - requestCancelled = false - proc onRequestCancelled(event: RequestCancelled) = - if requestId == event.requestId: - requestCancelled = true - debug "RequestCancelled", requestId = event.requestId - - let subscription = await marketplace.subscribe(RequestCancelled, onRequestCancelled) - subscription + proc checkSlotsFreed(requestId: RequestId, expectedSlotsFreed: int): bool = + events[$SlotFreed].filter( + e => (ref SlotFreed)(e).requestId == requestId) + .len == expectedSlotsFreed and + events[$RequestFailed].map( + e => (ref RequestFailed)(e).requestId).contains(requestId) - proc trackSlotsFreed(marketplace: Marketplace, requestId: RequestId): - Future[provider.Subscription] {.async.} = - slotsFreed = newSeq[SlotId]() - proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId: - let slotId = slotId(event.requestId, event.slotIndex) - slotsFreed.add(slotId) - debug "SlotFreed", requestId = event.requestId, slotIndex = event.slotIndex, - slotId = slotId, slotsFreed = slotsFreed.len - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - subscription - - proc startTrackingEvents(marketplace: Marketplace, requestId: RequestId) {.async.} = - slotFilledSubscription = await marketplace.trackSlotsFilled() - requestFailedSubscription = await marketplace.trackRequestsFailed() - slotFreedSubscription = await marketplace.trackSlotsFreed(requestId) - requestCancelledSubscription = - await marketplace.trackRequestCancelled(requestId) + proc isRequestCancelled(requestId: RequestId): bool = + events[$RequestCancelled].map(e => (ref RequestCancelled)(e).requestId) + .contains(requestId) - proc stopTrackingEvents() {.async.} = - await slotFilledSubscription.unsubscribe() - await slotFreedSubscription.unsubscribe() - await requestFailedSubscription.unsubscribe() - await requestCancelledSubscription.unsubscribe() + proc getSlots[T: MarketplaceEvent](requestId: RequestId): seq[SlotId] = + events[$T].filter( + e => (ref T)(e).requestId == requestId).map( + e => slotId((ref T)(e).requestId, (ref T)(e).slotIndex)) - proc checkSlotsFailed(marketplace: Marketplace, slotsFilled: seq[SlotId], - slotsFreed: seq[SlotId]) {.async.} = + proc checkSlotsFailed(marketplace: Marketplace, requestId: RequestId) {.async.} = + let slotsFreed = getSlots[SlotFreed](requestId) + let slotsFilled = getSlots[SlotFilled](requestId) let slotsNotFreed = slotsFilled.filter( slotId => not slotsFreed.contains(slotId) ).toHashSet @@ -167,6 +156,8 @@ marketplacesuite "Validation": # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) + await marketplace.startTrackingEvents() + let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, @@ -178,8 +169,6 @@ marketplacesuite "Validation": ) let requestId = client0.requestId(purchaseId).get - await marketplace.startTrackingEvents(requestId) - debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId echo fmt"expiry = {(expiry + 60).int.seconds}" @@ -207,13 +196,12 @@ marketplacesuite "Validation": # remaining nodes are be freed but marked as "Failed" as the whole # request fails. A couple of checks to capture this: let expectedSlotsFreed = tolerance + 1 - check eventuallyS((slotsFreed.len == expectedSlotsFreed and - requestsFailed.contains(requestId)), + check eventuallyS(checkSlotsFreed(requestId, expectedSlotsFreed), timeout = secondsTillRequestEnd + 60, step = 5, - cancelWhenExpression = requestCancelled) + cancelExpression = isRequestCancelled(requestId)) # extra check - await marketplace.checkSlotsFailed(slotsFilled, slotsFreed) + await marketplace.checkSlotsFailed(requestId) await stopTrackingEvents() @@ -253,6 +241,8 @@ marketplacesuite "Validation": # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) + await marketplace.startTrackingEvents() + let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, @@ -264,8 +254,6 @@ marketplacesuite "Validation": ) let requestId = client0.requestId(purchaseId).get - await marketplace.startTrackingEvents(requestId) - debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId echo fmt"expiry = {(expiry + 60).int.seconds}" @@ -314,12 +302,11 @@ marketplacesuite "Validation": # request fails. A couple of checks to capture this: let expectedSlotsFreed = tolerance + 1 - check eventuallyS((slotsFreed.len == expectedSlotsFreed and - requestsFailed.contains(requestId)), + check eventuallyS(checkSlotsFreed(requestId, expectedSlotsFreed), timeout = secondsTillRequestEnd + 60, step = 5, - cancelWhenExpression = requestCancelled) + cancelExpression = isRequestCancelled(requestId)) # extra check - await marketplace.checkSlotsFailed(slotsFilled, slotsFreed) + await marketplace.checkSlotsFailed(requestId) await stopTrackingEvents() From f48a9398d3d43e5c2bfc3285d529f16bb598b3a8 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 18 Oct 2024 05:11:02 +0200 Subject: [PATCH 36/71] deletes tmp file --- tests/integration/testvalidator2.nim | 181 --------------------------- 1 file changed, 181 deletions(-) delete mode 100644 tests/integration/testvalidator2.nim diff --git a/tests/integration/testvalidator2.nim b/tests/integration/testvalidator2.nim deleted file mode 100644 index b0e921884..000000000 --- a/tests/integration/testvalidator2.nim +++ /dev/null @@ -1,181 +0,0 @@ -from std/times import inMilliseconds, initDuration, inSeconds, fromUnix -import pkg/codex/logutils -import ../contracts/time -import ../contracts/deployment -import ../codex/helpers -import ../examples -import ./marketplacesuite -import ./nodeconfigs - -export logutils - -logScope: - topics = "integration test validation" - -marketplacesuite "Validation": - let nodes = 3 - let tolerance = 1 - let proofProbability = 1 - - test "validator marks proofs as missing when using validation groups", NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: - HardhatConfig.none, - - clients: - CodexConfigs.init(nodes=1) - # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - .withLogTopics("node", "purchases", "slotqueue", "market") - .some, - - providers: - CodexConfigs.init(nodes=1) - .withSimulateProofFailures(idx=0, failEveryNProofs=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") - .some, - - validators: - CodexConfigs.init(nodes=2) - .withValidationGroups(groups = 2) - .withValidationGroupIndex(idx = 0, groupIndex = 0) - .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator") # each topic as a separate string argument - .some - ): - let client0 = clients()[0].client - let expiry = 5.periods - let duration = expiry + 10.periods - - var currentTime = await ethProvider.currentTime() - let requestEndTime = currentTime.truncate(uint64) + duration - - let data = await RandomChunker.example(blocks=8) - - # TODO: better value for data.len below. This TODO is also present in - # testproofs.nim - we may want to address it or remove the comment. - createAvailabilities(data.len * 2, duration) - - let cid = client0.upload(data).get - - let purchaseId = await client0.requestStorage( - cid, - expiry=expiry, - duration=duration, - nodes=nodes, - tolerance=tolerance, - proofProbability=proofProbability - ) - let requestId = client0.requestId(purchaseId).get - - debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - - check eventually(client0.purchaseStateIs(purchaseId, "started"), - timeout = expiry.int * 1000) - - currentTime = await ethProvider.currentTime() - let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int - - debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - # Because of Erasure Coding, the expected number of slots being freed - # is tolerance + 1. When more than tolerance slots are freed, the whole - # request will fail. Thus, awaiting for a failing state should - # be sufficient to conclude that validators did their job correctly. - # NOTICE: We actually have to wait for the "errored" state, because - # immediately after withdrawing the funds the purchasing state machine - # transitions to the "errored" state. - check eventually(client0.purchaseStateIs(purchaseId, "errored"), - timeout = (secondsTillRequestEnd + 60) * 1000) - - test "validator uses historical state to mark missing proofs", NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: - HardhatConfig.none, - - clients: - CodexConfigs.init(nodes=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - # .withLogTopics("node", "purchases", "slotqueue", "market") - .some, - - providers: - CodexConfigs.init(nodes=1) - .withSimulateProofFailures(idx=0, failEveryNProofs=1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") - .some - ): - let client0 = clients()[0].client - let expiry = 5.periods - let duration = expiry + 10.periods - - var currentTime = await ethProvider.currentTime() - let requestEndTime = currentTime.truncate(uint64) + duration - - let data = await RandomChunker.example(blocks=8) - - # TODO: better value for data.len below. This TODO is also present in - # testproofs.nim - we may want to address it or remove the comment. - createAvailabilities(data.len * 2, duration) - - let cid = client0.upload(data).get - - let purchaseId = await client0.requestStorage( - cid, - expiry=expiry, - duration=duration, - nodes=nodes, - tolerance=tolerance, - proofProbability=proofProbability - ) - let requestId = client0.requestId(purchaseId).get - - debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - - check eventually(client0.purchaseStateIs(purchaseId, "started"), - timeout = expiry.int * 1000) - - # just to make sure we have a mined block that separates us - # from the block containing the last SlotFilled event - discard await ethProvider.send("evm_mine") - - var validators = CodexConfigs.init(nodes=2) - .withValidationGroups(groups = 2) - .withValidationGroupIndex(idx = 0, groupIndex = 0) - .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to: - # tests/integration/logs/ //_.log - .withLogTopics("validator") # each topic as a separate string argument - - failAndTeardownOnError "failed to start validator nodes": - for config in validators.configs.mitems: - let node = await startValidatorNode(config) - running.add RunningNode( - role: Role.Validator, - node: node - ) - - currentTime = await ethProvider.currentTime() - let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int - - debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - # Because of Erasure Coding, the expected number of slots being freed - # is tolerance + 1. When more than tolerance slots are freed, the whole - # request will fail. Thus, awaiting for a failing state should - # be sufficient to conclude that validators did their job correctly. - # NOTICE: We actually have to wait for the "errored" state, because - # immediately after withdrawing the funds the purchasing state machine - # transitions to the "errored" state. - check eventually(client0.purchaseStateIs(purchaseId, "errored"), - timeout = (secondsTillRequestEnd + 60) * 1000) From df1eea29a6c563db6549b5fb8717b1917bf9c628 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 22 Oct 2024 04:26:55 +0200 Subject: [PATCH 37/71] adds <> after forcing integration test to fail preliminarily --- tests/integration/testvalidator.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 914646c55..b9008f308 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -179,12 +179,14 @@ marketplacesuite "Validation": # if purchase state is not "started", it does not make sense to continue without purchaseState =? client0.getPurchase(purchaseId).?state: fail() + return debug "validation suite", purchaseState = purchaseState echo fmt"{purchaseState = }" if purchaseState != "started": fail() + return discard await ethProvider.send("evm_mine") currentTime = await ethProvider.currentTime() @@ -264,12 +266,14 @@ marketplacesuite "Validation": # if purchase state is not "started", it does not make sense to continue without purchaseState =? client0.getPurchase(purchaseId).?state: fail() + return debug "validation suite", purchaseState = purchaseState echo fmt"{purchaseState = }" if purchaseState != "started": fail() + return # extra block just to make sure we have one that separates us # from the block containing the last (past) SlotFilled event From c2eccc537453b666b70b270d2dcdd50ca8faf192 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 22 Oct 2024 04:31:31 +0200 Subject: [PATCH 38/71] re-enables all integration tests and matrix --- .github/workflows/ci.yml | 1 + tests/testIntegration.nim | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55d8ba0a6..0d27077a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} build: needs: matrix diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 91cc3630b..f0a59ee45 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -1,12 +1,12 @@ -# import ./integration/testcli -# import ./integration/testrestapi -# import ./integration/testupdownload -# import ./integration/testsales -# import ./integration/testpurchasing -# import ./integration/testblockexpiration -# import ./integration/testmarketplace -# import ./integration/testproofs +import ./integration/testcli +import ./integration/testrestapi +import ./integration/testupdownload +import ./integration/testsales +import ./integration/testpurchasing +import ./integration/testblockexpiration +import ./integration/testmarketplace +import ./integration/testproofs import ./integration/testvalidator -# import ./integration/testecbug +import ./integration/testecbug {.warning[UnusedImport]:off.} From 1a8a1481b7b4e26e7b7e8653a2354566ebddb505 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 22 Oct 2024 06:06:14 +0200 Subject: [PATCH 39/71] stops debug output in CI --- build.nims | 6 +++--- tests/integration/testvalidator.nim | 19 ++++++------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/build.nims b/build.nims index d26fd413c..a9a0e5534 100644 --- a/build.nims +++ b/build.nims @@ -40,10 +40,10 @@ task testContracts, "Build & run Codex Contract tests": task testIntegration, "Run integration tests": buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true" - # test "testIntegration" + test "testIntegration" # use params to enable logging from the integration test executable - test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & - "-d:chronicles_enabled_topics:integration:TRACE" + # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & + # "-d:chronicles_enabled_topics:integration:TRACE" task build, "build codex binary": codexTask() diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index b9008f308..a7f593758 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,5 +1,4 @@ from std/times import inMilliseconds, initDuration, inSeconds, fromUnix -import std/strformat import std/sequtils import std/sugar import pkg/codex/logutils @@ -25,7 +24,7 @@ template eventuallyS*(expression: untyped, timeout=10, step = 5, var i = 0 while not expression: inc i - echo (i*step).seconds + # echo (i*step).seconds if endTime < Moment.now(): return false if cancelExpression: @@ -117,7 +116,7 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("purchases", "onchain") .some, @@ -135,7 +134,7 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument .some @@ -171,8 +170,6 @@ marketplacesuite "Validation": debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - echo fmt"expiry = {(expiry + 60).int.seconds}" - check eventuallyS(client0.purchaseStateIs(purchaseId, "started"), timeout = (expiry + 60).int, step = 5) @@ -181,8 +178,7 @@ marketplacesuite "Validation": fail() return - debug "validation suite", purchaseState = purchaseState - echo fmt"{purchaseState = }" + debug "validation suite", purchaseState = purchaseState if purchaseState != "started": fail() @@ -214,7 +210,7 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("purchases", "onchain") .some, @@ -258,8 +254,6 @@ marketplacesuite "Validation": debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - echo fmt"expiry = {(expiry + 60).int.seconds}" - check eventuallyS(client0.purchaseStateIs(purchaseId, "started"), timeout = (expiry + 60).int, step = 5) @@ -269,7 +263,6 @@ marketplacesuite "Validation": return debug "validation suite", purchaseState = purchaseState - echo fmt"{purchaseState = }" if purchaseState != "started": fail() @@ -283,7 +276,7 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to: # tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument From 02da9defcb9ddbefe207694b17d983758e3241d8 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 22 Oct 2024 16:03:55 +0200 Subject: [PATCH 40/71] allows to choose a different RPC provider for a given integration test suite --- tests/integration/marketplacesuite.nim | 12 +++++++-- tests/integration/multinodes.nim | 34 ++++++++++++++++++++++---- tests/integration/testvalidator.nim | 15 +++++++----- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index d3b1ef577..065bab56e 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -11,9 +11,17 @@ import ../contracts/deployment export mp export multinodes -template marketplacesuite*(name: string, body: untyped) = +template marketplacesuite*(name: string, + body: untyped) = + marketplacesuiteWithProviderUrl name, "http://localhost:8545": + body + +# we can't just overload the name and use marketplacesuite here +# see: https://github.com/nim-lang/Nim/issues/14827 +template marketplacesuiteWithProviderUrl*(name: string, + jsonRpcProviderUrl: string, body: untyped) = - multinodesuite name: + multinodesuiteWithProviderUrl name, jsonRpcProviderUrl: var marketplace {.inject, used.}: Marketplace var period: uint64 diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index f29537030..22a181253 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -58,7 +58,35 @@ proc nextFreePort(startPort: int): Future[int] {.async.} = trace "port is not free", port inc port +# Following the problem described here: +# https://github.com/NomicFoundation/hardhat/issues/2053 +# It may be desireable to use http RPC provider. +# This turns out to be equally important in tests where +# subscriptions get wiped out after 5mins even when +# a new block is mined. +# For this reason, we are using http provider here as the default. +# To use a different provider in your test, you may use +# multinodesuiteWithProviderUrl template in your tests. +# The nodes are still using the default provider (which is ws://localhost:8545). +# If you want to use http provider url in the nodes, you can +# use withEthProvider config modifiers in the node configs +# to set the desired provider url. E.g.: +# NodeConfigs( +# hardhat: +# HardhatConfig.none, +# clients: +# CodexConfigs.init(nodes=1) +# .withEthProvider("http://localhost:8545") +# .some, +# ... template multinodesuite*(name: string, body: untyped) = + multinodesuiteWithProviderUrl name, "http://127.0.0.1:8545": + body + +# we can't just overload the name and use multinodesuite here +# see: https://github.com/nim-lang/Nim/issues/14827 +template multinodesuiteWithProviderUrl*(name: string, jsonRpcProviderUrl: string, + body: untyped) = asyncchecksuite name: @@ -67,10 +95,6 @@ template multinodesuite*(name: string, body: untyped) = let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") var currentTestName = "" var nodeConfigs: NodeConfigs - # Workaround for https://github.com/NomicFoundation/hardhat/issues/2053 - # Do not use websockets, but use http and polling to stop subscriptions - # from being removed after 5 minutes - let defaultProviderUrl = "http://127.0.0.1:8545" var ethProvider {.inject, used.}: JsonRpcProvider var accounts {.inject, used.}: seq[Address] var snapshot: JsonNode @@ -269,7 +293,7 @@ template multinodesuite*(name: string, body: untyped) = # Do not use websockets, but use http and polling to stop subscriptions # from being removed after 5 minutes ethProvider = JsonRpcProvider.new( - defaultProviderUrl, + jsonRpcProviderUrl, pollingInterval = chronos.milliseconds(100) ) # if hardhat was NOT started by the test, take a snapshot so it can be diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index a7f593758..6d3268801 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -15,16 +15,21 @@ export logutils logScope: topics = "integration test validation" -template eventuallyS*(expression: untyped, timeout=10, step = 5, +template eventuallyS(expression: untyped, timeout=10, step = 5, cancelExpression: untyped = false): bool = bind Moment, now, seconds proc eventuallyS: Future[bool] {.async.} = let endTime = Moment.now() + timeout.seconds var i = 0 + var secondsElapsed = 0 while not expression: inc i - # echo (i*step).seconds + secondsElapsed = i*step + # echo secondsElapsed.seconds + if secondsElapsed mod 180 == 0: + await stopTrackingEvents() + await marketplace.startTrackingEvents() if endTime < Moment.now(): return false if cancelExpression: @@ -34,13 +39,11 @@ template eventuallyS*(expression: untyped, timeout=10, step = 5, await eventuallyS() -marketplacesuite "Validation": +marketplacesuiteWithProviderUrl "Validation", "ws://localhost:8545": let nodes = 3 let tolerance = 1 let proofProbability = 1 - # var slotsAndRequests = initTable[string, seq[UInt256]]() - # var events = initTable[string, seq[ref MarketplaceEvent]]() var events = { $SlotFilled: newSeq[ref MarketplaceEvent](), $SlotFreed: newSeq[ref MarketplaceEvent](), @@ -107,7 +110,7 @@ marketplacesuite "Validation": slotsFailed.incl(slotId) debug "slots failed", slotsFailed = slotsFailed, slotsNotFreed = slotsNotFreed - check slotsNotFreed == slotsFailed + check slotsNotFreed == slotsFailed test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally From 75e86bad28ac514f48803b87746f9bed47d1bb5f Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 25 Nov 2024 03:08:39 +0100 Subject: [PATCH 41/71] fixes signature of <> method in mockProvider --- tests/contracts/helpers/mockprovider.nim | 32 +++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/contracts/helpers/mockprovider.nim b/tests/contracts/helpers/mockprovider.nim index 1ef826353..7bae34d41 100644 --- a/tests/contracts/helpers/mockprovider.nim +++ b/tests/contracts/helpers/mockprovider.nim @@ -14,20 +14,24 @@ type MockProvider* = ref object of Provider method getBlock*( provider: MockProvider, tag: BlockTag -): Future[?Block] {.async.} = - if $tag == "latest": - if latestBlock =? provider.latest: - if provider.blocks.hasKey(latestBlock): - return provider.blocks[latestBlock].some - elif $tag == "earliest": - if earliestBlock =? provider.earliest: - if provider.blocks.hasKey(earliestBlock): - return provider.blocks[earliestBlock].some - else: - let blockNumber = parseHexInt($tag) - if provider.blocks.hasKey(blockNumber): - return provider.blocks[blockNumber].some - return Block.none +): Future[?Block] {.async: (raises:[ProviderError]).} = + try: + if $tag == "latest": + if latestBlock =? provider.latest: + if provider.blocks.hasKey(latestBlock): + return provider.blocks[latestBlock].some + elif $tag == "earliest": + if earliestBlock =? provider.earliest: + if provider.blocks.hasKey(earliestBlock): + return provider.blocks[earliestBlock].some + else: + let blockNumber = parseHexInt($tag) + if provider.blocks.hasKey(blockNumber): + return provider.blocks[blockNumber].some + return Block.none + except: + return Block.none + proc updateEarliestAndLatest(provider: MockProvider, blockNumber: int) = if provider.earliest.isNone: From 5d1c1fe3867e04e28dc25e37847ed8fa02b12b30 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Tue, 26 Nov 2024 04:01:43 +0100 Subject: [PATCH 42/71] adds missing import which seem to be braking integration tests on windows --- tests/integration/testvalidator.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 6d3268801..cf458f52f 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -3,6 +3,7 @@ import std/sequtils import std/sugar import pkg/codex/logutils import pkg/questionable/results +import pkg/ethers/provider import ../contracts/time import ../contracts/deployment import ../codex/helpers From e6e0db124dd57adee2e6f5ab62ee5e30d9954a66 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 27 Nov 2024 04:49:20 +0100 Subject: [PATCH 43/71] makes sure that clients, SPs, and validators use the same provider url --- tests/integration/multinodes.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 22a181253..6d4dd4c8b 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -224,6 +224,7 @@ template multinodesuiteWithProviderUrl*(name: string, jsonRpcProviderUrl: string proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = let clientIdx = clients().len var config = conf + config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) return await newCodexProcess(clientIdx, config, Role.Client) @@ -231,6 +232,7 @@ template multinodesuiteWithProviderUrl*(name: string, jsonRpcProviderUrl: string let providerIdx = providers().len var config = conf config.addCliOption("--bootstrap-node", bootstrap) + config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) config.addCliOption(PersistenceCmd.prover, "--circom-r1cs", "vendor/codex-contracts-eth/verifier/networks/hardhat/proof_main.r1cs") @@ -245,6 +247,7 @@ template multinodesuiteWithProviderUrl*(name: string, jsonRpcProviderUrl: string let validatorIdx = validators().len var config = conf config.addCliOption("--bootstrap-node", bootstrap) + config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) config.addCliOption(StartUpCmd.persistence, "--eth-account", $accounts[running.len]) config.addCliOption(StartUpCmd.persistence, "--validator") From 9eb68a0e4906db0adb964fdbdf4e45f7a0c2e12b Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 27 Nov 2024 05:50:06 +0100 Subject: [PATCH 44/71] makes validator integration tests using http at 127.0.0.1:8545 --- tests/integration/testvalidator.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index cf458f52f..9846ec6e4 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -40,7 +40,7 @@ template eventuallyS(expression: untyped, timeout=10, step = 5, await eventuallyS() -marketplacesuiteWithProviderUrl "Validation", "ws://localhost:8545": +marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": let nodes = 3 let tolerance = 1 let proofProbability = 1 From 36ad92bd6a098e14fe8e759fba04779113b7f60c Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 27 Nov 2024 12:58:57 +0100 Subject: [PATCH 45/71] testvalidator: stop resubscribing as we are now using http polling as rpc provider --- tests/integration/testvalidator.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 9846ec6e4..8d646ebcf 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -26,11 +26,11 @@ template eventuallyS(expression: untyped, timeout=10, step = 5, var secondsElapsed = 0 while not expression: inc i - secondsElapsed = i*step - # echo secondsElapsed.seconds - if secondsElapsed mod 180 == 0: - await stopTrackingEvents() - await marketplace.startTrackingEvents() + # secondsElapsed = i*step + # # echo secondsElapsed.seconds + # if secondsElapsed mod 180 == 0: + # await stopTrackingEvents() + # await marketplace.startTrackingEvents() if endTime < Moment.now(): return false if cancelExpression: From 321212dd3ff27830e41e6daeaa550a70d911a7d9 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 28 Nov 2024 16:50:01 +0100 Subject: [PATCH 46/71] applying review comments --- codex/contracts/market.nim | 6 +++--- codex/validation.nim | 2 +- tests/integration/testvalidator.nim | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 63a37c034..36095f2cb 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -466,7 +466,7 @@ method subscribeProofSubmission*(market: OnChainMarket, method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() -proc blockNumberForBlocksEgo*(provider: Provider, +proc blockNumberForBlocksAgo*(provider: Provider, blocksAgo: int): Future[BlockTag] {.async.} = let head = await provider.getBlockNumber() return BlockTag.init(head - blocksAgo.abs.u256) @@ -599,7 +599,7 @@ method queryPastSlotFilledEvents*( convertEthersError: let fromBlock = - await blockNumberForBlocksEgo(market.contract.provider, blocksAgo) + await blockNumberForBlocksAgo(market.contract.provider, blocksAgo) return await market.queryPastSlotFilledEvents(fromBlock) @@ -628,6 +628,6 @@ method queryPastStorageRequestedEvents*( convertEthersError: let fromBlock = - await blockNumberForBlocksEgo(market.contract.provider, blocksAgo) + await blockNumberForBlocksAgo(market.contract.provider, blocksAgo) return await market.queryPastStorageRequestedEvents(fromBlock) diff --git a/codex/validation.nim b/codex/validation.nim index 2f39dbed8..d918d8f23 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -139,7 +139,7 @@ proc epochForDurationBackFromNow(validation: Validation, duration: times.Duration): SecondsSince1970 = return validation.clock.now - duration.inSeconds -proc restoreHistoricalState(validation: Validation) {.async} = +proc restoreHistoricalState(validation: Validation) {.async.} = logScope: groups = validation.config.groups groupIndex = validation.config.groupIndex diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 8d646ebcf..b59989c25 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -195,7 +195,7 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds # Because of erasure coding, after (tolerance + 1) slots are freed, the - # remaining nodes are be freed but marked as "Failed" as the whole + # remaining nodes will not be freed but marked as "Failed" as the whole # request fails. A couple of checks to capture this: let expectedSlotsFreed = tolerance + 1 check eventuallyS(checkSlotsFreed(requestId, expectedSlotsFreed), From a5a006df3277faba589ba360e38c87c7db567c8c Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 4 Dec 2024 01:19:17 +0100 Subject: [PATCH 47/71] groups queryPastStorage overrides together (review comment) --- codex/market.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codex/market.nim b/codex/market.nim index 3b2c1c406..38df96693 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -256,17 +256,17 @@ method queryPastSlotFilledEvents*( blocksAgo: int): Future[seq[SlotFilled]] {.base, async.} = raiseAssert("not implemented") -method queryPastStorageRequestedEvents*( +method queryPastSlotFilledEvents*( market: Market, - fromBlock: BlockTag): Future[seq[StorageRequested]] {.base, async.} = + fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.base, async.} = raiseAssert("not implemented") method queryPastStorageRequestedEvents*( market: Market, - blocksAgo: int): Future[seq[StorageRequested]] {.base, async.} = + fromBlock: BlockTag): Future[seq[StorageRequested]] {.base, async.} = raiseAssert("not implemented") -method queryPastSlotFilledEvents*( +method queryPastStorageRequestedEvents*( market: Market, - fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.base, async.} = + blocksAgo: int): Future[seq[StorageRequested]] {.base, async.} = raiseAssert("not implemented") From 79325c357c25da707f7be1698917494bc5fe8c78 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 4 Dec 2024 16:54:36 +0100 Subject: [PATCH 48/71] groups the historical validation tests into a sub suite --- tests/codex/testvalidation.nim | 112 ++++++++++++++++----------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/tests/codex/testvalidation.nim b/tests/codex/testvalidation.nim index 4cfcd60c9..2cfe2f063 100644 --- a/tests/codex/testvalidation.nim +++ b/tests/codex/testvalidation.nim @@ -156,68 +156,66 @@ asyncchecksuite "validation": await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) check validation.slots.len == maxSlots - test "[restoring historical state] it retrieves the historical state " & - "for max 30 days in the past": - let earlySlot = Slot.example - await market.fillSlot(earlySlot.request.id, earlySlot.slotIndex, proof, collateral) - let fromTime = clock.now() - clock.set(fromTime + 1) - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - - let duration: times.Duration = initDuration(days = 30) - clock.set(fromTime + duration.inSeconds + 1) - - validation = newValidation(clock, market, maxSlots = 0, - ValidationGroups.none) - await validation.start() - - check validation.slots == @[slot.id] - - for state in [SlotState.Finished, SlotState.Failed]: - test "[restoring historical state] when restoring historical state, " & - fmt"it excludes slots in {state} state": - let slot1 = Slot.example - let slot2 = Slot.example - await market.fillSlot(slot1.request.id, slot1.slotIndex, - proof, collateral) - await market.fillSlot(slot2.request.id, slot2.slotIndex, - proof, collateral) + suite "restoring historical state": + test "it retrieves the historical state " & + "for max 30 days in the past": + let earlySlot = Slot.example + await market.fillSlot(earlySlot.request.id, earlySlot.slotIndex, proof, collateral) + let fromTime = clock.now() + clock.set(fromTime + 1) + await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - market.slotState[slot1.id] = state + let duration: times.Duration = initDuration(days = 30) + clock.set(fromTime + duration.inSeconds + 1) validation = newValidation(clock, market, maxSlots = 0, ValidationGroups.none) await validation.start() + + check validation.slots == @[slot.id] + + for state in [SlotState.Finished, SlotState.Failed]: + test "when restoring historical state, " & + fmt"it excludes slots in {state} state": + let slot1 = Slot.example + let slot2 = Slot.example + await market.fillSlot(slot1.request.id, slot1.slotIndex, + proof, collateral) + await market.fillSlot(slot2.request.id, slot2.slotIndex, + proof, collateral) + + market.slotState[slot1.id] = state + + validation = newValidation(clock, market, maxSlots = 0, + ValidationGroups.none) + await validation.start() + + check validation.slots == @[slot2.id] + + test "it does not monitor more than the maximum number of slots ": + for _ in 0.. Date: Thu, 5 Dec 2024 03:42:35 +0100 Subject: [PATCH 49/71] removes the temporary extensions in marketplacesuite and multinodesuite allowing to specify provider url --- tests/integration/marketplacesuite.nim | 9 +---- tests/integration/multinodes.nim | 51 +++++++++++--------------- 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 065bab56e..e666ad173 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -13,15 +13,8 @@ export multinodes template marketplacesuite*(name: string, body: untyped) = - marketplacesuiteWithProviderUrl name, "http://localhost:8545": - body - -# we can't just overload the name and use marketplacesuite here -# see: https://github.com/nim-lang/Nim/issues/14827 -template marketplacesuiteWithProviderUrl*(name: string, - jsonRpcProviderUrl: string, body: untyped) = - multinodesuiteWithProviderUrl name, jsonRpcProviderUrl: + multinodesuite name: var marketplace {.inject, used.}: Marketplace var period: uint64 diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 6d4dd4c8b..9d20153cf 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -58,38 +58,31 @@ proc nextFreePort(startPort: int): Future[int] {.async.} = trace "port is not free", port inc port -# Following the problem described here: -# https://github.com/NomicFoundation/hardhat/issues/2053 -# It may be desireable to use http RPC provider. -# This turns out to be equally important in tests where -# subscriptions get wiped out after 5mins even when -# a new block is mined. -# For this reason, we are using http provider here as the default. -# To use a different provider in your test, you may use -# multinodesuiteWithProviderUrl template in your tests. -# The nodes are still using the default provider (which is ws://localhost:8545). -# If you want to use http provider url in the nodes, you can -# use withEthProvider config modifiers in the node configs -# to set the desired provider url. E.g.: -# NodeConfigs( -# hardhat: -# HardhatConfig.none, -# clients: -# CodexConfigs.init(nodes=1) -# .withEthProvider("http://localhost:8545") -# .some, -# ... template multinodesuite*(name: string, body: untyped) = - multinodesuiteWithProviderUrl name, "http://127.0.0.1:8545": - body - -# we can't just overload the name and use multinodesuite here -# see: https://github.com/nim-lang/Nim/issues/14827 -template multinodesuiteWithProviderUrl*(name: string, jsonRpcProviderUrl: string, - body: untyped) = asyncchecksuite name: - + # Following the problem described here: + # https://github.com/NomicFoundation/hardhat/issues/2053 + # It may be desirable to use http RPC provider. + # This turns out to be equally important in tests where + # subscriptions get wiped out after 5mins even when + # a new block is mined. + # For this reason, we are using http provider here as the default. + # To use a different provider in your test, you may use + # multinodesuiteWithProviderUrl template in your tests. + # The nodes are still using the default provider (which is ws://localhost:8545). + # If you want to use http provider url in the nodes, you can + # use withEthProvider config modifiers in the node configs + # to set the desired provider url. E.g.: + # NodeConfigs( + # hardhat: + # HardhatConfig.none, + # clients: + # CodexConfigs.init(nodes=1) + # .withEthProvider("http://localhost:8545") + # .some, + # ... + let jsonRpcProviderUrl = "http://127.0.0.1:8545" var running {.inject, used.}: seq[RunningNode] var bootstrap: string let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") From 580bc5c79c69e973444bdad31a724536c45d01c0 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 5 Dec 2024 03:46:32 +0100 Subject: [PATCH 50/71] simplifies validation integration tests --- tests/integration/testvalidator.nim | 165 +++++++--------------------- 1 file changed, 37 insertions(+), 128 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index b59989c25..34d1417e4 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -1,5 +1,4 @@ from std/times import inMilliseconds, initDuration, inSeconds, fromUnix -import std/sequtils import std/sugar import pkg/codex/logutils import pkg/questionable/results @@ -22,15 +21,8 @@ template eventuallyS(expression: untyped, timeout=10, step = 5, proc eventuallyS: Future[bool] {.async.} = let endTime = Moment.now() + timeout.seconds - var i = 0 var secondsElapsed = 0 while not expression: - inc i - # secondsElapsed = i*step - # # echo secondsElapsed.seconds - # if secondsElapsed mod 180 == 0: - # await stopTrackingEvents() - # await marketplace.startTrackingEvents() if endTime < Moment.now(): return false if cancelExpression: @@ -40,79 +32,31 @@ template eventuallyS(expression: untyped, timeout=10, step = 5, await eventuallyS() -marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": +marketplacesuite "Validation": let nodes = 3 let tolerance = 1 let proofProbability = 1 - var events = { - $SlotFilled: newSeq[ref MarketplaceEvent](), - $SlotFreed: newSeq[ref MarketplaceEvent](), - $RequestFailed: newSeq[ref MarketplaceEvent](), - $RequestCancelled: newSeq[ref MarketplaceEvent]() - }.toTable - var eventSubscriptions = newSeq[provider.Subscription]() - - proc box[T](x: T): ref T = - new(result); - result[] = x - - proc onMarketplaceEvent[T: MarketplaceEvent](event: T) {.gcsafe, raises:[].} = - try: - debug "onMarketplaceEvent", eventType = $T, event = event - events[$T].add(box(event)) - except KeyError: - discard - - proc startTrackingEvents(marketplace: Marketplace) {.async.} = - eventSubscriptions.add( - await marketplace.subscribe(SlotFilled, onMarketplaceEvent[SlotFilled]) - ) - eventSubscriptions.add( - await marketplace.subscribe(RequestFailed, onMarketplaceEvent[RequestFailed]) - ) - eventSubscriptions.add( - await marketplace.subscribe(SlotFreed, onMarketplaceEvent[SlotFreed]) - ) - eventSubscriptions.add( - await marketplace.subscribe(RequestCancelled, onMarketplaceEvent[RequestCancelled]) - ) - - proc stopTrackingEvents() {.async.} = - for event in eventSubscriptions: - await event.unsubscribe() - - proc checkSlotsFreed(requestId: RequestId, expectedSlotsFreed: int): bool = - events[$SlotFreed].filter( - e => (ref SlotFreed)(e).requestId == requestId) - .len == expectedSlotsFreed and - events[$RequestFailed].map( - e => (ref RequestFailed)(e).requestId).contains(requestId) - - proc isRequestCancelled(requestId: RequestId): bool = - events[$RequestCancelled].map(e => (ref RequestCancelled)(e).requestId) - .contains(requestId) - - proc getSlots[T: MarketplaceEvent](requestId: RequestId): seq[SlotId] = - events[$T].filter( - e => (ref T)(e).requestId == requestId).map( - e => slotId((ref T)(e).requestId, (ref T)(e).slotIndex)) - - proc checkSlotsFailed(marketplace: Marketplace, requestId: RequestId) {.async.} = - let slotsFreed = getSlots[SlotFreed](requestId) - let slotsFilled = getSlots[SlotFilled](requestId) - let slotsNotFreed = slotsFilled.filter( - slotId => not slotsFreed.contains(slotId) - ).toHashSet - var slotsFailed = initHashSet[SlotId]() - for slotId in slotsFilled: - let state = await marketplace.slotState(slotId) - if state == SlotState.Failed: - slotsFailed.incl(slotId) - - debug "slots failed", slotsFailed = slotsFailed, slotsNotFreed = slotsNotFreed - check slotsNotFreed == slotsFailed - + proc waitForRequestFailed( + marketplace: Marketplace, + requestId: RequestId, + timeout=10, + step = 5, + ): Future[bool] {.async.} = + let endTime = Moment.now() + timeout.seconds + + var requestState = await marketplace.requestState(requestId) + debug "waitForRequestFailed", requestId = requestId, requestState = requestState + while requestState != RequestState.Failed: + if endTime < Moment.now(): + return false + if requestState == RequestState.Cancelled: + return false + await sleepAsync(step.seconds) + requestState = await marketplace.requestState(requestId) + debug "waitForRequestFailed", requestId = requestId, requestState = requestState + return true + test "validator marks proofs as missing when using validation groups", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally hardhat: @@ -159,8 +103,6 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) - await marketplace.startTrackingEvents() - let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, @@ -174,17 +116,9 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - check eventuallyS(client0.purchaseStateIs(purchaseId, "started"), - timeout = (expiry + 60).int, step = 5) - - # if purchase state is not "started", it does not make sense to continue - without purchaseState =? client0.getPurchase(purchaseId).?state: - fail() - return - - debug "validation suite", purchaseState = purchaseState - - if purchaseState != "started": + if not eventuallyS(client0.purchaseStateIs(purchaseId, "started"), + timeout = (expiry + 60).int, step = 5): + debug "validation suite: timed out waiting for the purchase to start" fail() return @@ -193,19 +127,12 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - # Because of erasure coding, after (tolerance + 1) slots are freed, the - # remaining nodes will not be freed but marked as "Failed" as the whole - # request fails. A couple of checks to capture this: - let expectedSlotsFreed = tolerance + 1 - check eventuallyS(checkSlotsFreed(requestId, expectedSlotsFreed), - timeout = secondsTillRequestEnd + 60, step = 5, - cancelExpression = isRequestCancelled(requestId)) - - # extra check - await marketplace.checkSlotsFailed(requestId) - await stopTrackingEvents() + check await marketplace.waitForRequestFailed( + requestId, + timeout = secondsTillRequestEnd + 60, + step = 5 + ) test "validator uses historical state to mark missing proofs", NodeConfigs( # Uncomment to start Hardhat automatically, typically so logs can be inspected locally @@ -243,8 +170,6 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": # testproofs.nim - we may want to address it or remove the comment. createAvailabilities(data.len * 2, duration) - await marketplace.startTrackingEvents() - let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, @@ -258,17 +183,9 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - check eventuallyS(client0.purchaseStateIs(purchaseId, "started"), - timeout = (expiry + 60).int, step = 5) - - # if purchase state is not "started", it does not make sense to continue - without purchaseState =? client0.getPurchase(purchaseId).?state: - fail() - return - - debug "validation suite", purchaseState = purchaseState - - if purchaseState != "started": + if not eventuallyS(client0.purchaseStateIs(purchaseId, "started"), + timeout = (expiry + 60).int, step = 5): + debug "validation suite: timed out waiting for the purchase to start" fail() return @@ -297,17 +214,9 @@ marketplacesuiteWithProviderUrl "Validation", "http://127.0.0.1:8545": let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - # Because of erasure coding, after (tolerance + 1) slots are freed, the - # remaining nodes are be freed but marked as "Failed" as the whole - # request fails. A couple of checks to capture this: - let expectedSlotsFreed = tolerance + 1 - - check eventuallyS(checkSlotsFreed(requestId, expectedSlotsFreed), - timeout = secondsTillRequestEnd + 60, step = 5, - cancelExpression = isRequestCancelled(requestId)) - - # extra check - await marketplace.checkSlotsFailed(requestId) - await stopTrackingEvents() + check await marketplace.waitForRequestFailed( + requestId, + timeout = secondsTillRequestEnd + 60, + step = 5 + ) From bb784a5d85ae1c594931040b2f462962eaf0aa3e Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 5 Dec 2024 03:50:52 +0100 Subject: [PATCH 51/71] Removes debug logs when waiting for request to fail --- tests/integration/testvalidator.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 34d1417e4..6d4ae0f9f 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -46,7 +46,6 @@ marketplacesuite "Validation": let endTime = Moment.now() + timeout.seconds var requestState = await marketplace.requestState(requestId) - debug "waitForRequestFailed", requestId = requestId, requestState = requestState while requestState != RequestState.Failed: if endTime < Moment.now(): return false @@ -54,7 +53,6 @@ marketplacesuite "Validation": return false await sleepAsync(step.seconds) requestState = await marketplace.requestState(requestId) - debug "waitForRequestFailed", requestId = requestId, requestState = requestState return true test "validator marks proofs as missing when using validation groups", NodeConfigs( From a9a1f500c0a422c50d0066123928415a21d2d996 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 5 Dec 2024 03:54:35 +0100 Subject: [PATCH 52/71] Renaming waitForRequestFailed => waitForRequestToFail --- tests/integration/testvalidator.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 6d4ae0f9f..0f6674d98 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -37,7 +37,7 @@ marketplacesuite "Validation": let tolerance = 1 let proofProbability = 1 - proc waitForRequestFailed( + proc waitForRequestToFail( marketplace: Marketplace, requestId: RequestId, timeout=10, @@ -126,7 +126,7 @@ marketplacesuite "Validation": debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - check await marketplace.waitForRequestFailed( + check await marketplace.waitForRequestToFail( requestId, timeout = secondsTillRequestEnd + 60, step = 5 @@ -213,7 +213,7 @@ marketplacesuite "Validation": debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - check await marketplace.waitForRequestFailed( + check await marketplace.waitForRequestToFail( requestId, timeout = secondsTillRequestEnd + 60, step = 5 From 044251ec62f54428ba0a77cb0bab10fd09fca424 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sat, 7 Dec 2024 19:34:08 +0100 Subject: [PATCH 53/71] renames blockNumberForBlocksAgo to pastBlockTag and makes it private --- codex/contracts/market.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 36095f2cb..3bb5a603a 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -466,7 +466,7 @@ method subscribeProofSubmission*(market: OnChainMarket, method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() -proc blockNumberForBlocksAgo*(provider: Provider, +proc pastBlockTag(provider: Provider, blocksAgo: int): Future[BlockTag] {.async.} = let head = await provider.getBlockNumber() return BlockTag.init(head - blocksAgo.abs.u256) @@ -599,7 +599,7 @@ method queryPastSlotFilledEvents*( convertEthersError: let fromBlock = - await blockNumberForBlocksAgo(market.contract.provider, blocksAgo) + await pastBlockTag(market.contract.provider, blocksAgo) return await market.queryPastSlotFilledEvents(fromBlock) @@ -628,6 +628,6 @@ method queryPastStorageRequestedEvents*( convertEthersError: let fromBlock = - await blockNumberForBlocksAgo(market.contract.provider, blocksAgo) + await pastBlockTag(market.contract.provider, blocksAgo) return await market.queryPastStorageRequestedEvents(fromBlock) From abdf711991ee6a7db961f74f384f3704403014b7 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sat, 7 Dec 2024 19:48:26 +0100 Subject: [PATCH 54/71] removes redundant debugging logs --- codex/contracts/market.nim | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 3bb5a603a..d3e9ccc22 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -489,8 +489,6 @@ proc binarySearchFindClosestBlock*(provider: Provider, await provider.blockNumberAndTimestamp(BlockTag.init(low)) let (_, highTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(high)) - debug "[binarySearchFindClosestBlock]:", epochTime = epochTime, - lowTimestamp = lowTimestamp, highTimestamp = highTimestamp, low = low, high = high if abs(lowTimestamp.truncate(int) - epochTime) < abs(highTimestamp.truncate(int) - epochTime): return low @@ -505,12 +503,10 @@ proc binarySearchBlockNumberForEpoch*(provider: Provider, var low = earliestBlockNumber var high = latestBlockNumber - debug "[binarySearchBlockNumberForEpoch]:", low = low, high = high while low <= high: if low == 0 and high == 0: return low let mid = (low + high) div 2 - debug "[binarySearchBlockNumberForEpoch]:", low = low, mid = mid, high = high let (midBlockNumber, midBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(mid)) @@ -528,17 +524,11 @@ proc binarySearchBlockNumberForEpoch*(provider: Provider, proc blockNumberForEpoch*(provider: Provider, epochTime: SecondsSince1970): Future[UInt256] {.async.} = - debug "[blockNumberForEpoch]:", epochTime = epochTime let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) let (earliestBlockNumber, earliestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.earliest) - - debug "[blockNumberForEpoch]:", latestBlockNumber = latestBlockNumber, - latestBlockTimestamp = latestBlockTimestamp - debug "[blockNumberForEpoch]:", earliestBlockNumber = earliestBlockNumber, - earliestBlockTimestamp = earliestBlockTimestamp # Initially we used the average block time to predict # the number of blocks we need to look back in order to find @@ -610,7 +600,6 @@ method queryPastSlotFilledEvents*( convertEthersError: let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime) - debug "[queryPastSlotFilledEvents]", fromTime=fromTime, fromBlock=parseHexInt($fromBlock) return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock)) method queryPastStorageRequestedEvents*( From d7736f113411becb74c3299541cbc6a891011009 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sun, 8 Dec 2024 02:18:42 +0100 Subject: [PATCH 55/71] refines logging in validation --- codex/validation.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index d918d8f23..f971a908b 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -132,8 +132,7 @@ proc run(validation: Validation) {.async.} = trace "Validation stopped" discard except CatchableError as e: - error "Validation failed", msg = e.msg, groups = validation.config.groups, - groupIndex = validation.config.groupIndex + error "Validation failed", msg = e.msg proc epochForDurationBackFromNow(validation: Validation, duration: times.Duration): SecondsSince1970 = @@ -147,7 +146,7 @@ proc restoreHistoricalState(validation: Validation) {.async.} = let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) - trace "Found slot filled events", numberOfSlots = slotFilledEvents.len + trace "Found filled slots", numberOfSlots = slotFilledEvents.len for event in slotFilledEvents: let slotId = slotId(event.requestId, event.slotIndex) if validation.shouldValidateSlot(slotId): From bccfcc4291785ffcdceb89c37b9592ed3b995155 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sun, 8 Dec 2024 02:23:56 +0100 Subject: [PATCH 56/71] removes dev logging from mockmarket --- tests/codex/helpers/mockmarket.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 25a021d83..7a5c94b8c 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -518,12 +518,8 @@ method queryPastSlotFilledEvents*( method queryPastSlotFilledEvents*( market: MockMarket, fromTime: SecondsSince1970): Future[seq[SlotFilled]] {.async.} = - debug "queryPastSlotFilledEvents:market.filled", - numOfFilledSlots = market.filled.len let filtered = market.filled.filter( proc (slot: MockSlot): bool = - debug "queryPastSlotFilledEvents:fromTime", timestamp = slot.timestamp, - fromTime = fromTime if timestamp =? slot.timestamp: return timestamp >= fromTime else: From 9459508a4f30803ada3140cb4933d689fdd5dac7 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sun, 8 Dec 2024 03:43:46 +0100 Subject: [PATCH 57/71] improves exception handling in provider helper procs and prepares for extraction to a separate module --- codex/contracts/market.nim | 40 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index d3e9ccc22..8e0fd99c4 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -467,24 +467,27 @@ method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() proc pastBlockTag(provider: Provider, - blocksAgo: int): Future[BlockTag] {.async.} = + blocksAgo: int): + Future[BlockTag] {.async: (raises: [ProviderError]).} = let head = await provider.getBlockNumber() return BlockTag.init(head - blocksAgo.abs.u256) proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag): - Future[(UInt256, UInt256)] {.async.} = - without latestBlock =? await provider.getBlock(blockTag), error: - raise error + Future[(UInt256, UInt256)] {.async: (raises: [ProviderError, MarketError]).} = + without latestBlock =? await provider.getBlock(blockTag): + raiseMarketError("Could not get latest block") without latestBlockNumber =? latestBlock.number: - raise newException(EthersError, "Could not get latest block number") + raiseMarketError("Could not get latest block number") - (latestBlockNumber, latestBlock.timestamp) + return (latestBlockNumber, latestBlock.timestamp) -proc binarySearchFindClosestBlock*(provider: Provider, - epochTime: int, - low: UInt256, - high: UInt256): Future[UInt256] {.async.} = +proc binarySearchFindClosestBlock( + provider: Provider, + epochTime: int, + low: UInt256, + high: UInt256): + Future[UInt256] {.async: (raises: [ProviderError, MarketError]).} = let (_, lowTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(low)) let (_, highTimestamp) = @@ -495,11 +498,12 @@ proc binarySearchFindClosestBlock*(provider: Provider, else: return high -proc binarySearchBlockNumberForEpoch*(provider: Provider, - epochTime: UInt256, - latestBlockNumber: UInt256, - earliestBlockNumber: UInt256): - Future[UInt256] {.async.} = +proc binarySearchBlockNumberForEpoch( + provider: Provider, + epochTime: UInt256, + latestBlockNumber: UInt256, + earliestBlockNumber: UInt256): + Future[UInt256] {.async: (raises: [ProviderError, MarketError]).} = var low = earliestBlockNumber var high = latestBlockNumber @@ -522,8 +526,10 @@ proc binarySearchBlockNumberForEpoch*(provider: Provider, await provider.binarySearchFindClosestBlock( epochTime.truncate(int), low=high, high=low) -proc blockNumberForEpoch*(provider: Provider, - epochTime: SecondsSince1970): Future[UInt256] {.async.} = +proc blockNumberForEpoch*( + provider: Provider, + epochTime: SecondsSince1970): + Future[UInt256] {.async: (raises: [ProviderError, MarketError]).} = let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) From f57d6e362a5ce6ed02208d08a787cf6c2ecee9ac Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sun, 8 Dec 2024 20:49:36 +0100 Subject: [PATCH 58/71] Uses chronos instead of std/times for Duration --- codex/validation.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index f971a908b..a538b6455 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -1,4 +1,3 @@ -import std/times import std/sets import std/sequtils import pkg/chronos @@ -25,7 +24,7 @@ type config: ValidationConfig const - MaxStorageRequestDuration: times.Duration = initDuration(days = 30) + MaxStorageRequestDuration = 30.days logScope: topics = "codex validator" @@ -135,8 +134,8 @@ proc run(validation: Validation) {.async.} = error "Validation failed", msg = e.msg proc epochForDurationBackFromNow(validation: Validation, - duration: times.Duration): SecondsSince1970 = - return validation.clock.now - duration.inSeconds + duration: Duration): SecondsSince1970 = + return validation.clock.now - duration.secs proc restoreHistoricalState(validation: Validation) {.async.} = logScope: From dc1869a4f8c967dccdb96dde21a8439143388d31 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Sun, 8 Dec 2024 21:09:15 +0100 Subject: [PATCH 59/71] extracts provider and binary search helpers to a separate module --- codex/contracts.nim | 2 + codex/contracts/market.nim | 116 +--------------------- codex/contracts/provider.nim | 131 +++++++++++++++++++++++++ tests/contracts/testProvider.nim | 161 +++++++++++++++++++++++++++++++ 4 files changed, 295 insertions(+), 115 deletions(-) create mode 100644 codex/contracts/provider.nim create mode 100644 tests/contracts/testProvider.nim diff --git a/codex/contracts.nim b/codex/contracts.nim index ecf298f4e..512571a81 100644 --- a/codex/contracts.nim +++ b/codex/contracts.nim @@ -2,8 +2,10 @@ import contracts/requests import contracts/marketplace import contracts/market import contracts/interactions +import contracts/provider export requests export marketplace export market export interactions +export provider diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 8e0fd99c4..d9e7e232f 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,5 +1,4 @@ import std/strutils -import std/times import pkg/ethers import pkg/upraises import pkg/questionable @@ -8,6 +7,7 @@ import ../logutils import ../market import ./marketplace import ./proofs +import ./provider export market @@ -466,120 +466,6 @@ method subscribeProofSubmission*(market: OnChainMarket, method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = await subscription.eventSubscription.unsubscribe() -proc pastBlockTag(provider: Provider, - blocksAgo: int): - Future[BlockTag] {.async: (raises: [ProviderError]).} = - let head = await provider.getBlockNumber() - return BlockTag.init(head - blocksAgo.abs.u256) - -proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag): - Future[(UInt256, UInt256)] {.async: (raises: [ProviderError, MarketError]).} = - without latestBlock =? await provider.getBlock(blockTag): - raiseMarketError("Could not get latest block") - - without latestBlockNumber =? latestBlock.number: - raiseMarketError("Could not get latest block number") - - return (latestBlockNumber, latestBlock.timestamp) - -proc binarySearchFindClosestBlock( - provider: Provider, - epochTime: int, - low: UInt256, - high: UInt256): - Future[UInt256] {.async: (raises: [ProviderError, MarketError]).} = - let (_, lowTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.init(low)) - let (_, highTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.init(high)) - if abs(lowTimestamp.truncate(int) - epochTime) < - abs(highTimestamp.truncate(int) - epochTime): - return low - else: - return high - -proc binarySearchBlockNumberForEpoch( - provider: Provider, - epochTime: UInt256, - latestBlockNumber: UInt256, - earliestBlockNumber: UInt256): - Future[UInt256] {.async: (raises: [ProviderError, MarketError]).} = - var low = earliestBlockNumber - var high = latestBlockNumber - - while low <= high: - if low == 0 and high == 0: - return low - let mid = (low + high) div 2 - let (midBlockNumber, midBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.init(mid)) - - if midBlockTimestamp < epochTime: - low = mid + 1 - elif midBlockTimestamp > epochTime: - high = mid - 1 - else: - return midBlockNumber - # NOTICE that by how the binaty search is implemented, when it finishes - # low is always greater than high - this is why we return high, where - # intuitively we would return low. - await provider.binarySearchFindClosestBlock( - epochTime.truncate(int), low=high, high=low) - -proc blockNumberForEpoch*( - provider: Provider, - epochTime: SecondsSince1970): - Future[UInt256] {.async: (raises: [ProviderError, MarketError]).} = - let epochTimeUInt256 = epochTime.u256 - let (latestBlockNumber, latestBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.latest) - let (earliestBlockNumber, earliestBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.earliest) - - # Initially we used the average block time to predict - # the number of blocks we need to look back in order to find - # the block number corresponding to the given epoch time. - # This estimation can be highly inaccurate if block time - # was changing in the past or is fluctuating and therefore - # we used that information initially only to find out - # if the available history is long enough to perform effective search. - # It turns out we do not have to do that. There is an easier way. - # - # First we check if the given epoch time equals the timestamp of either - # the earliest or the latest block. If it does, we just return the - # block number of that block. - # - # Otherwise, if the earliest available block is not the genesis block, - # we should check the timestamp of that earliest block and if it is greater - # than the epoch time, we should issue a warning and return - # that earliest block number. - # In all other cases, thus when the earliest block is not the genesis - # block but its timestamp is not greater than the requested epoch time, or - # if the earliest available block is the genesis block, - # (which means we have the whole history available), we should proceed with - # the binary search. - # - # Additional benefit of this method is that we do not have to rely - # on the average block time, which not only makes the whole thing - # more reliable, but also easier to test. - - # Are lucky today? - if earliestBlockTimestamp == epochTimeUInt256: - return earliestBlockNumber - if latestBlockTimestamp == epochTimeUInt256: - return latestBlockNumber - - if earliestBlockNumber > 0 and earliestBlockTimestamp > epochTimeUInt256: - let availableHistoryInDays = - (latestBlockTimestamp - earliestBlockTimestamp) div - initDuration(days = 1).inSeconds.u256 - warn "Short block history detected.", earliestBlockTimestamp = - earliestBlockTimestamp, days = availableHistoryInDays - return earliestBlockNumber - - return await provider.binarySearchBlockNumberForEpoch( - epochTimeUInt256, latestBlockNumber, earliestBlockNumber) - method queryPastSlotFilledEvents*( market: OnChainMarket, fromBlock: BlockTag): Future[seq[SlotFilled]] {.async.} = diff --git a/codex/contracts/provider.nim b/codex/contracts/provider.nim new file mode 100644 index 000000000..397d0ed3d --- /dev/null +++ b/codex/contracts/provider.nim @@ -0,0 +1,131 @@ +import pkg/ethers/provider +import pkg/chronos +import pkg/questionable + +import ../logutils + +from ../errors import CodexError +from ../clock import SecondsSince1970 + +logScope: + topics = "marketplace onchain provider" + +type CodexProviderError* = object of CodexError + +proc raiseCodexProviderError(message: string) {.raises: [CodexProviderError].} = + raise newException(CodexProviderError, message) + +proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag): + Future[(UInt256, UInt256)] + {.async: (raises: [ProviderError, CodexProviderError]).} = + without latestBlock =? await provider.getBlock(blockTag): + raiseCodexProviderError("Could not get latest block") + + without latestBlockNumber =? latestBlock.number: + raiseCodexProviderError("Could not get latest block number") + + return (latestBlockNumber, latestBlock.timestamp) + +proc binarySearchFindClosestBlock( + provider: Provider, + epochTime: int, + low: UInt256, + high: UInt256): Future[UInt256] + {.async: (raises: [ProviderError, CodexProviderError]).} = + let (_, lowTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.init(low)) + let (_, highTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.init(high)) + if abs(lowTimestamp.truncate(int) - epochTime) < + abs(highTimestamp.truncate(int) - epochTime): + return low + else: + return high + +proc binarySearchBlockNumberForEpoch( + provider: Provider, + epochTime: UInt256, + latestBlockNumber: UInt256, + earliestBlockNumber: UInt256): Future[UInt256] + {.async: (raises: [ProviderError, CodexProviderError]).} = + var low = earliestBlockNumber + var high = latestBlockNumber + + while low <= high: + if low == 0 and high == 0: + return low + let mid = (low + high) div 2 + let (midBlockNumber, midBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.init(mid)) + + if midBlockTimestamp < epochTime: + low = mid + 1 + elif midBlockTimestamp > epochTime: + high = mid - 1 + else: + return midBlockNumber + # NOTICE that by how the binary search is implemented, when it finishes + # low is always greater than high - this is why we use high, where + # intuitively we would use low: + await provider.binarySearchFindClosestBlock( + epochTime.truncate(int), low=high, high=low) + +proc blockNumberForEpoch*( + provider: Provider, + epochTime: SecondsSince1970): Future[UInt256] + {.async: (raises: [ProviderError, CodexProviderError]).} = + let epochTimeUInt256 = epochTime.u256 + let (latestBlockNumber, latestBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.latest) + let (earliestBlockNumber, earliestBlockTimestamp) = + await provider.blockNumberAndTimestamp(BlockTag.earliest) + + # Initially we used the average block time to predict + # the number of blocks we need to look back in order to find + # the block number corresponding to the given epoch time. + # This estimation can be highly inaccurate if block time + # was changing in the past or is fluctuating and therefore + # we used that information initially only to find out + # if the available history is long enough to perform effective search. + # It turns out we do not have to do that. There is an easier way. + # + # First we check if the given epoch time equals the timestamp of either + # the earliest or the latest block. If it does, we just return the + # block number of that block. + # + # Otherwise, if the earliest available block is not the genesis block, + # we should check the timestamp of that earliest block and if it is greater + # than the epoch time, we should issue a warning and return + # that earliest block number. + # In all other cases, thus when the earliest block is not the genesis + # block but its timestamp is not greater than the requested epoch time, or + # if the earliest available block is the genesis block, + # (which means we have the whole history available), we should proceed with + # the binary search. + # + # Additional benefit of this method is that we do not have to rely + # on the average block time, which not only makes the whole thing + # more reliable, but also easier to test. + + # Are lucky today? + if earliestBlockTimestamp == epochTimeUInt256: + return earliestBlockNumber + if latestBlockTimestamp == epochTimeUInt256: + return latestBlockNumber + + if earliestBlockNumber > 0 and earliestBlockTimestamp > epochTimeUInt256: + let availableHistoryInDays = + (latestBlockTimestamp - earliestBlockTimestamp) div + 1.days.secs.u256 + warn "Short block history detected.", earliestBlockTimestamp = + earliestBlockTimestamp, days = availableHistoryInDays + return earliestBlockNumber + + return await provider.binarySearchBlockNumberForEpoch( + epochTimeUInt256, latestBlockNumber, earliestBlockNumber) + +proc pastBlockTag*(provider: Provider, + blocksAgo: int): + Future[BlockTag] {.async: (raises: [ProviderError]).} = + let head = await provider.getBlockNumber() + return BlockTag.init(head - blocksAgo.abs.u256) diff --git a/tests/contracts/testProvider.nim b/tests/contracts/testProvider.nim new file mode 100644 index 000000000..1321e25fb --- /dev/null +++ b/tests/contracts/testProvider.nim @@ -0,0 +1,161 @@ +import pkg/chronos +import codex/contracts +import ../ethertest +import ./time +import ./helpers/mockprovider + +# to see supportive information in the test output +# use `-d:"chronicles_enabled_topics:testProvider:DEBUG` option +# when compiling the test file +logScope: + topics = "testProvider" + +ethersuite "Provider": + proc mineNBlocks(provider: JsonRpcProvider, n: int) {.async.} = + for _ in 0.. 291 + # 1728436104 => 291 + # 1728436105 => 292 + # 1728436106 => 292 + # 1728436110 => 292 + proc generateExpectations( + blocks: seq[(UInt256, UInt256)]): seq[Expectations] = + var expectations: seq[Expectations] = @[] + for i in 0.. Date: Sun, 8 Dec 2024 21:43:37 +0100 Subject: [PATCH 60/71] removes redundant log entry params from validator --- codex/validation.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index a538b6455..ce61cc492 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -46,7 +46,8 @@ proc getCurrentPeriod(validation: Validation): UInt256 = proc waitUntilNextPeriod(validation: Validation) {.async.} = let period = validation.getCurrentPeriod() let periodEnd = validation.periodicity.periodEnd(period) - trace "Waiting until next period", currentPeriod = period, groups = validation.config.groups, + trace "Waiting until next period", currentPeriod = period, + groups = validation.config.groups, groupIndex = validation.config.groupIndex await validation.clock.waitUntil(periodEnd.truncate(int64) + 1) @@ -120,8 +121,7 @@ proc run(validation: Validation) {.async.} = logScope: groups = validation.config.groups groupIndex = validation.config.groupIndex - trace "Validation started", currentTime = validation.clock.now, - currentTime = validation.clock.now.fromUnix + trace "Validation started" try: while true: await validation.waitUntilNextPeriod() From a09f07dffc41ab0c945652e68343a38187bcca9e Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 11 Dec 2024 23:46:20 +0100 Subject: [PATCH 61/71] unifies the notation to consistently use method call syntax --- codex/contracts/market.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index d9e7e232f..a06a9486f 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -481,7 +481,7 @@ method queryPastSlotFilledEvents*( convertEthersError: let fromBlock = - await pastBlockTag(market.contract.provider, blocksAgo) + await market.contract.provider.pastBlockTag(blocksAgo) return await market.queryPastSlotFilledEvents(fromBlock) @@ -509,6 +509,6 @@ method queryPastStorageRequestedEvents*( convertEthersError: let fromBlock = - await pastBlockTag(market.contract.provider, blocksAgo) + await market.contract.provider.pastBlockTag(blocksAgo) return await market.queryPastStorageRequestedEvents(fromBlock) From 14c308d7f6b088f34e8c56739cc325105f030fed Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 11 Dec 2024 23:50:31 +0100 Subject: [PATCH 62/71] reuses ProviderError from nim-ethers in the provider extension --- codex/contracts/provider.nim | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/codex/contracts/provider.nim b/codex/contracts/provider.nim index 397d0ed3d..62098fb5f 100644 --- a/codex/contracts/provider.nim +++ b/codex/contracts/provider.nim @@ -4,25 +4,21 @@ import pkg/questionable import ../logutils -from ../errors import CodexError from ../clock import SecondsSince1970 logScope: topics = "marketplace onchain provider" -type CodexProviderError* = object of CodexError - -proc raiseCodexProviderError(message: string) {.raises: [CodexProviderError].} = - raise newException(CodexProviderError, message) +proc raiseProviderError(message: string) {.raises: [ProviderError].} = + raise newException(ProviderError, message) proc blockNumberAndTimestamp*(provider: Provider, blockTag: BlockTag): - Future[(UInt256, UInt256)] - {.async: (raises: [ProviderError, CodexProviderError]).} = + Future[(UInt256, UInt256)] {.async: (raises: [ProviderError]).} = without latestBlock =? await provider.getBlock(blockTag): - raiseCodexProviderError("Could not get latest block") + raiseProviderError("Could not get latest block") without latestBlockNumber =? latestBlock.number: - raiseCodexProviderError("Could not get latest block number") + raiseProviderError("Could not get latest block number") return (latestBlockNumber, latestBlock.timestamp) @@ -30,8 +26,7 @@ proc binarySearchFindClosestBlock( provider: Provider, epochTime: int, low: UInt256, - high: UInt256): Future[UInt256] - {.async: (raises: [ProviderError, CodexProviderError]).} = + high: UInt256): Future[UInt256] {.async: (raises: [ProviderError]).} = let (_, lowTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(low)) let (_, highTimestamp) = @@ -47,7 +42,7 @@ proc binarySearchBlockNumberForEpoch( epochTime: UInt256, latestBlockNumber: UInt256, earliestBlockNumber: UInt256): Future[UInt256] - {.async: (raises: [ProviderError, CodexProviderError]).} = + {.async: (raises: [ProviderError]).} = var low = earliestBlockNumber var high = latestBlockNumber @@ -73,7 +68,7 @@ proc binarySearchBlockNumberForEpoch( proc blockNumberForEpoch*( provider: Provider, epochTime: SecondsSince1970): Future[UInt256] - {.async: (raises: [ProviderError, CodexProviderError]).} = + {.async: (raises: [ProviderError]).} = let epochTimeUInt256 = epochTime.u256 let (latestBlockNumber, latestBlockTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.latest) From 539877add2184d8828e0db32ce64a68636417857 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 12 Dec 2024 16:18:10 +0100 Subject: [PATCH 63/71] clarifies the comment in multinodesuite --- tests/integration/multinodes.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 9d20153cf..da516f3ba 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -70,16 +70,15 @@ template multinodesuite*(name: string, body: untyped) = # For this reason, we are using http provider here as the default. # To use a different provider in your test, you may use # multinodesuiteWithProviderUrl template in your tests. - # The nodes are still using the default provider (which is ws://localhost:8545). - # If you want to use http provider url in the nodes, you can - # use withEthProvider config modifiers in the node configs + # If you want to use a different provider url in the nodes, you can + # use withEthProvider config modifier in the node config # to set the desired provider url. E.g.: # NodeConfigs( # hardhat: # HardhatConfig.none, # clients: # CodexConfigs.init(nodes=1) - # .withEthProvider("http://localhost:8545") + # .withEthProvider("ws://localhost:8545") # .some, # ... let jsonRpcProviderUrl = "http://127.0.0.1:8545" From fa9d6e1ba8a48fe89ee7543827ddaefc1c1e6217 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 12 Dec 2024 16:44:39 +0100 Subject: [PATCH 64/71] uses == operator to check the predefined tags and raises exception when `BlockTag.pending` is requested. --- tests/contracts/helpers/mockprovider.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/contracts/helpers/mockprovider.nim b/tests/contracts/helpers/mockprovider.nim index 7bae34d41..ce6e9e346 100644 --- a/tests/contracts/helpers/mockprovider.nim +++ b/tests/contracts/helpers/mockprovider.nim @@ -16,14 +16,16 @@ method getBlock*( tag: BlockTag ): Future[?Block] {.async: (raises:[ProviderError]).} = try: - if $tag == "latest": + if tag == BlockTag.latest: if latestBlock =? provider.latest: if provider.blocks.hasKey(latestBlock): return provider.blocks[latestBlock].some - elif $tag == "earliest": + elif tag == BlockTag.earliest: if earliestBlock =? provider.earliest: if provider.blocks.hasKey(earliestBlock): return provider.blocks[earliestBlock].some + elif tag == BlockTag.pending: + raiseAssert "MockProvider does not yet support BlockTag.pending" else: let blockNumber = parseHexInt($tag) if provider.blocks.hasKey(blockNumber): From c9f66a0b9326151e87dc7e20b494e94f3994a335 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 12 Dec 2024 16:50:28 +0100 Subject: [PATCH 65/71] when waiting for request to fail, we break on any request state that is not Started --- tests/integration/testvalidator.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 0f6674d98..f010a23fd 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -49,7 +49,7 @@ marketplacesuite "Validation": while requestState != RequestState.Failed: if endTime < Moment.now(): return false - if requestState == RequestState.Cancelled: + if requestState != RequestState.Started: return false await sleepAsync(step.seconds) requestState = await marketplace.requestState(requestId) From 19e576e9e7d80d2d85fe9154d28098db74d41d59 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 12 Dec 2024 17:43:50 +0100 Subject: [PATCH 66/71] removes tests that were moved to testProvider from testMarket --- tests/contracts/testMarket.nim | 150 --------------------------------- 1 file changed, 150 deletions(-) diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 509a11e8a..d3423f965 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -64,10 +64,6 @@ ethersuite "On-Chain Market": proc advanceToCancelledRequest(request: StorageRequest) {.async.} = let expiry = (await market.requestExpiresAt(request.id)) + 1 await ethProvider.advanceTimeTo(expiry.u256) - - proc mineNBlocks(provider: JsonRpcProvider, n: int) {.async.} = - for _ in 0.. 291 - # 1728436104 => 291 - # 1728436105 => 292 - # 1728436106 => 292 - # 1728436110 => 292 - proc generateExpectations( - blocks: seq[(UInt256, UInt256)]): seq[Expectations] = - var expectations: seq[Expectations] = @[] - for i in 0.. Date: Thu, 12 Dec 2024 18:01:44 +0100 Subject: [PATCH 67/71] extracts tests that use MockProvider to a separate async suite --- tests/contracts/testProvider.nim | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/contracts/testProvider.nim b/tests/contracts/testProvider.nim index 1321e25fb..cf28b92cc 100644 --- a/tests/contracts/testProvider.nim +++ b/tests/contracts/testProvider.nim @@ -1,5 +1,6 @@ import pkg/chronos import codex/contracts +import ../asynctest import ../ethertest import ./time import ./helpers/mockprovider @@ -10,11 +11,7 @@ import ./helpers/mockprovider logScope: topics = "testProvider" -ethersuite "Provider": - proc mineNBlocks(provider: JsonRpcProvider, n: int) {.async.} = - for _ in 0.. Date: Thu, 12 Dec 2024 18:07:08 +0100 Subject: [PATCH 68/71] improves performance of the historical state restoration --- codex/validation.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index ce61cc492..6b5ac447a 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -61,15 +61,15 @@ func maxSlotsConstraintRespected(validation: Validation): bool = validation.slots.len < validation.config.maxSlots func shouldValidateSlot(validation: Validation, slotId: SlotId): bool = - if (validationGroups =? validation.config.groups): - (groupIndexForSlotId(slotId, validationGroups) == - validation.config.groupIndex) and - validation.maxSlotsConstraintRespected - else: - validation.maxSlotsConstraintRespected + without validationGroups =? validation.config.groups: + return true + groupIndexForSlotId(slotId, validationGroups) == + validation.config.groupIndex proc subscribeSlotFilled(validation: Validation) {.async.} = proc onSlotFilled(requestId: RequestId, slotIndex: UInt256) = + if not validation.maxSlotsConstraintRespected: + return let slotId = slotId(requestId, slotIndex) if validation.shouldValidateSlot(slotId): trace "Adding slot", slotId, groups = validation.config.groups, @@ -145,14 +145,14 @@ proc restoreHistoricalState(validation: Validation) {.async.} = let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( fromTime = startTimeEpoch) - trace "Found filled slots", numberOfSlots = slotFilledEvents.len for event in slotFilledEvents: + if not validation.maxSlotsConstraintRespected: + break let slotId = slotId(event.requestId, event.slotIndex) - if validation.shouldValidateSlot(slotId): + let slotState = await validation.market.slotState(slotId) + if slotState == SlotState.Filled and validation.shouldValidateSlot(slotId): trace "Adding slot [historical]", slotId validation.slots.incl(slotId) - trace "Removing slots that have ended..." - await removeSlotsThatHaveEnded(validation) trace "Historical state restored", numberOfSlots = validation.slots.len proc start*(validation: Validation) {.async.} = From 5c48ca47a92a6635c47885e52b4ca342f500e18a Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 13 Dec 2024 22:46:06 +0100 Subject: [PATCH 69/71] removing redundant log messages in validator (groupIndex and groups) --- codex/validation.nim | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/codex/validation.nim b/codex/validation.nim index 6b5ac447a..cc663c5f0 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -46,9 +46,7 @@ proc getCurrentPeriod(validation: Validation): UInt256 = proc waitUntilNextPeriod(validation: Validation) {.async.} = let period = validation.getCurrentPeriod() let periodEnd = validation.periodicity.periodEnd(period) - trace "Waiting until next period", currentPeriod = period, - groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Waiting until next period", currentPeriod = period await validation.clock.waitUntil(periodEnd.truncate(int64) + 1) func groupIndexForSlotId*(slotId: SlotId, @@ -72,16 +70,12 @@ proc subscribeSlotFilled(validation: Validation) {.async.} = return let slotId = slotId(requestId, slotIndex) if validation.shouldValidateSlot(slotId): - trace "Adding slot", slotId, groups = validation.config.groups, - groupIndex = validation.config.groupIndex + trace "Adding slot", slotId validation.slots.incl(slotId) let subscription = await validation.market.subscribeSlotFilled(onSlotFilled) validation.subscriptions.add(subscription) proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = - logScope: - groups = validation.config.groups - groupIndex = validation.config.groupIndex var ended: HashSet[SlotId] let slots = validation.slots for slotId in slots: @@ -96,8 +90,6 @@ proc markProofAsMissing(validation: Validation, period: Period) {.async.} = logScope: currentPeriod = validation.getCurrentPeriod() - groups = validation.config.groups - groupIndex = validation.config.groupIndex try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): @@ -118,9 +110,6 @@ proc markProofsAsMissing(validation: Validation) {.async.} = await validation.markProofAsMissing(slotId, previousPeriod) proc run(validation: Validation) {.async.} = - logScope: - groups = validation.config.groups - groupIndex = validation.config.groupIndex trace "Validation started" try: while true: @@ -138,9 +127,6 @@ proc epochForDurationBackFromNow(validation: Validation, return validation.clock.now - duration.secs proc restoreHistoricalState(validation: Validation) {.async.} = - logScope: - groups = validation.config.groups - groupIndex = validation.config.groupIndex trace "Restoring historical state..." let startTimeEpoch = validation.epochForDurationBackFromNow(MaxStorageRequestDuration) let slotFilledEvents = await validation.market.queryPastSlotFilledEvents( @@ -156,6 +142,8 @@ proc restoreHistoricalState(validation: Validation) {.async.} = trace "Historical state restored", numberOfSlots = validation.slots.len proc start*(validation: Validation) {.async.} = + trace "Starting validator", groups = validation.config.groups, + groupIndex = validation.config.groupIndex validation.periodicity = await validation.market.periodicity() validation.proofTimeout = await validation.market.proofTimeout() await validation.subscribeSlotFilled() From 217b588cedf2c8f526343e35d413b1b57512e235 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 13 Dec 2024 23:04:54 +0100 Subject: [PATCH 70/71] adds testProvider to testContracts group --- tests/testContracts.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testContracts.nim b/tests/testContracts.nim index 4283c10a6..aff2c1d77 100644 --- a/tests/testContracts.nim +++ b/tests/testContracts.nim @@ -2,5 +2,6 @@ import ./contracts/testContracts import ./contracts/testMarket import ./contracts/testDeployment import ./contracts/testClock +import ./contracts/testProvider {.warning[UnusedImport]:off.} From 8b91c6817850eef656d329f585cf4239b73cb61a Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 13 Dec 2024 23:07:34 +0100 Subject: [PATCH 71/71] removes unused import in testMarket --- tests/contracts/testMarket.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index d3423f965..c25955293 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -7,7 +7,6 @@ import ../ethertest import ./examples import ./time import ./deployment -import ./helpers/mockprovider privateAccess(OnChainMarket) # enable access to private fields