Merge pull request #202 from LN-Zap/feat/onchain-pagination-temp

Only fetch onchain transactions from LND within start_time and end_time range

accounting/config.go

@@ -97,8 +97,10 @@ type CommonConfig struct {
 // The txLookup function may be nil if a connection to a bitcoin backend is not
 // available. If this is the case, the fee report will log warnings indicating
 // that fee lookups are not possible in certain cases.
-func NewOnChainConfig(ctx context.Context, lnd lndclient.LndServices, startTime,
-	endTime time.Time, disableFiat bool, txLookup fees.GetDetailsFunc,
+func NewOnChainConfig(ctx context.Context,
+	lnd lndclient.LndServices, startTime, endTime time.Time,
+	blockRangeLookup func(start, end time.Time) (uint32, uint32, error),
+	disableFiat bool, txLookup fees.GetDetailsFunc,
 	priceCfg *fiat.PriceSourceConfig,
 	categories []CustomCategory) *OnChainConfig {
 
@@ -109,6 +111,29 @@ func NewOnChainConfig(ctx context.Context, lnd lndclient.LndServices, startTime,
 		}
 	}
 
+	// Set both start and end height to 0, meaning we will query for all
+	// onchain history.
+	startHeight := uint32(0)
+	endHeight := uint32(0)
+
+	if blockRangeLookup != nil {
+		var err error
+
+		startHeight, endHeight, err = blockRangeLookup(startTime, endTime)
+		if err != nil {
+			log.Errorf("Error finding block height range for start time: %v "+
+				"end time: %v error: %v", startTime, endTime, err)
+
+			// If we cannot find the block height range, set both start and end
+			// height to 0, meaning we will query for all onchain history.
+			startHeight = 0
+			endHeight = 0
+		}
+	}
+
+	log.Debugf("Using startheight: %v endheight: %v while querying onchain "+
+		"activity", startHeight, endHeight)
+
 	return &OnChainConfig{
 		OpenChannels: lndwrap.ListChannels(
 			ctx, lnd.Client, false,
@@ -120,10 +145,12 @@ func NewOnChainConfig(ctx context.Context, lnd lndclient.LndServices, startTime,
 			return lnd.Client.PendingChannels(ctx)
 		},
 		OnChainTransactions: func() ([]lndclient.Transaction, error) {
-			return lnd.Client.ListTransactions(ctx, 0, 0)
+			return lnd.Client.ListTransactions(
+				ctx, int32(startHeight), int32(endHeight),
+			)
 		},
 		ListSweeps: func() ([]string, error) {
-			return lnd.WalletKit.ListSweeps(ctx, 0)
+			return lnd.WalletKit.ListSweeps(ctx, int32(startHeight))
 		},
 		CommonConfig: CommonConfig{
 			StartTime:      startTime,

frdrpcserver/node_audit.go

@@ -7,14 +7,23 @@ import (
 	"sort"
 	"time"
 
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/lightninglabs/faraday/accounting"
 	"github.com/lightninglabs/faraday/fees"
 	"github.com/lightninglabs/faraday/fiat"
 	"github.com/lightninglabs/faraday/frdrpc"
+	"github.com/lightninglabs/lndclient"
 	"github.com/lightningnetwork/lnd/routing/route"
 	"github.com/shopspring/decimal"
 )
 
+// Since Bitcoin blocks are not guaranteed to be completely ordered
+// by timestamp, and the timestamps can be manipulated by miners within a
+// certain range, we will apply a buffer on the time range which we use to
+// find start and end block heights. This should ensure we widen the block
+// height range enough to fetch all relevant transactions within a time range.
+const blockTimeRangeBuffer = time.Hour * 24
+
 var (
 	// ErrNoCategoryName is returned if a category does not have a name.
 	ErrNoCategoryName = errors.New("category must have a name")
@@ -84,8 +93,21 @@ func parseNodeAuditRequest(ctx context.Context, cfg *Config,
 			"backend, some fee entries will be missing (see logs)")
 	}
 
+	var blockRangeLookup func(start, end time.Time) (uint32, uint32, error)
+
+	// If a time range is set, we will use a block height lookup function
+	// to find the block heights for the start and end time.
+	timeRangeSet := req.StartTime > 0 || req.EndTime > 0
+	if timeRangeSet {
+		blockRangeLookup = func(start, end time.Time) (uint32, uint32, error) {
+			return resolveBlockHeightRange(
+				ctx, cfg.Lnd, info.BlockHeight, start, end,
+			)
+		}
+	}
+
 	onChain := accounting.NewOnChainConfig(
-		ctx, cfg.Lnd, start, end, req.DisableFiat,
+		ctx, cfg.Lnd, start, end, blockRangeLookup, req.DisableFiat,
 		feeLookup, priceSourceCfg, onChainCategories,
 	)
 
@@ -278,3 +300,129 @@ func rpcEntryType(t accounting.EntryType) (frdrpc.EntryType, error) {
 		return 0, fmt.Errorf("unknown entrytype: %v", t)
 	}
 }
+
+// resolveBlockHeightRange determines the block height range that should be
+// used for in queries based on the start and end time of the report.
+// The function will apply a buffer to ensure the block height range is
+// too large rather than too small, so that all relevant transactions are
+// fetched from the backend.
+func resolveBlockHeightRange(ctx context.Context,
+	lndClient lndclient.LndServices, latestHeight uint32,
+	startTime, endTime time.Time) (uint32, uint32, error) {
+
+	// Apply a buffer on the start time which we use to find the block height.
+	// This should ensure we use a low enough height to fetch all relevant
+	// transactions following the start time.
+	bufferedStartTime := startTime.Add(-blockTimeRangeBuffer)
+
+	if bufferedStartTime.Before(time.Unix(0, 0)) {
+		bufferedStartTime = time.Unix(0, 0)
+	}
+
+	startHeight, err := findFirstBlockBeforeTimestamp(
+		ctx, lndClient, latestHeight, bufferedStartTime,
+	)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// Apply a buffer on the end time which we use to find the block height.
+	// This should ensure we use a high enough height to fetch all relevant
+	// transactions up to the end time.
+	bufferedEndTime := endTime.Add(blockTimeRangeBuffer)
+
+	endHeight, err := findFirstBlockBeforeTimestamp(
+		ctx, lndClient, latestHeight, bufferedEndTime,
+	)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	if startHeight > endHeight {
+		log.Errorf("Start height: %v is greater than end height: %v, "+
+			"setting both to 0", startHeight, endHeight)
+
+		// If startHeight somehow ended up being greater than endHeight,
+		// set both start and end height to 0, meaning we will query for
+		// all onchain history.
+		startHeight = 0
+		endHeight = 0
+	}
+
+	return startHeight, endHeight, nil
+}
+
+// findFirstBlockBeforeTimestamp finds the block height from just before the
+// given timestamp.
+func findFirstBlockBeforeTimestamp(ctx context.Context,
+	lndClient lndclient.LndServices, latestHeight uint32,
+	targetTime time.Time) (uint32, error) {
+
+	targetTimestamp := targetTime.Unix()
+
+	// Set the search range to the genesis block and the latest block.
+	low := uint32(0)
+	high := latestHeight
+
+	// Perform binary search to find the block height that is just before the
+	// target timestamp.
+	for low <= high {
+		mid := (low + high) / 2
+
+		// Lookup the block in the middle of the search range.
+		blockHash, err := getBlockHash(ctx, lndClient, mid)
+		if err != nil {
+			return 0, err
+		}
+
+		blockTime, err := getBlockTimestamp(ctx, lndClient, blockHash)
+		if err != nil {
+			return 0, err
+		}
+
+		blockTimestamp := blockTime.Unix()
+		if blockTimestamp < targetTimestamp {
+			// If the block we looked up is before the target timestamp,
+			// we set the new low height to the next block after that.
+			low = mid + 1
+		} else if blockTimestamp > targetTimestamp {
+			// If the block we looked up is after the target timestamp,
+			// we set the new high height to the block before that.
+			high = mid - 1
+		} else {
+			// If we find an exact match of block timestamp and target
+			// timestamp, ruturn the height of this block.
+			return mid, nil
+		}
+	}
+
+	log.Debugf("Binary search done for targetTimestamp: %v. "+
+		"Returning height: %v", targetTimestamp, high)
+
+	// Closest block before the timestamp.
+	return high, nil
+}
+
+// getBlockHash retrieves the block hash for a given height.
+func getBlockHash(ctx context.Context, lndClient lndclient.LndServices,
+	height uint32) (chainhash.Hash, error) {
+
+	blockHash, err := lndClient.ChainKit.GetBlockHash(ctx, int64(height))
+	if err != nil {
+		return chainhash.Hash{}, err
+	}
+
+	return blockHash, nil
+}
+
+// getBlockTimestamp retrieves the block timestamp for a given block hash.
+func getBlockTimestamp(ctx context.Context,
+	lndClient lndclient.LndServices, hash chainhash.Hash) (time.Time, error) {
+
+	blockHeader, err := lndClient.ChainKit.GetBlockHeader(ctx, hash)
+	if err != nil {
+		return time.Time{}, err
+	}
+
+	return blockHeader.Timestamp, nil
+}