routing+channeldb: use a more minimal encoding for MC routes

Before this commit, Mission Control would serialise the entire
route.Route for each mission control entry. However, MC only uses a
fraction of the info in route.Route. Continuing to persist the entire
encoding may become more constly once we start sending to more blinded
routes because then this encoding will also include the encrypted cipher
text sent to each hop on the blinded path.
So this commit migrates the MC store to only persist the data that the
MC actually uses.

channeldb/db.go

@@ -26,6 +26,7 @@ import (
 	"github.com/lightningnetwork/lnd/channeldb/migration29"
 	"github.com/lightningnetwork/lnd/channeldb/migration30"
 	"github.com/lightningnetwork/lnd/channeldb/migration31"
+	"github.com/lightningnetwork/lnd/channeldb/migration32"
 	"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
 	"github.com/lightningnetwork/lnd/clock"
 	"github.com/lightningnetwork/lnd/invoices"
@@ -286,6 +287,10 @@ var (
 			number:    31,
 			migration: migration31.DeleteLastPublishedTxTLB,
 		},
+		{
+			number:    32,
+			migration: migration32.MigrateMCRouteSerialisation,
+		},
 	}
 
 	// optionalVersions stores all optional migrations that are applied

channeldb/log.go

@@ -10,6 +10,7 @@ import (
 	"github.com/lightningnetwork/lnd/channeldb/migration24"
 	"github.com/lightningnetwork/lnd/channeldb/migration30"
 	"github.com/lightningnetwork/lnd/channeldb/migration31"
+	"github.com/lightningnetwork/lnd/channeldb/migration32"
 	"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
 	"github.com/lightningnetwork/lnd/kvdb"
 )
@@ -42,5 +43,6 @@ func UseLogger(logger btclog.Logger) {
 	migration24.UseLogger(logger)
 	migration30.UseLogger(logger)
 	migration31.UseLogger(logger)
+	migration32.UseLogger(logger)
 	kvdb.UseLogger(logger)
 }

channeldb/migration32/codec.go

@@ -0,0 +1,140 @@
+package migration32
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/btcsuite/btcd/wire"
+	lnwire "github.com/lightningnetwork/lnd/channeldb/migration/lnwire21"
+)
+
+// ReadElement is a one-stop utility function to deserialize any datastructure
+// encoded using the serialization format of the database.
+func ReadElement(r io.Reader, element interface{}) error {
+	switch e := element.(type) {
+	case *uint32:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *lnwire.MilliSatoshi:
+		var a uint64
+		if err := binary.Read(r, byteOrder, &a); err != nil {
+			return err
+		}
+
+		*e = lnwire.MilliSatoshi(a)
+
+	case *[]byte:
+		bytes, err := wire.ReadVarBytes(r, 0, 66000, "[]byte")
+		if err != nil {
+			return err
+		}
+
+		*e = bytes
+
+	case *int64, *uint64:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *bool:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	case *int32:
+		if err := binary.Read(r, byteOrder, e); err != nil {
+			return err
+		}
+
+	default:
+		return UnknownElementType{"ReadElement", e}
+	}
+
+	return nil
+}
+
+// ReadElements deserializes a variable number of elements into the passed
+// io.Reader, with each element being deserialized according to the ReadElement
+// function.
+func ReadElements(r io.Reader, elements ...interface{}) error {
+	for _, element := range elements {
+		err := ReadElement(r, element)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// UnknownElementType is an error returned when the codec is unable to encode or
+// decode a particular type.
+type UnknownElementType struct {
+	method  string
+	element interface{}
+}
+
+// Error returns the name of the method that encountered the error, as well as
+// the type that was unsupported.
+func (e UnknownElementType) Error() string {
+	return fmt.Sprintf("Unknown type in %s: %T", e.method, e.element)
+}
+
+// WriteElement is a one-stop shop to write the big endian representation of
+// any element which is to be serialized for storage on disk. The passed
+// io.Writer should be backed by an appropriately sized byte slice, or be able
+// to dynamically expand to accommodate additional data.
+func WriteElement(w io.Writer, element interface{}) error {
+	switch e := element.(type) {
+	case int64, uint64:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case uint32:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case int32:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case bool:
+		if err := binary.Write(w, byteOrder, e); err != nil {
+			return err
+		}
+
+	case lnwire.MilliSatoshi:
+		if err := binary.Write(w, byteOrder, uint64(e)); err != nil {
+			return err
+		}
+
+	case []byte:
+		if err := wire.WriteVarBytes(w, 0, e); err != nil {
+			return err
+		}
+
+	default:
+		return UnknownElementType{"WriteElement", e}
+	}
+
+	return nil
+}
+
+// WriteElements is writes each element in the elements slice to the passed
+// io.Writer using WriteElement.
+func WriteElements(w io.Writer, elements ...interface{}) error {
+	for _, element := range elements {
+		err := WriteElement(w, element)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

channeldb/migration32/hop.go

@@ -0,0 +1,99 @@
+package migration32
+
+import (
+	"github.com/btcsuite/btcd/btcec/v2"
+	"github.com/lightningnetwork/lnd/tlv"
+)
+
+const (
+	// AmtOnionType is the type used in the onion to reference the amount to
+	// send to the next hop.
+	AmtOnionType tlv.Type = 2
+
+	// LockTimeOnionType is the type used in the onion to reference the CLTV
+	// value that should be used for the next hop's HTLC.
+	LockTimeOnionType tlv.Type = 4
+
+	// NextHopOnionType is the type used in the onion to reference the ID
+	// of the next hop.
+	NextHopOnionType tlv.Type = 6
+
+	// EncryptedDataOnionType is the type used to include encrypted data
+	// provided by the receiver in the onion for use in blinded paths.
+	EncryptedDataOnionType tlv.Type = 10
+
+	// BlindingPointOnionType is the type used to include receiver provided
+	// ephemeral keys in the onion that are used in blinded paths.
+	BlindingPointOnionType tlv.Type = 12
+
+	// MetadataOnionType is the type used in the onion for the payment
+	// metadata.
+	MetadataOnionType tlv.Type = 16
+
+	// TotalAmtMsatBlindedType is the type used in the onion for the total
+	// amount field that is included in the final hop for blinded payments.
+	TotalAmtMsatBlindedType tlv.Type = 18
+)
+
+// NewAmtToFwdRecord creates a tlv.Record that encodes the amount_to_forward
+// (type 2) for an onion payload.
+func NewAmtToFwdRecord(amt *uint64) tlv.Record {
+	return tlv.MakeDynamicRecord(
+		AmtOnionType, amt, func() uint64 {
+			return tlv.SizeTUint64(*amt)
+		},
+		tlv.ETUint64, tlv.DTUint64,
+	)
+}
+
+// NewLockTimeRecord creates a tlv.Record that encodes the outgoing_cltv_value
+// (type 4) for an onion payload.
+func NewLockTimeRecord(lockTime *uint32) tlv.Record {
+	return tlv.MakeDynamicRecord(
+		LockTimeOnionType, lockTime, func() uint64 {
+			return tlv.SizeTUint32(*lockTime)
+		},
+		tlv.ETUint32, tlv.DTUint32,
+	)
+}
+
+// NewNextHopIDRecord creates a tlv.Record that encodes the short_channel_id
+// (type 6) for an onion payload.
+func NewNextHopIDRecord(cid *uint64) tlv.Record {
+	return tlv.MakePrimitiveRecord(NextHopOnionType, cid)
+}
+
+// NewEncryptedDataRecord creates a tlv.Record that encodes the encrypted_data
+// (type 10) record for an onion payload.
+func NewEncryptedDataRecord(data *[]byte) tlv.Record {
+	return tlv.MakePrimitiveRecord(EncryptedDataOnionType, data)
+}
+
+// NewBlindingPointRecord creates a tlv.Record that encodes the blinding_point
+// (type 12) record for an onion payload.
+func NewBlindingPointRecord(point **btcec.PublicKey) tlv.Record {
+	return tlv.MakePrimitiveRecord(BlindingPointOnionType, point)
+}
+
+// NewMetadataRecord creates a tlv.Record that encodes the metadata (type 10)
+// for an onion payload.
+func NewMetadataRecord(metadata *[]byte) tlv.Record {
+	return tlv.MakeDynamicRecord(
+		MetadataOnionType, metadata,
+		func() uint64 {
+			return uint64(len(*metadata))
+		},
+		tlv.EVarBytes, tlv.DVarBytes,
+	)
+}
+
+// NewTotalAmtMsatBlinded creates a tlv.Record that encodes the
+// total_amount_msat for the final an onion payload within a blinded route.
+func NewTotalAmtMsatBlinded(amt *uint64) tlv.Record {
+	return tlv.MakeDynamicRecord(
+		TotalAmtMsatBlindedType, amt, func() uint64 {
+			return tlv.SizeTUint64(*amt)
+		},
+		tlv.ETUint64, tlv.DTUint64,
+	)
+}

channeldb/migration32/log.go

@@ -0,0 +1,14 @@
+package migration32
+
+import (
+	"github.com/btcsuite/btclog"
+)
+
+// log is a logger that is initialized as disabled.  This means the package will
+// not perform any logging by default until a logger is set.
+var log = btclog.Disabled
+
+// UseLogger uses a specified Logger to output package logging info.
+func UseLogger(logger btclog.Logger) {
+	log = logger
+}

channeldb/migration32/migration.go

@@ -0,0 +1,53 @@
+package migration32
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/lightningnetwork/lnd/kvdb"
+)
+
+// MigrateMCRouteSerialisation reads all the mission control store entries and
+// re-serializes them using a minimal route serialisation so that only the parts
+// of the route that are actually required for mission control are persisted
+func MigrateMCRouteSerialisation(tx kvdb.RwTx) error {
+	log.Infof("Migrating Mission Control store to use a more minimal " +
+		"encoding for routes")
+
+	resultBucket := tx.ReadWriteBucket(resultsKey)
+
+	// If the results bucket does not exist then there are no entries in
+	// the mission control store yet and so there is nothing to migrate.
+	if resultBucket == nil {
+		return nil
+	}
+
+	// For each entry, read it into memory using the old encoding. Then,
+	// extract the more minimal route, re-encode and persist the entry.
+	return resultBucket.ForEach(func(k, v []byte) error {
+		// Read the entry using the old encoding.
+		resultOld, err := deserializeOldResult(k, v)
+		if err != nil {
+			return err
+		}
+
+		// Convert to the new payment result format with the minimal
+		// route.
+		resultNew := convertPaymentResult(resultOld)
+
+		// Serialise the new payment result using the new encoding.
+		key, resultNewBytes, err := serializeNewResult(resultNew)
+		if err != nil {
+			return err
+		}
+
+		// Make sure that the derived key is the same.
+		if !bytes.Equal(key, k) {
+			return fmt.Errorf("new payment result key (%v) is "+
+				"not the same as the old key (%v)", key, k)
+		}
+
+		// Finally, overwrite the previous value with the new encoding.
+		return resultBucket.Put(k, resultNewBytes)
+	})
+}