Ignore Ordinals in Golomb filters (trezor#967)

bchain/coins/btc/bitcoinlikeparser.go

@@ -231,6 +231,7 @@ func (p *BitcoinLikeParser) TxFromMsgTx(t *wire.MsgTx, parseAddresses bool) bcha
 			Vout:      in.PreviousOutPoint.Index,
 			Sequence:  in.Sequence,
 			ScriptSig: s,
+			Witness:   in.Witness,
 		}
 	}
 	vout := make([]bchain.Vout, len(t.TxOut))

bchain/coins/btc/bitcoinrpc.go

@@ -34,6 +34,7 @@ type BitcoinRPC struct {
 	RPCMarshaler         RPCMarshaler
 	mempoolGolombFilterP uint8
 	mempoolFilterScripts string
+	mempoolUseZeroedKey  bool
 }
 
 // Configuration represents json config file
@@ -63,6 +64,7 @@ type Configuration struct {
 	MinimumCoinbaseConfirmations int    `json:"minimumCoinbaseConfirmations,omitempty"`
 	MempoolGolombFilterP         uint8  `json:"mempool_golomb_filter_p,omitempty"`
 	MempoolFilterScripts         string `json:"mempool_filter_scripts,omitempty"`
+	MempoolFilterUseZeroedKey    bool   `json:"mempool_filter_use_zeroed_key,omitempty"`
 }
 
 // NewBitcoinRPC returns new BitcoinRPC instance.
@@ -110,6 +112,7 @@ func NewBitcoinRPC(config json.RawMessage, pushHandler func(bchain.NotificationT
 		RPCMarshaler:         JSONMarshalerV2{},
 		mempoolGolombFilterP: c.MempoolGolombFilterP,
 		mempoolFilterScripts: c.MempoolFilterScripts,
+		mempoolUseZeroedKey:  c.MempoolFilterUseZeroedKey,
 	}
 
 	return s, nil
@@ -155,7 +158,7 @@ func (b *BitcoinRPC) Initialize() error {
 // CreateMempool creates mempool if not already created, however does not initialize it
 func (b *BitcoinRPC) CreateMempool(chain bchain.BlockChain) (bchain.Mempool, error) {
 	if b.Mempool == nil {
-		b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers, b.mempoolGolombFilterP, b.mempoolFilterScripts)
+		b.Mempool = bchain.NewMempoolBitcoinType(chain, b.ChainConfig.MempoolWorkers, b.ChainConfig.MempoolSubWorkers, b.mempoolGolombFilterP, b.mempoolFilterScripts, b.mempoolUseZeroedKey)
 	}
 	return b.Mempool, nil
 }

bchain/golomb.go

@@ -1,6 +1,7 @@
 package bchain
 
 import (
+	"bytes"
 	"encoding/hex"
 
 	"github.com/golang/glog"
@@ -14,69 +15,165 @@ const (
 	FilterScriptsInvalid = FilterScriptsType(iota)
 	FilterScriptsAll
 	FilterScriptsTaproot
+	FilterScriptsTaprootNoOrdinals
 )
 
 // GolombFilter is computing golomb filter of address descriptors
 type GolombFilter struct {
 	Enabled           bool
+	UseZeroedKey      bool
 	p                 uint8
 	key               string
 	filterScripts     string
 	filterScriptsType FilterScriptsType
 	filterData        [][]byte
 	uniqueData        map[string]struct{}
+	// All the unique txids that contain ordinal data
+	ordinalTxIds map[string]struct{}
+	// Mapping of txid to address descriptors - only used in case of taproot-noordinals
+	allAddressDescriptors map[string][]AddressDescriptor
 }
 
 // NewGolombFilter initializes the GolombFilter handler
-func NewGolombFilter(p uint8, filterScripts string, key string) (*GolombFilter, error) {
+func NewGolombFilter(p uint8, filterScripts string, key string, useZeroedKey bool) (*GolombFilter, error) {
 	if p == 0 {
 		return &GolombFilter{Enabled: false}, nil
 	}
 	gf := GolombFilter{
 		Enabled:           true,
+		UseZeroedKey:      useZeroedKey,
 		p:                 p,
 		key:               key,
 		filterScripts:     filterScripts,
 		filterScriptsType: filterScriptsToScriptsType(filterScripts),
 		filterData:        make([][]byte, 0),
 		uniqueData:        make(map[string]struct{}),
 	}
-	// only taproot and all is supported
+	// reject invalid filterScripts
 	if gf.filterScriptsType == FilterScriptsInvalid {
 		return nil, errors.Errorf("Invalid/unsupported filterScripts parameter %s", filterScripts)
 	}
+	// set ordinal-related fields if needed
+	if gf.ignoreOrdinals() {
+		gf.ordinalTxIds = make(map[string]struct{})
+		gf.allAddressDescriptors = make(map[string][]AddressDescriptor)
+	}
 	return &gf, nil
 }
 
+// Gets the M parameter that we are using for the filter
+// Currently it relies on P parameter, but that can change
+func GetGolombParamM(p uint8) uint64 {
+	return uint64(1 << uint64(p))
+}
+
+// Checks whether this input contains ordinal data
+func isInputOrdinal(vin Vin) bool {
+	byte_pattern := []byte{
+		0x00, // OP_0, OP_FALSE
+		0x63, // OP_IF
+		0x03, // OP_PUSHBYTES_3
+		0x6f, // "o"
+		0x72, // "r"
+		0x64, // "d"
+		0x01, // OP_PUSHBYTES_1
+	}
+	// Witness needs to have at least 3 items and the second one needs to contain certain pattern
+	return len(vin.Witness) > 2 && bytes.Contains(vin.Witness[1], byte_pattern)
+}
+
+// Whether a transaction contains any ordinal data
+func txContainsOrdinal(tx *Tx) bool {
+	for _, vin := range tx.Vin {
+		if isInputOrdinal(vin) {
+			return true
+		}
+	}
+	return false
+}
+
+// Saving all the ordinal-related txIds so we can later ignore their address descriptors
+func (f *GolombFilter) markTxAndParentsAsOrdinals(tx *Tx) {
+	f.ordinalTxIds[tx.Txid] = struct{}{}
+	for _, vin := range tx.Vin {
+		f.ordinalTxIds[vin.Txid] = struct{}{}
+	}
+}
+
+// Adding a new address descriptor mapped to a txid
+func (f *GolombFilter) addTxIdMapping(ad AddressDescriptor, tx *Tx) {
+	f.allAddressDescriptors[tx.Txid] = append(f.allAddressDescriptors[tx.Txid], ad)
+}
+
 // AddAddrDesc adds taproot address descriptor to the data for the filter
-func (f *GolombFilter) AddAddrDesc(ad AddressDescriptor) {
-	if f.filterScriptsType == FilterScriptsTaproot && !ad.IsTaproot() {
+func (f *GolombFilter) AddAddrDesc(ad AddressDescriptor, tx *Tx) {
+	if f.ignoreNonTaproot() && !ad.IsTaproot() {
+		return
+	}
+	if f.ignoreOrdinals() && tx != nil && txContainsOrdinal(tx) {
+		f.markTxAndParentsAsOrdinals(tx)
 		return
 	}
 	if len(ad) == 0 {
 		return
 	}
+	// When ignoring ordinals, we need to save all the address descriptors before
+	// filtering out the "invalid" ones.
+	if f.ignoreOrdinals() && tx != nil {
+		f.addTxIdMapping(ad, tx)
+		return
+	}
+	f.includeAddrDesc(ad)
+}
+
+// Private function to be called with descriptors that were already validated
+func (f *GolombFilter) includeAddrDesc(ad AddressDescriptor) {
 	s := string(ad)
 	if _, found := f.uniqueData[s]; !found {
 		f.filterData = append(f.filterData, ad)
 		f.uniqueData[s] = struct{}{}
 	}
 }
 
+// Including all the address descriptors from non-ordinal transactions
+func (f *GolombFilter) includeAllAddressDescriptorsOrdinals() {
+	for txid, ads := range f.allAddressDescriptors {
+		// Ignoring the txids that contain ordinal data
+		if _, found := f.ordinalTxIds[txid]; found {
+			continue
+		}
+		for _, ad := range ads {
+			f.includeAddrDesc(ad)
+		}
+	}
+}
+
 // Compute computes golomb filter from the data
 func (f *GolombFilter) Compute() []byte {
-	m := uint64(1 << uint64(f.p))
+	m := GetGolombParamM(f.p)
+
+	// In case of ignoring the ordinals, we still need to assemble the filter data
+	if f.ignoreOrdinals() {
+		f.includeAllAddressDescriptorsOrdinals()
+	}
 
 	if len(f.filterData) == 0 {
 		return nil
 	}
 
-	b, _ := hex.DecodeString(f.key)
-	if len(b) < gcs.KeySize {
-		return nil
+	// Used key is possibly just zeroes, otherwise get it from the supplied key
+	var key [gcs.KeySize]byte
+	if f.UseZeroedKey {
+		key = [gcs.KeySize]byte{}
+	} else {
+		b, _ := hex.DecodeString(f.key)
+		if len(b) < gcs.KeySize {
+			return nil
+		}
+		copy(key[:], b[:gcs.KeySize])
 	}
 
-	filter, err := gcs.BuildGCSFilter(f.p, m, *(*[gcs.KeySize]byte)(b[:gcs.KeySize]), f.filterData)
+	filter, err := gcs.BuildGCSFilter(f.p, m, key, f.filterData)
 	if err != nil {
 		glog.Error("Cannot create golomb filter for ", f.key, ", ", err)
 		return nil
@@ -91,12 +188,30 @@ func (f *GolombFilter) Compute() []byte {
 	return fb
 }
 
+func (f *GolombFilter) ignoreNonTaproot() bool {
+	switch f.filterScriptsType {
+	case FilterScriptsTaproot, FilterScriptsTaprootNoOrdinals:
+		return true
+	}
+	return false
+}
+
+func (f *GolombFilter) ignoreOrdinals() bool {
+	switch f.filterScriptsType {
+	case FilterScriptsTaprootNoOrdinals:
+		return true
+	}
+	return false
+}
+
 func filterScriptsToScriptsType(filterScripts string) FilterScriptsType {
 	switch filterScripts {
 	case "":
 		return FilterScriptsAll
 	case "taproot":
 		return FilterScriptsTaproot
+	case "taproot-noordinals":
+		return FilterScriptsTaprootNoOrdinals
 	}
 	return FilterScriptsInvalid
 }

bchain/mempool_bitcoin_type.go

@@ -22,11 +22,12 @@ type MempoolBitcoinType struct {
 	AddrDescForOutpoint AddrDescForOutpointFunc
 	golombFilterP       uint8
 	filterScripts       string
+	useZeroedKey        bool
 }
 
 // NewMempoolBitcoinType creates new mempool handler.
 // For now there is no cleanup of sync routines, the expectation is that the mempool is created only once per process
-func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8, filterScripts string) *MempoolBitcoinType {
+func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golombFilterP uint8, filterScripts string, useZeroedKey bool) *MempoolBitcoinType {
 	m := &MempoolBitcoinType{
 		BaseMempool: BaseMempool{
 			chain:        chain,
@@ -37,6 +38,7 @@ func NewMempoolBitcoinType(chain BlockChain, workers int, subworkers int, golomb
 		chanAddrIndex: make(chan txidio, 1),
 		golombFilterP: golombFilterP,
 		filterScripts: filterScripts,
+		useZeroedKey:  useZeroedKey,
 	}
 	for i := 0; i < workers; i++ {
 		go func(i int) {
@@ -97,18 +99,18 @@ func (m *MempoolBitcoinType) getInputAddress(payload *chanInputPayload) *addrInd
 
 }
 
-func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx) string {
-	gf, _ := NewGolombFilter(m.golombFilterP, m.filterScripts, mtx.Txid)
+func (m *MempoolBitcoinType) computeGolombFilter(mtx *MempoolTx, tx *Tx) string {
+	gf, _ := NewGolombFilter(m.golombFilterP, "", mtx.Txid, m.useZeroedKey)
 	if gf == nil || !gf.Enabled {
 		return ""
 	}
 	for _, vin := range mtx.Vin {
-		gf.AddAddrDesc(vin.AddrDesc)
+		gf.AddAddrDesc(vin.AddrDesc, tx)
 	}
 	for _, vout := range mtx.Vout {
 		b, err := hex.DecodeString(vout.ScriptPubKey.Hex)
 		if err == nil {
-			gf.AddAddrDesc(b)
+			gf.AddAddrDesc(b, tx)
 		}
 	}
 	fb := gf.Compute()
@@ -168,7 +170,7 @@ func (m *MempoolBitcoinType) getTxAddrs(txid string, chanInput chan chanInputPay
 	}
 	var golombFilter string
 	if m.golombFilterP > 0 {
-		golombFilter = m.computeGolombFilter(mtx)
+		golombFilter = m.computeGolombFilter(mtx, tx)
 	}
 	if m.OnNewTx != nil {
 		m.OnNewTx(mtx)
@@ -249,5 +251,5 @@ func (m *MempoolBitcoinType) GetTxidFilterEntries(filterScripts string, fromTime
 		}
 	}
 	m.mux.Unlock()
-	return MempoolTxidFilterEntries{entries}, nil
+	return MempoolTxidFilterEntries{entries, m.useZeroedKey}, nil
 }

bchain/mempool_bitcoin_type_test.go

@@ -18,7 +18,7 @@ func TestMempoolBitcoinType_computeGolombFilter_taproot(t *testing.T) {
 		golombFilterP: 20,
 		filterScripts: "taproot",
 	}
-	golombFilterM := uint64(1 << uint64(m.golombFilterP))
+	golombFilterM := GetGolombParamM(m.golombFilterP)
 	tests := []struct {
 		name string
 		mtx  MempoolTx

bchain/types.go

@@ -57,6 +57,7 @@ type Vin struct {
 	ScriptSig ScriptSig `json:"scriptSig"`
 	Sequence  uint32    `json:"sequence"`
 	Addresses []string  `json:"addresses"`
+	Witness   [][]byte  `json:"witness"`
 }
 
 // ScriptPubKey contains data about output script
@@ -273,8 +274,10 @@ type XpubDescriptor struct {
 type MempoolTxidEntries []MempoolTxidEntry
 
 // MempoolTxidFilterEntries is a map of txids to mempool golomb filters
+// Also contains a flag whether constant zeroed key was used when calculating the filters
 type MempoolTxidFilterEntries struct {
-	Entries map[string]string `json:"entries,omitempty"`
+	Entries       map[string]string `json:"entries,omitempty"`
+	UsedZeroedKey bool              `json:"usedZeroedKey,omitempty"`
 }
 
 // OnNewBlockFunc is used to send notification about a new block

common/config.go

@@ -9,15 +9,16 @@ import (
 
 // Config struct
 type Config struct {
-	CoinName              string `json:"coin_name"`
-	CoinShortcut          string `json:"coin_shortcut"`
-	CoinLabel             string `json:"coin_label"`
-	FourByteSignatures    string `json:"fourByteSignatures"`
-	FiatRates             string `json:"fiat_rates"`
-	FiatRatesParams       string `json:"fiat_rates_params"`
-	FiatRatesVsCurrencies string `json:"fiat_rates_vs_currencies"`
-	BlockGolombFilterP    uint8  `json:"block_golomb_filter_p"`
-	BlockFilterScripts    string `json:"block_filter_scripts"`
+	CoinName                string `json:"coin_name"`
+	CoinShortcut            string `json:"coin_shortcut"`
+	CoinLabel               string `json:"coin_label"`
+	FourByteSignatures      string `json:"fourByteSignatures"`
+	FiatRates               string `json:"fiat_rates"`
+	FiatRatesParams         string `json:"fiat_rates_params"`
+	FiatRatesVsCurrencies   string `json:"fiat_rates_vs_currencies"`
+	BlockGolombFilterP      uint8  `json:"block_golomb_filter_p"`
+	BlockFilterScripts      string `json:"block_filter_scripts"`
+	BlockFilterUseZeroedKey bool   `json:"block_filter_use_zeroed_key"`
 }
 
 // GetConfig loads and parses the config file and returns Config struct

common/internalstate.go

@@ -94,8 +94,9 @@ type InternalState struct {
 	SortedAddressContracts bool `json:"sortedAddressContracts"`
 
 	// golomb filter settings
-	BlockGolombFilterP uint8  `json:"block_golomb_filter_p"`
-	BlockFilterScripts string `json:"block_filter_scripts"`
+	BlockGolombFilterP      uint8  `json:"block_golomb_filter_p"`
+	BlockFilterScripts      string `json:"block_filter_scripts"`
+	BlockFilterUseZeroedKey bool   `json:"block_filter_use_zeroed_key"`
 }
 
 // StartedSync signals start of synchronization

db/bulkconnect.go

@@ -185,7 +185,7 @@ func (b *BulkConnect) storeBulkBlockFilters(wb *grocksdb.WriteBatch) error {
 
 func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs bool) error {
 	addresses := make(addressesMap)
-	gf, err := bchain.NewGolombFilter(b.d.is.BlockGolombFilterP, b.d.is.BlockFilterScripts, block.BlockHeader.Hash)
+	gf, err := bchain.NewGolombFilter(b.d.is.BlockGolombFilterP, b.d.is.BlockFilterScripts, block.BlockHeader.Hash, b.d.is.BlockFilterUseZeroedKey)
 	if err != nil {
 		glog.Error("connectBlockBitcoinType golomb filter error ", err)
 		gf = nil