diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index e513bb69..958a544e 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2380,7 +2380,7 @@ def getUreLast(self, key): def getUreItemIter(self, key=b''): """ - Use sgKey() + Use snKey() Return iterator of partial signed escrowed event triple items at next key after key. Items is (key, val) where proem has already been stripped from val diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index f190ba65..1a8430a5 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -64,7 +64,8 @@ OnIoDupSuber provides set of insertion ordered values where the where trailing part of key is serialized ordinal number so that the ordering within each - key prefix is monotonically increasing numeric. + key prefix is monotonically increasing numeric. Useful to provide omndices + for sn ordering of superseding KEL events. Each of these base types for managing the key space may be mixed with other Classes that provide different types of values these include. @@ -638,6 +639,10 @@ class B64SuberBase(SuberBase): db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes + for db key and also used to convert val iterator to val bytes + Must not be Base64 character. + default is self.Sep == '.' + """ def __init__(self, *pa, **kwa): @@ -759,8 +764,6 @@ class B64Suber(B64SuberBase, Suber): .sep must not be Base64 character. - Each key consistes of pre joined with .sep to ordinal suffix - Assumes dupsort==False """ @@ -775,6 +778,7 @@ def __init__(self, *pa, **kwa): each key. Set to False sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + also used to convert val iterator to val bytes Must not be Base64 character. verify (bool): True means reverify when ._des from db when applicable False means do not reverify. Default False @@ -2158,13 +2162,44 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", yield (self._tokeys(key), self._des(val)) +class B64IoDupSuber(B64SuberBase, IoDupSuber): + """ + Subclass of B64SuberBase and IoDupSuber that serializes and deserializes + values as .sep joined strings of Base64 components in insertion ordered + duplicates with leading value proem. Proem + .sep joined value and must fit + in 511 bytes of keyspace as duplicate + + Assumes dupsort==True + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator for combining keys tuple of strs into key bytes + for db key and also used to convert val iterator to val bytes + Must not be Base64 character. + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(B64IoDupSuber, self).__init__(*pa, **kwa) + + + class OnIoDupSuber(OnSuberBase, IoDupSuber): """ Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering (IoDup) of duplicates but where the trailing part of the key space is a serialized monotonically increasing ordinal number. This is useful for - escrows of key events etc where duplicates of likely events are maintained - in insertion order. + escrows of key events which are ordinally numbered such as sn but where + duplicates of likely events are also maintained in insertion order. + Insertion order is maintained by automagically prepending and stripping an ordinal ordering proem to/from each duplicate value at a given key. @@ -2412,3 +2447,39 @@ def getOnItemBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0) for keys, on, val in (self.db.getOnIoDupItemBackIter(db=self.sdb, key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) + + +class B64OnIoDupSuber(B64SuberBase, OnIoDupSuber): + """ + Subclass of B64SuberBase and OnIoDupSuber that serializes and deserializes + values as .sep joined strings of Base64 components in insertion ordered + duplicates with leading value proem but where the trailing part of the + key space is a serialized monotonically increasing ordinal number. + Proem + .sep joined value and must fit n 511 bytes of keyspace as duplicate. + + Insertion order is maintained by automagically prepending and stripping an + ordinal ordering proem to/from each duplicate value at a given key. + + OnIoDupSuber adds the convenience methods from OnSuberBase to OnIoDupSuber + for those cases where the keyspace has a trailing ordinal part. + + Assumes dupsort==True + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator for combining keys tuple of strs into key bytes + for db key and also used to convert val iterator to val bytes + Must not be Base64 character. + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(B64OnIoDupSuber, self).__init__(*pa, **kwa) diff --git a/src/keri/help/__init__.py b/src/keri/help/__init__.py index 2223582c..6671405f 100644 --- a/src/keri/help/__init__.py +++ b/src/keri/help/__init__.py @@ -17,4 +17,4 @@ ogler = ogling.initOgler(prefix='keri', syslogged=False) # inits once only on first import from .helping import (nowIso8601, toIso8601, fromIso8601, - nonStringSequence, nonStringIterable) + nonStringSequence, nonStringIterable, Reb64) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 58c72ccd..7443652a 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -752,6 +752,268 @@ def test_iodup_suber(): assert not db.opened +def test_B64_iodup_suber(): + """ + Test B64IoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + # Test Single klas + iobuber = subing.B64IoDupSuber(db=db, subkey='bags.') + assert isinstance(iobuber, subing.B64IoDupSuber) + assert iobuber.sdb.flags()["dupsort"] + assert iobuber.sep == '.' + assert not help.Reb64.match(iobuber.sep.encode()) + + # val as joined iterator + vals0 = ("alpha", "beta") + vals1 = ("gamma", ) + + keys0 = ("cat", "dog") # keys as tuple + keys1 = "{}.{}".format("bird", "fish") # keys as string not tuple + + + assert iobuber.put(keys=keys0, vals=(vals0,)) + actuals = iobuber.get(keys=keys0) + assert actuals == [vals0] + + assert iobuber.rem(keys0) + assert not iobuber.get(keys=keys0) + + assert iobuber.put(keys=keys0, vals=(vals0,)) + actuals = iobuber.get(keys=keys0) + assert actuals == [vals0] + + assert iobuber.put(keys=keys0, vals=(vals1,)) + actuals = iobuber.get(keys=keys0) + assert actuals == [vals0, vals1] + + assert iobuber.pin(keys=keys0, vals=(vals1,)) + actuals = iobuber.get(keys=keys0) + assert actuals == [vals1] + + assert iobuber.rem(keys0) + assert not iobuber.get(keys=keys0) + + # test with vals as non Iterable on put but Iterable on get + assert iobuber.put(keys=keys0, vals=["gamma"]) + actuals = iobuber.get(keys=keys0) + assert actuals == [vals1] + + assert iobuber.rem(keys0) + + # test val as already joined string since has non Base64 + with pytest.raises(ValueError): + assert iobuber.put(keys=keys0, vals=["alpha.beta"]) + + # test bytes for val + assert iobuber.put(keys=keys0, vals=[(b"alpha", b"beta")]) + actuals = iobuber.get(keys=keys0) + assert actuals == [vals0] + assert iobuber.rem(keys0) + + # test with keys1 + assert iobuber.put(keys=keys1, vals=[vals1]) + actuals = iobuber.get(keys=keys1) + assert actuals == [vals1] + + assert iobuber.rem(keys1) + assert not iobuber.get(keys=keys1) + + # test missing entry at keys + badkey = "badkey" + assert not iobuber.get(badkey) + + # test iteritems + assert iobuber.put(keys0, [vals0]) + assert iobuber.put(keys1, [vals1]) + + items = [ items for items in iobuber.getItemIter()] + assert items == [(('bird', 'fish'), ('gamma',)), (('cat', 'dog'), ('alpha', 'beta'))] + + keys3 = ("b","1") + keys4 = ("b","2") + keys5 = ("c","1") + keys6 = ("c","2") + + iobuber.put(keys=keys3, vals=[vals0]) + iobuber.put(keys=keys4, vals=[vals1]) + iobuber.put(keys=keys5, vals=[vals0]) + iobuber.put(keys=keys6, vals=[vals1]) + + topkeys = ("b","") # last element empty to force trailing separator + items = [items for items in iobuber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), ('alpha', 'beta')), + (('b', '2'), ('gamma',))] + + assert iobuber.rem(keys0) + assert iobuber.rem(keys1) + assert iobuber.rem(keys3) + assert iobuber.rem(keys4) + assert iobuber.rem(keys5) + assert iobuber.rem(keys6) + + + # iodup testing + # singular values B64 + sue = "Hello" + sal = "Toodles" + sam = "Bye" + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + assert iobuber.put(keys=keys0, vals=[sal, sue]) + actuals = iobuber.get(keys=keys0) + assert actuals == [(sal,), (sue,)] # insertion order not lexicographic + assert iobuber.cnt(keys0) == 2 + actual = iobuber.getLast(keys=keys0) + assert actual == (sue,) + + assert iobuber.rem(keys0) + actuals = iobuber.get(keys=keys0) + assert not actuals + assert actuals == [] + assert iobuber.cnt(keys0) == 0 + + assert iobuber.put(keys=keys0, vals=[sue, sal]) + actuals = iobuber.get(keys=keys0) + assert actuals == [(sue,), (sal, )] # insertion order + actual = iobuber.getLast(keys=keys0) + assert actual == (sal,) + + + assert iobuber.add(keys=keys0, val=sam) + actuals = iobuber.get(keys=keys0) + assert actuals == [(sue,), (sal,), (sam,)] # insertion order + + + sue = ("Hello",) + sal = ("Toodles",) + sam = ("Bye",) + zoe = ('later', 'alligator') + zia = ('soon', 'baboon') + + assert iobuber.pin(keys=keys0, vals=[zoe, zia]) + actuals = iobuber.get(keys=keys0) + assert actuals == [zoe, zia] # insertion order + + assert iobuber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = iobuber.get(keys=keys1) + assert actuals == [sal, sue, sam] + + for i, val in enumerate(iobuber.getIter(keys=keys1)): + assert val == actuals[i] + + items = [(keys, val) for keys, val in iobuber.getItemIter()] + assert items == [(('test_key', '0001'), ('later', 'alligator')), + (('test_key', '0001'), ('soon', 'baboon')), + (('test_key', '0002'), ('Toodles',)), + (('test_key', '0002'), ('Hello',)), + (('test_key', '0002'), ('Bye',))] + + items = list(iobuber.getFullItemIter()) + assert items == [ + (('test_key', '0001'), ('00000000000000000000000000000000', 'later', 'alligator')), + (('test_key', '0001'), ('00000000000000000000000000000001', 'soon', 'baboon')), + (('test_key', '0002'), ('00000000000000000000000000000000', 'Toodles')), + (('test_key', '0002'), ('00000000000000000000000000000001', 'Hello')), + (('test_key', '0002'), ('00000000000000000000000000000002', 'Bye')) + ] + + items = [(keys, val) for keys, val in iobuber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), ('Toodles',)), + (('test_key', '0002'), ('Hello',)), + (('test_key', '0002'), ('Bye',))] + + items = [(keys, val) for keys, val in iobuber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), ('later', 'alligator')), + (('test_key', '0001'), ('soon', 'baboon'))] + + # Test with top keys + assert iobuber.put(keys=("test", "pop"), vals=[sal, sue, sam]) + topkeys = ("test", "") + items = [(keys, val) for keys, val in iobuber.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), ('Toodles',)), + (('test', 'pop'), ('Hello',)), + (('test', 'pop'), ('Bye',))] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in iobuber.getItemIter(keys=keys, topive=True)] + assert items == [(('test', 'pop'), ('Toodles',)), + (('test', 'pop'), ('Hello',)), + (('test', 'pop'), ('Bye',))] + + # IoItems + items = list(iobuber.getFullItemIter(keys=topkeys)) + assert items == [(('test', 'pop'), ('00000000000000000000000000000000', 'Toodles')), + (('test', 'pop'), ('00000000000000000000000000000001', 'Hello')), + (('test', 'pop'), ('00000000000000000000000000000002', 'Bye'))] + + # test remove with a specific val + assert iobuber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in iobuber.getItemIter()] + assert items ==[(('test', 'pop'), ('Toodles',)), + (('test', 'pop'), ('Hello',)), + (('test', 'pop'), ('Bye',)), + (('test_key', '0001'), ('later', 'alligator')), + (('test_key', '0001'), ('soon', 'baboon')), + (('test_key', '0002'), ('Toodles',)), + (('test_key', '0002'), ('Bye',))] + + + assert iobuber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in iobuber.getItemIter()] + assert items == [(('test_key', '0001'), ('later', 'alligator')), + (('test_key', '0001'), ('soon', 'baboon')), + (('test_key', '0002'), ('Toodles',)), + (('test_key', '0002'), ('Bye',))] + + + assert iobuber.cnt(keys=keys0) == 2 + assert iobuber.cnt(keys=keys1) == 2 + + # test with keys as string not tuple + keys2 = "keystr" + bob = ("go", "figure") + assert iobuber.put(keys=keys2, vals=[bob]) + actuals = iobuber.get(keys=keys2) + assert actuals == [bob] + assert iobuber.cnt(keys2) == 1 + assert iobuber.rem(keys2) + actuals = iobuber.get(keys=keys2) + assert actuals == [] + assert iobuber.cnt(keys2) == 0 + + assert iobuber.put(keys=keys2, vals=[bob]) + actuals = iobuber.get(keys=keys2) + assert actuals == [bob] + + bil = ("Go", "away") + assert iobuber.pin(keys=keys2, vals=[bil]) + actuals = iobuber.get(keys=keys2) + assert actuals == [bil] + + assert iobuber.add(keys=keys2, val=bob) + actuals = iobuber.get(keys=keys2) + assert actuals == [bil, bob] + + # Test trim + assert iobuber.trim() # default trims whole database + assert iobuber.put(keys=keys1, vals=[bob, bil]) + assert iobuber.get(keys=keys1) == [bob, bil] + + + assert not os.path.exists(db.path) + assert not db.opened + """Done Test""" + + def test_on_iodup_suber(): """ Test OnIoDupSuber LMDBer sub database class @@ -2752,6 +3014,7 @@ def test_crypt_signer_suber(): test_B64_suber() test_dup_suber() test_iodup_suber() + test_B64_iodup_suber() test_on_iodup_suber() test_ioset_suber() test_cat_cesr_suber()