From b2e977231d37f7df43e8aeb80a240db67e682321 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Tue, 26 Sep 2023 09:38:51 -0400 Subject: [PATCH] LMDBer methods throw KeyError on malformed keys. LMBD throws lmdb.BadValSizeError which isn't very helpful (or caught) in the codebase although KeyErrors are because of the dict abstraction elsewhere. (Found while trying to resolve Issue #575) So now the LMDB methods throw KeyError where appropriate and tests for all methods with empty keys now execute regardless of whether those methods had the issue or not. --- src/keri/db/dbing.py | 162 ++++++++++++++++++++++++++++++----------- tests/db/test_dbing.py | 58 +++++++++++++++ 2 files changed, 177 insertions(+), 43 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 500623e65..8c5edc049 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -417,7 +417,11 @@ def putVal(self, db, key, val): val is bytes of value to be written """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.put(key, val, overwrite=False)) + try: + return (txn.put(key, val, overwrite=False)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def setVal(self, db, key, val): @@ -432,7 +436,11 @@ def setVal(self, db, key, val): val is bytes of value to be written """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.put(key, val)) + try: + return (txn.put(key, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getVal(self, db, key): @@ -446,7 +454,11 @@ def getVal(self, db, key): """ with self.env.begin(db=db, write=False, buffers=True) as txn: - return( txn.get(key)) + try: + return(txn.get(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def delVal(self, db, key): @@ -459,7 +471,11 @@ def delVal(self, db, key): key is bytes of key within sub db's keyspace """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.delete(key)) + try: + return (txn.delete(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def cnt(self, db): @@ -1108,7 +1124,11 @@ def delIoSetIokey(self, db, iokey): iokey (bytes): actual key with ordinal key suffix """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return txn.delete(iokey) + try: + return txn.delete(iokey) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") # For subdbs that support duplicates at each key (dupsort==True) @@ -1130,8 +1150,12 @@ def putVals(self, db, key, vals): """ with self.env.begin(db=db, write=True, buffers=True) as txn: result = True - for val in vals: - result = result and txn.put(key, val, dupdata=True) + try: + for val in vals: + result = result and txn.put(key, val, dupdata=True) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return result @@ -1158,7 +1182,11 @@ def addVal(self, db, key, val): result = False if val not in dups: with self.env.begin(db=db, write=True, buffers=True) as txn: - result = txn.put(key, val, dupdata=True) + try: + result = txn.put(key, val, dupdata=True) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return result @@ -1177,8 +1205,12 @@ def getVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - vals = [val for val in cursor.iternext_dup()] + try: + if cursor.set_key(key): # moves to first_dup + vals = [val for val in cursor.iternext_dup()] + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return vals @@ -1196,9 +1228,13 @@ def getValLast(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() val = None - if cursor.set_key(key): # move to first_dup - if cursor.last_dup(): # move to last_dup - val = cursor.value() + try: + if cursor.set_key(key): # move to first_dup + if cursor.last_dup(): # move to last_dup + val = cursor.value() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return val @@ -1216,9 +1252,13 @@ def getValsIter(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val + try: + if cursor.set_key(key): # moves to first_dup + for val in cursor.iternext_dup(): + yield val + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def cntVals(self, db, key): @@ -1232,8 +1272,12 @@ def cntVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() count = 0 - if cursor.set_key(key): # moves to first_dup - count = cursor.count() + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return count @@ -1263,6 +1307,7 @@ def cntValsAllPre(self, db, pre, on=0): return count + def delVals(self, db, key, val=b''): """ Deletes all values at key in db if val=b'' else deletes the dup @@ -1275,7 +1320,11 @@ def delVals(self, db, key, val=b''): val is bytes of dup val at key to delete """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.delete(key, val)) + try: + return (txn.delete(key, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") # For subdbs that support insertion order preserving duplicates at each key. @@ -1309,9 +1358,13 @@ def putIoVals(self, db, key, vals): with self.env.begin(db=db, write=True, buffers=True) as txn: idx = 0 cursor = txn.cursor() - if cursor.set_key(key): # move to key if any - if cursor.last_dup(): # move to last dup - idx = 1 + int(bytes(cursor.value()[:32]), 16) # get last index as int + try: + if cursor.set_key(key): # move to key if any + if cursor.last_dup(): # move to last dup + idx = 1 + int(bytes(cursor.value()[:32]), 16) # get last index as int + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") for val in vals: if val not in dups: @@ -1353,10 +1406,14 @@ def getIoVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - # slice off prepended ordering proem - vals = [val[33:] for val in cursor.iternext_dup()] - return vals + try: + if cursor.set_key(key): # moves to first_dup + # slice off prepended ordering proem + vals = [val[33:] for val in cursor.iternext_dup()] + return vals + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getIoValsIter(self, db, key): @@ -1374,9 +1431,13 @@ def getIoValsIter(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering proem + try: + if cursor.set_key(key): # moves to first_dup + for val in cursor.iternext_dup(): + yield val[33:] # slice off prepended ordering proem + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getIoValLast(self, db, key): @@ -1394,10 +1455,14 @@ def getIoValLast(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() val = None - if cursor.set_key(key): # move to first_dup - if cursor.last_dup(): # move to last_dup - val = cursor.value()[33:] # slice off prepended ordering proem - return val + try: + if cursor.set_key(key): # move to first_dup + if cursor.last_dup(): # move to last_dup + val = cursor.value()[33:] # slice off prepended ordering proem + return val + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getIoItemsNext(self, db, key=b"", skip=True): @@ -1479,12 +1544,16 @@ def cntIoVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() count = 0 - if cursor.set_key(key): # moves to first_dup - count = cursor.count() + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return count - def delIoVals(self,db, key): + def delIoVals(self, db, key): """ Deletes all values at key in db if key present. Returns True If key exists @@ -1495,7 +1564,11 @@ def delIoVals(self,db, key): """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.delete(key)) + try: + return (txn.delete(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def delIoVal(self, db, key, val): @@ -1529,10 +1602,14 @@ def delIoVal(self, db, key, val): with self.env.begin(db=db, write=True, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_key(key): # move to first_dup - for proval in cursor.iternext_dup(): # value with proem - if val == proval[33:]: # strip of proem - return cursor.delete() + try: + if cursor.set_key(key): # move to first_dup + for proval in cursor.iternext_dup(): # value with proem + if val == proval[33:]: # strip of proem + return cursor.delete() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return False @@ -1563,6 +1640,7 @@ def getIoValsAllPreIter(self, db, pre): yield val[33:] key = snKey(pre, cnt:=cnt+1) + def getIoValsAllPreBackIter(self, db, pre, fn): """ Returns iterator of all dup vals in insertion order for all entries @@ -1657,5 +1735,3 @@ def getIoValsAnyPreIter(self, db, pre): yield val[33:] # slice off prepended ordering prefix cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) - - diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 8d1ea1f62..b4118d4b5 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -791,6 +791,64 @@ def test_lmdber(): assert dber.setIoSetVals(db, key2, vals3) assert dber.getIoSetVals(db, key2) == vals3 + # Empty keys cause lmdb.BalValsizeError so LMDBer now throws a KeyError + # if it catches this kind of thing in the various places where it gets + # thrown + empty_key = ''.encode('utf8') + some_value = 'foo'.encode('utf8') + with pytest.raises(KeyError): + dber.putVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.setVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.getVal(db, empty_key) + with pytest.raises(KeyError): + dber.delVal(db, empty_key) + dber.putIoSetVals(db, empty_key, [some_value]) + dber.addIoSetVal(db, empty_key, some_value) + dber.setIoSetVals(db, empty_key, [some_value]) + dber.appendIoSetVal(db, empty_key, some_value) + dber.getIoSetVals(db, empty_key) + [_ for _ in dber.getIoSetValsIter(db, empty_key)] + dber.getIoSetValLast(db, empty_key) + dber.cntIoSetVals(db, empty_key) + dber.delIoSetVals(db, empty_key) + dber.delIoSetVal(db, empty_key, some_value) + dber.getIoSetItems(db, empty_key) + dber.getIoSetItemsIter(db, empty_key) + with pytest.raises(KeyError): + dber.delIoSetIokey(db, empty_key) + with pytest.raises(KeyError): + dber.putVals(db, empty_key, [some_value]) + with pytest.raises(KeyError): + dber.addVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.getVals(db, empty_key) + with pytest.raises(KeyError): + dber.getValLast(db, empty_key) + with pytest.raises(KeyError): + [_ for _ in dber.getValsIter(db, empty_key)] + with pytest.raises(KeyError): + dber.cntVals(db, empty_key) + with pytest.raises(KeyError): + dber.delVals(db, empty_key) + with pytest.raises(KeyError): + dber.putIoVals(db, empty_key, [some_value]) + with pytest.raises(KeyError): + dber.addIoVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.getIoVals(db, empty_key) + with pytest.raises(KeyError): + [_ for _ in dber.getIoValsIter(db, empty_key)] + with pytest.raises(KeyError): + dber.getIoValLast(db, empty_key) + with pytest.raises(KeyError): + dber.cntIoVals(db, empty_key) + with pytest.raises(KeyError): + dber.delIoVals(db, empty_key) + with pytest.raises(KeyError): + dber.delIoVal(db, empty_key, some_value) + assert not os.path.exists(dber.path) """ End Test """