From 8337de07fab32773457864c0c9ccd83d13639328 Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Thu, 28 Sep 2023 10:08:54 -0400 Subject: [PATCH] LMDBer methods throw KeyError on malformed keys. (#576) LMBD throws lmdb.BadValSizeError which isn't very helpful (or caught) in the codebase although KeyErrors are because of the dict abstraction elsewhere. (Found while trying to resolve Issue #575) So now the LMDB methods throw KeyError where appropriate and tests for all methods with empty keys now execute regardless of whether those methods had the issue or not. --- src/keri/db/dbing.py | 162 ++++++++++++++++++++++++++++++----------- tests/db/test_dbing.py | 58 +++++++++++++++ 2 files changed, 177 insertions(+), 43 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 500623e65..8c5edc049 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -417,7 +417,11 @@ def putVal(self, db, key, val): val is bytes of value to be written """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.put(key, val, overwrite=False)) + try: + return (txn.put(key, val, overwrite=False)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def setVal(self, db, key, val): @@ -432,7 +436,11 @@ def setVal(self, db, key, val): val is bytes of value to be written """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.put(key, val)) + try: + return (txn.put(key, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getVal(self, db, key): @@ -446,7 +454,11 @@ def getVal(self, db, key): """ with self.env.begin(db=db, write=False, buffers=True) as txn: - return( txn.get(key)) + try: + return(txn.get(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def delVal(self, db, key): @@ -459,7 +471,11 @@ def delVal(self, db, key): key is bytes of key within sub db's keyspace """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.delete(key)) + try: + return (txn.delete(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def cnt(self, db): @@ -1108,7 +1124,11 @@ def delIoSetIokey(self, db, iokey): iokey (bytes): actual key with ordinal key suffix """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return txn.delete(iokey) + try: + return txn.delete(iokey) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") # For subdbs that support duplicates at each key (dupsort==True) @@ -1130,8 +1150,12 @@ def putVals(self, db, key, vals): """ with self.env.begin(db=db, write=True, buffers=True) as txn: result = True - for val in vals: - result = result and txn.put(key, val, dupdata=True) + try: + for val in vals: + result = result and txn.put(key, val, dupdata=True) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return result @@ -1158,7 +1182,11 @@ def addVal(self, db, key, val): result = False if val not in dups: with self.env.begin(db=db, write=True, buffers=True) as txn: - result = txn.put(key, val, dupdata=True) + try: + result = txn.put(key, val, dupdata=True) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return result @@ -1177,8 +1205,12 @@ def getVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - vals = [val for val in cursor.iternext_dup()] + try: + if cursor.set_key(key): # moves to first_dup + vals = [val for val in cursor.iternext_dup()] + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return vals @@ -1196,9 +1228,13 @@ def getValLast(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() val = None - if cursor.set_key(key): # move to first_dup - if cursor.last_dup(): # move to last_dup - val = cursor.value() + try: + if cursor.set_key(key): # move to first_dup + if cursor.last_dup(): # move to last_dup + val = cursor.value() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return val @@ -1216,9 +1252,13 @@ def getValsIter(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val + try: + if cursor.set_key(key): # moves to first_dup + for val in cursor.iternext_dup(): + yield val + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def cntVals(self, db, key): @@ -1232,8 +1272,12 @@ def cntVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() count = 0 - if cursor.set_key(key): # moves to first_dup - count = cursor.count() + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return count @@ -1263,6 +1307,7 @@ def cntValsAllPre(self, db, pre, on=0): return count + def delVals(self, db, key, val=b''): """ Deletes all values at key in db if val=b'' else deletes the dup @@ -1275,7 +1320,11 @@ def delVals(self, db, key, val=b''): val is bytes of dup val at key to delete """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.delete(key, val)) + try: + return (txn.delete(key, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") # For subdbs that support insertion order preserving duplicates at each key. @@ -1309,9 +1358,13 @@ def putIoVals(self, db, key, vals): with self.env.begin(db=db, write=True, buffers=True) as txn: idx = 0 cursor = txn.cursor() - if cursor.set_key(key): # move to key if any - if cursor.last_dup(): # move to last dup - idx = 1 + int(bytes(cursor.value()[:32]), 16) # get last index as int + try: + if cursor.set_key(key): # move to key if any + if cursor.last_dup(): # move to last dup + idx = 1 + int(bytes(cursor.value()[:32]), 16) # get last index as int + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") for val in vals: if val not in dups: @@ -1353,10 +1406,14 @@ def getIoVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - # slice off prepended ordering proem - vals = [val[33:] for val in cursor.iternext_dup()] - return vals + try: + if cursor.set_key(key): # moves to first_dup + # slice off prepended ordering proem + vals = [val[33:] for val in cursor.iternext_dup()] + return vals + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getIoValsIter(self, db, key): @@ -1374,9 +1431,13 @@ def getIoValsIter(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() vals = [] - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering proem + try: + if cursor.set_key(key): # moves to first_dup + for val in cursor.iternext_dup(): + yield val[33:] # slice off prepended ordering proem + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getIoValLast(self, db, key): @@ -1394,10 +1455,14 @@ def getIoValLast(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() val = None - if cursor.set_key(key): # move to first_dup - if cursor.last_dup(): # move to last_dup - val = cursor.value()[33:] # slice off prepended ordering proem - return val + try: + if cursor.set_key(key): # move to first_dup + if cursor.last_dup(): # move to last_dup + val = cursor.value()[33:] # slice off prepended ordering proem + return val + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def getIoItemsNext(self, db, key=b"", skip=True): @@ -1479,12 +1544,16 @@ def cntIoVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() count = 0 - if cursor.set_key(key): # moves to first_dup - count = cursor.count() + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return count - def delIoVals(self,db, key): + def delIoVals(self, db, key): """ Deletes all values at key in db if key present. Returns True If key exists @@ -1495,7 +1564,11 @@ def delIoVals(self,db, key): """ with self.env.begin(db=db, write=True, buffers=True) as txn: - return (txn.delete(key)) + try: + return (txn.delete(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") def delIoVal(self, db, key, val): @@ -1529,10 +1602,14 @@ def delIoVal(self, db, key, val): with self.env.begin(db=db, write=True, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_key(key): # move to first_dup - for proval in cursor.iternext_dup(): # value with proem - if val == proval[33:]: # strip of proem - return cursor.delete() + try: + if cursor.set_key(key): # move to first_dup + for proval in cursor.iternext_dup(): # value with proem + if val == proval[33:]: # strip of proem + return cursor.delete() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return False @@ -1563,6 +1640,7 @@ def getIoValsAllPreIter(self, db, pre): yield val[33:] key = snKey(pre, cnt:=cnt+1) + def getIoValsAllPreBackIter(self, db, pre, fn): """ Returns iterator of all dup vals in insertion order for all entries @@ -1657,5 +1735,3 @@ def getIoValsAnyPreIter(self, db, pre): yield val[33:] # slice off prepended ordering prefix cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) - - diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 8d1ea1f62..b4118d4b5 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -791,6 +791,64 @@ def test_lmdber(): assert dber.setIoSetVals(db, key2, vals3) assert dber.getIoSetVals(db, key2) == vals3 + # Empty keys cause lmdb.BalValsizeError so LMDBer now throws a KeyError + # if it catches this kind of thing in the various places where it gets + # thrown + empty_key = ''.encode('utf8') + some_value = 'foo'.encode('utf8') + with pytest.raises(KeyError): + dber.putVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.setVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.getVal(db, empty_key) + with pytest.raises(KeyError): + dber.delVal(db, empty_key) + dber.putIoSetVals(db, empty_key, [some_value]) + dber.addIoSetVal(db, empty_key, some_value) + dber.setIoSetVals(db, empty_key, [some_value]) + dber.appendIoSetVal(db, empty_key, some_value) + dber.getIoSetVals(db, empty_key) + [_ for _ in dber.getIoSetValsIter(db, empty_key)] + dber.getIoSetValLast(db, empty_key) + dber.cntIoSetVals(db, empty_key) + dber.delIoSetVals(db, empty_key) + dber.delIoSetVal(db, empty_key, some_value) + dber.getIoSetItems(db, empty_key) + dber.getIoSetItemsIter(db, empty_key) + with pytest.raises(KeyError): + dber.delIoSetIokey(db, empty_key) + with pytest.raises(KeyError): + dber.putVals(db, empty_key, [some_value]) + with pytest.raises(KeyError): + dber.addVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.getVals(db, empty_key) + with pytest.raises(KeyError): + dber.getValLast(db, empty_key) + with pytest.raises(KeyError): + [_ for _ in dber.getValsIter(db, empty_key)] + with pytest.raises(KeyError): + dber.cntVals(db, empty_key) + with pytest.raises(KeyError): + dber.delVals(db, empty_key) + with pytest.raises(KeyError): + dber.putIoVals(db, empty_key, [some_value]) + with pytest.raises(KeyError): + dber.addIoVal(db, empty_key, some_value) + with pytest.raises(KeyError): + dber.getIoVals(db, empty_key) + with pytest.raises(KeyError): + [_ for _ in dber.getIoValsIter(db, empty_key)] + with pytest.raises(KeyError): + dber.getIoValLast(db, empty_key) + with pytest.raises(KeyError): + dber.cntIoVals(db, empty_key) + with pytest.raises(KeyError): + dber.delIoVals(db, empty_key) + with pytest.raises(KeyError): + dber.delIoVal(db, empty_key, some_value) + assert not os.path.exists(dber.path) """ End Test """