Skip to content

Commit

Permalink
LMDBer methods throw KeyError on malformed keys. (#576)
Browse files Browse the repository at this point in the history 
LMBD throws lmdb.BadValSizeError which isn't very helpful (or caught) in
the codebase although KeyErrors are because of the dict abstraction
elsewhere.  (Found while trying to resolve Issue #575)

So now the LMDB methods throw KeyError where appropriate and tests for
all methods with empty keys now execute regardless of whether those
methods had the issue or not.
  • Loading branch information
@daidoji
daidoji authored Sep 28, 2023
1 parent 5fc7271 commit 8337de0
Show file tree
Hide file tree
Showing 2 changed files with 177 additions and 43 deletions.
162 changes: 119 additions & 43 deletions src/keri/db/dbing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,11 @@ def putVal(self, db, key, val):
val is bytes of value to be written
"""
with self.env.begin(db=db, write=True, buffers=True) as txn:
return (txn.put(key, val, overwrite=False))
try:
return (txn.put(key, val, overwrite=False))
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def setVal(self, db, key, val):
Expand All @@ -432,7 +436,11 @@ def setVal(self, db, key, val):
val is bytes of value to be written
"""
with self.env.begin(db=db, write=True, buffers=True) as txn:
return (txn.put(key, val))
try:
return (txn.put(key, val))
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def getVal(self, db, key):
Expand All @@ -446,7 +454,11 @@ def getVal(self, db, key):
"""
with self.env.begin(db=db, write=False, buffers=True) as txn:
return( txn.get(key))
try:
return(txn.get(key))
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def delVal(self, db, key):
Expand All @@ -459,7 +471,11 @@ def delVal(self, db, key):
key is bytes of key within sub db's keyspace
"""
with self.env.begin(db=db, write=True, buffers=True) as txn:
return (txn.delete(key))
try:
return (txn.delete(key))
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def cnt(self, db):
Expand Down Expand Up @@ -1108,7 +1124,11 @@ def delIoSetIokey(self, db, iokey):
iokey (bytes): actual key with ordinal key suffix
"""
with self.env.begin(db=db, write=True, buffers=True) as txn:
return txn.delete(iokey)
try:
return txn.delete(iokey)
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


# For subdbs that support duplicates at each key (dupsort==True)
Expand All @@ -1130,8 +1150,12 @@ def putVals(self, db, key, vals):
"""
with self.env.begin(db=db, write=True, buffers=True) as txn:
result = True
for val in vals:
result = result and txn.put(key, val, dupdata=True)
try:
for val in vals:
result = result and txn.put(key, val, dupdata=True)
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return result


Expand All @@ -1158,7 +1182,11 @@ def addVal(self, db, key, val):
result = False
if val not in dups:
with self.env.begin(db=db, write=True, buffers=True) as txn:
result = txn.put(key, val, dupdata=True)
try:
result = txn.put(key, val, dupdata=True)
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return result


Expand All @@ -1177,8 +1205,12 @@ def getVals(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
vals = []
if cursor.set_key(key): # moves to first_dup
vals = [val for val in cursor.iternext_dup()]
try:
if cursor.set_key(key): # moves to first_dup
vals = [val for val in cursor.iternext_dup()]
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return vals


Expand All @@ -1196,9 +1228,13 @@ def getValLast(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
val = None
if cursor.set_key(key): # move to first_dup
if cursor.last_dup(): # move to last_dup
val = cursor.value()
try:
if cursor.set_key(key): # move to first_dup
if cursor.last_dup(): # move to last_dup
val = cursor.value()
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return val


Expand All @@ -1216,9 +1252,13 @@ def getValsIter(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
vals = []
if cursor.set_key(key): # moves to first_dup
for val in cursor.iternext_dup():
yield val
try:
if cursor.set_key(key): # moves to first_dup
for val in cursor.iternext_dup():
yield val
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def cntVals(self, db, key):
Expand All @@ -1232,8 +1272,12 @@ def cntVals(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
count = 0
if cursor.set_key(key): # moves to first_dup
count = cursor.count()
try:
if cursor.set_key(key): # moves to first_dup
count = cursor.count()
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return count


Expand Down Expand Up @@ -1263,6 +1307,7 @@ def cntValsAllPre(self, db, pre, on=0):

return count


def delVals(self, db, key, val=b''):
"""
Deletes all values at key in db if val=b'' else deletes the dup
Expand All @@ -1275,7 +1320,11 @@ def delVals(self, db, key, val=b''):
val is bytes of dup val at key to delete
"""
with self.env.begin(db=db, write=True, buffers=True) as txn:
return (txn.delete(key, val))
try:
return (txn.delete(key, val))
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


# For subdbs that support insertion order preserving duplicates at each key.
Expand Down Expand Up @@ -1309,9 +1358,13 @@ def putIoVals(self, db, key, vals):
with self.env.begin(db=db, write=True, buffers=True) as txn:
idx = 0
cursor = txn.cursor()
if cursor.set_key(key): # move to key if any
if cursor.last_dup(): # move to last dup
idx = 1 + int(bytes(cursor.value()[:32]), 16) # get last index as int
try:
if cursor.set_key(key): # move to key if any
if cursor.last_dup(): # move to last dup
idx = 1 + int(bytes(cursor.value()[:32]), 16) # get last index as int
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")

for val in vals:
if val not in dups:
Expand Down Expand Up @@ -1353,10 +1406,14 @@ def getIoVals(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
vals = []
if cursor.set_key(key): # moves to first_dup
# slice off prepended ordering proem
vals = [val[33:] for val in cursor.iternext_dup()]
return vals
try:
if cursor.set_key(key): # moves to first_dup
# slice off prepended ordering proem
vals = [val[33:] for val in cursor.iternext_dup()]
return vals
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def getIoValsIter(self, db, key):
Expand All @@ -1374,9 +1431,13 @@ def getIoValsIter(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
vals = []
if cursor.set_key(key): # moves to first_dup
for val in cursor.iternext_dup():
yield val[33:] # slice off prepended ordering proem
try:
if cursor.set_key(key): # moves to first_dup
for val in cursor.iternext_dup():
yield val[33:] # slice off prepended ordering proem
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def getIoValLast(self, db, key):
Expand All @@ -1394,10 +1455,14 @@ def getIoValLast(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
val = None
if cursor.set_key(key): # move to first_dup
if cursor.last_dup(): # move to last_dup
val = cursor.value()[33:] # slice off prepended ordering proem
return val
try:
if cursor.set_key(key): # move to first_dup
if cursor.last_dup(): # move to last_dup
val = cursor.value()[33:] # slice off prepended ordering proem
return val
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def getIoItemsNext(self, db, key=b"", skip=True):
Expand Down Expand Up @@ -1479,12 +1544,16 @@ def cntIoVals(self, db, key):
with self.env.begin(db=db, write=False, buffers=True) as txn:
cursor = txn.cursor()
count = 0
if cursor.set_key(key): # moves to first_dup
count = cursor.count()
try:
if cursor.set_key(key): # moves to first_dup
count = cursor.count()
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return count


def delIoVals(self,db, key):
def delIoVals(self, db, key):
"""
Deletes all values at key in db if key present.
Returns True If key exists
Expand All @@ -1495,7 +1564,11 @@ def delIoVals(self,db, key):
"""

with self.env.begin(db=db, write=True, buffers=True) as txn:
return (txn.delete(key))
try:
return (txn.delete(key))
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")


def delIoVal(self, db, key, val):
Expand Down Expand Up @@ -1529,10 +1602,14 @@ def delIoVal(self, db, key, val):

with self.env.begin(db=db, write=True, buffers=True) as txn:
cursor = txn.cursor()
if cursor.set_key(key): # move to first_dup
for proval in cursor.iternext_dup(): # value with proem
if val == proval[33:]: # strip of proem
return cursor.delete()
try:
if cursor.set_key(key): # move to first_dup
for proval in cursor.iternext_dup(): # value with proem
if val == proval[33:]: # strip of proem
return cursor.delete()
except lmdb.BadValsizeError as ex:
raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb),"
" or wrong DUPFIXED size. ref) lmdb.BadValsizeError")
return False


Expand Down Expand Up @@ -1563,6 +1640,7 @@ def getIoValsAllPreIter(self, db, pre):
yield val[33:]
key = snKey(pre, cnt:=cnt+1)


def getIoValsAllPreBackIter(self, db, pre, fn):
"""
Returns iterator of all dup vals in insertion order for all entries
Expand Down Expand Up @@ -1657,5 +1735,3 @@ def getIoValsAnyPreIter(self, db, pre):
yield val[33:] # slice off prepended ordering prefix
cnt = int(back, 16)
key = snKey(pre, cnt:=cnt+1)


58 changes: 58 additions & 0 deletions tests/db/test_dbing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,64 @@ def test_lmdber():
assert dber.setIoSetVals(db, key2, vals3)
assert dber.getIoSetVals(db, key2) == vals3

# Empty keys cause lmdb.BalValsizeError so LMDBer now throws a KeyError
# if it catches this kind of thing in the various places where it gets
# thrown
empty_key = ''.encode('utf8')
some_value = 'foo'.encode('utf8')
with pytest.raises(KeyError):
dber.putVal(db, empty_key, some_value)
with pytest.raises(KeyError):
dber.setVal(db, empty_key, some_value)
with pytest.raises(KeyError):
dber.getVal(db, empty_key)
with pytest.raises(KeyError):
dber.delVal(db, empty_key)
dber.putIoSetVals(db, empty_key, [some_value])
dber.addIoSetVal(db, empty_key, some_value)
dber.setIoSetVals(db, empty_key, [some_value])
dber.appendIoSetVal(db, empty_key, some_value)
dber.getIoSetVals(db, empty_key)
[_ for _ in dber.getIoSetValsIter(db, empty_key)]
dber.getIoSetValLast(db, empty_key)
dber.cntIoSetVals(db, empty_key)
dber.delIoSetVals(db, empty_key)
dber.delIoSetVal(db, empty_key, some_value)
dber.getIoSetItems(db, empty_key)
dber.getIoSetItemsIter(db, empty_key)
with pytest.raises(KeyError):
dber.delIoSetIokey(db, empty_key)
with pytest.raises(KeyError):
dber.putVals(db, empty_key, [some_value])
with pytest.raises(KeyError):
dber.addVal(db, empty_key, some_value)
with pytest.raises(KeyError):
dber.getVals(db, empty_key)
with pytest.raises(KeyError):
dber.getValLast(db, empty_key)
with pytest.raises(KeyError):
[_ for _ in dber.getValsIter(db, empty_key)]
with pytest.raises(KeyError):
dber.cntVals(db, empty_key)
with pytest.raises(KeyError):
dber.delVals(db, empty_key)
with pytest.raises(KeyError):
dber.putIoVals(db, empty_key, [some_value])
with pytest.raises(KeyError):
dber.addIoVal(db, empty_key, some_value)
with pytest.raises(KeyError):
dber.getIoVals(db, empty_key)
with pytest.raises(KeyError):
[_ for _ in dber.getIoValsIter(db, empty_key)]
with pytest.raises(KeyError):
dber.getIoValLast(db, empty_key)
with pytest.raises(KeyError):
dber.cntIoVals(db, empty_key)
with pytest.raises(KeyError):
dber.delIoVals(db, empty_key)
with pytest.raises(KeyError):
dber.delIoVal(db, empty_key, some_value)

assert not os.path.exists(dber.path)

""" End Test """
Expand Down

0 comments on commit 8337de0

Please sign in to comment.