From 8ae368317f8daf1b5c0b6dbf1bee564c4649573b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 18:04:34 -0600 Subject: [PATCH 1/6] fixed comments in Codex --- src/keri/core/coring.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 44f962c1d..7b6c8f0f3 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -280,10 +280,10 @@ class MatterCodex: Large: str = 'S' # Large 11 byte b2 number Great: str = 'T' # Great 14 byte b2 number Vast: str = 'U' # Vast 17 byte b2 number - Label1: str = 'V' # Label1 as 1 bytes for label lead size 1 - Label2: str = 'W' # Label2 as 2 bytes for label lead size 0 - Tag3: str = 'X' # Tag3 3 B64 encoded chars for field tag - Tag7: str = 'Y' # Tag7 7 B64 encoded chars for field tag + Label1: str = 'V' # Label1 1 bytes for label lead size 1 + Label2: str = 'W' # Label2 2 bytes for label lead size 0 + Tag3: str = 'X' # Tag3 3 B64 encoded chars for special values + Tag7: str = 'Y' # Tag7 7 B64 encoded chars for special values Blind: str = 'Z' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt Salt_128: str = '0A' # random salt/seed/nonce/private key or number of length 128 bits (Huge) Ed25519_Sig: str = '0B' # Ed25519 signature. @@ -294,12 +294,12 @@ class MatterCodex: SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. Long: str = '0H' # Long 4 byte b2 number ECDSA_256r1_Sig: str = '0I' # ECDSA secp256r1 signature. - Tag1: str = '0J' # Tag1 1 prepad + 1 B64 encoded char for field tag - Tag2: str = '0K' # Tag2 2 B64 encoded chars for field tag - Tag5: str = '0L' # Tag5 1 prepad + 5 B64 encoded chars for field tag - Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag - Tag9: str = '0N' # Tag9 1 prepad + 9 B64 encoded chars for field tag - Tag10: str = '0O' # Tag10 10 B64 encoded chars for field tag + Tag1: str = '0J' # Tag1 1 B64 encoded char + 1 prepad for special values + Tag2: str = '0K' # Tag2 2 B64 encoded chars for for special values + Tag5: str = '0L' # Tag5 5 B64 encoded chars + 1 prepad for special values + Tag6: str = '0M' # Tag6 6 B64 encoded chars for special values + Tag9: str = '0N' # Tag9 9 B64 encoded chars + 1 prepad for special values + Tag10: str = '0O' # Tag10 10 B64 encoded chars for special values ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. @@ -313,8 +313,8 @@ class MatterCodex: Null: str = '1AAK' # Null None or empty value No: str = '1AAL' # No Falsey Boolean value Yes: str = '1AAM' # Yes Truthy Boolean value - Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for field tag - Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for field tag + Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for special values + Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for special values TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 From 0808304af1fb5771fe47fdf399e3062d39d7a98b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 28 Mar 2024 18:49:41 -0600 Subject: [PATCH 2/6] started unit test dev for Matter with special soft --- src/keri/core/coring.py | 4 +-- tests/core/test_coring.py | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 7b6c8f0f3..797b25163 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -926,7 +926,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, soft = soft[:ss] - if not Reb64.match(soft): + if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") @@ -961,7 +961,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, soft = soft[:ss] - if not Reb64.match(soft): + if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") self._code = code # str hard part of code diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 96e84ad0c..d8b52195e 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -1471,6 +1471,62 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == False + """ Done Test """ + +def test_matter_special(): + """ + Test Matter instances using code with special soft values + """ + # test Tag3 + + code = MtrDex.Tag3 + soft = 'icp' + qb64 = 'Xicp' + qb2 = b"^')" + raw = b'' + + matter = Matter(code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + code = matter.code + soft = matter.soft + qb2 = matter.qb2 + qb64 = matter.qb64 + + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + # Test corner conditions + # Empty raw + matter = Matter(raw=b'', code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + + + + """ Done Test """ @@ -6534,6 +6590,7 @@ def test_tholder(): if __name__ == "__main__": test_matter_class() test_matter() + test_matter_special() test_verser() #test_texter() #test_counter() From 6473d7efe811254763986555dc1d9873b9d8524d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 08:32:09 -0600 Subject: [PATCH 3/6] Refactor so Matter.soft is determinative and .size is derived --- src/keri/core/coring.py | 35 +++++++++++++---------------------- tests/core/test_coring.py | 11 ++++++++--- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 797b25163..91ea41e1d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -871,12 +871,12 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, size = (rize + ls) // 3 # calculate value of size in triplets if code[0] in SmallVrzDex: # compute code with sizes - if size <= (64 ** 2 - 1): + if size <= (64 ** 2 - 1): # ss = 2 hs = 2 s = astuple(SmallVrzDex)[ls] code = f"{s}{code[1:hs]}" ss = 2 - elif size <= (64 ** 4 - 1): # make big version of code + elif size <= (64 ** 4 - 1): # ss = 4 make big version of code hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{'A' * (hs - 2)}{code[1]}" @@ -886,7 +886,7 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise InvalidVarRawSizeError(f"Unsupported raw size for " f"{code=}.") elif code[0] in LargeVrzDex: # compute code with sizes - if size <= (64 ** 4 - 1): + if size <= (64 ** 4 - 1): # ss = 4 hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{code[1:hs]}" @@ -1082,7 +1082,8 @@ def size(self): sized primitive material (fs = None). """ - return self._size + #return self._size + return (b64ToInt(self.soft) if self.soft else None) @property @@ -1093,10 +1094,12 @@ def both(self): """ _, ss, _, _ = self.Sizes[self.code] - if self.size is not None: - return (f"{self.code}{intToB64(self.size, l=ss)}") - else: - return (f"{self.code}{self.soft}") + #if self.size is not None: + #return (f"{self.code}{intToB64(self.size, l=ss)}") + #else: + #return (f"{self.code}{self.soft}") + + return (f"{self.code}{self.soft}") @property @@ -1211,16 +1214,10 @@ def _infil(self): ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes hs, ss, fs, ls = self.Sizes[code] + # assumes unit tests on Matter.Sizes ensure valid size entries if not fs: # variable sized, compute code ss value from .size cs = hs + ss # both hard + soft size - if ss < 1: # ss < 1 so not variable sized - raise InvalidCodeSizeError(f"Soft size {ss=} must be positive for " - f" variable length material.") - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if size < 0 or size > (64 ** ss - 1): raise InvalidVarSizeError("Invalid size={} for code={}." "".format(size, code)) @@ -1258,14 +1255,8 @@ def _binfil(self): hs, ss, fs, ls = self.Sizes[code] cs = hs + ss - + # assumes unit tests on Matter.Sizes ensure valid size entries if not fs: # compute both and fs from size - if ss < 1: # ss < 1 so not variable sized - raise InvalidCodeSizeError(f"Soft size {ss=} must be positive for " - f" variable length material.") - if cs % 4: - raise InvalidCodeSizeError("Whole code size not multiple of 4 for " - "variable length material. cs={}.".format(cs)) if size < 0 or size > (64 ** ss - 1): raise InvalidVarSizeError("Invalid size={} for code={}." diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index d8b52195e..7011892b4 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -3,7 +3,7 @@ tests.core.test_coring module """ -from dataclasses import asdict +from dataclasses import asdict, astuple import hashlib import json from base64 import urlsafe_b64decode as decodeB64 @@ -261,8 +261,7 @@ def test_matter_class(): # verify all Codes - # if fs None else not (hs + ss) % 4 - for val in Matter.Sizes.values(): + for code, val in Matter.Sizes.items(): # hard code assert (isinstance(val.hs, int) and isinstance(val.ss, int) and isinstance(val.ls, int)) assert val.hs > 0 and val.ss >= 0 and val.ls >= 0 @@ -276,6 +275,12 @@ def test_matter_class(): assert val.ss == 0 else: # variable sized assert val.ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + if code[0] in coring.SmallVrzDex: # small variable sized code + assert val.hs == 2 and val.ss == 2 and val.fs == None + assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + elif code[0] in coring.LargeVrzDex: # large veriable sized code + assert val.hs == 4 and val.ss == 4 and val.fs == None + assert code[0] == astuple(coring.LargeVrzDex)[val.ls] # Test .Hards # verify first hs Sizes matches hs in Codes for same first char From 338507db9a15cb6bb99d939d75b7855db54239c5 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 15:44:41 -0600 Subject: [PATCH 4/6] added support for special soft when fixed size and non-empty raw in code table. Need to update Matter init still. Added more thorough unit tests of valid Matter.Sizes table entries. --- src/keri/core/coring.py | 141 +++++++++++++++++--------------------- src/keri/help/helping.py | 2 +- tests/core/test_coring.py | 109 +++++++++++++++++++---------- 3 files changed, 136 insertions(+), 116 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 91ea41e1d..244333f89 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -315,8 +315,12 @@ class MatterCodex: Yes: str = '1AAM' # Yes Truthy Boolean value Tag4: str = '1AAN' # Tag4 4 B64 encoded chars for special values Tag8: str = '1AAO' # Tag8 8 B64 encoded chars for special values - TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 - TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 + TBD0S: str = '1__-' # Testing purposes only, fixed special values with non-empty raw lead size 0 + TBD0: str = '1___' # Testing purposes only, fixed with lead size 0 + TBD1S: str = '2__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 + TBD1: str = '2___' # Testing purposes only, fixed with lead size 1 + TBD2S: str = '3__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 + TBD2: str = '3___' # Testing purposes only, fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 StrB64_L1: str = '5A' # String Base64 only lead size 1 StrB64_L2: str = '6A' # String Base64 only lead size 2 @@ -677,9 +681,6 @@ class Matter: Hidden: _code (str): value for .code property _soft (str): soft value of full code - _size (int): value for .size property. Number of triplets of bytes - including lead bytes (quadlets of chars) of variable sized material - else None. _raw (bytes): value for .raw property _rawSize(): _leadSize(): @@ -690,7 +691,6 @@ class Matter: _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) - Size table rules for special soft values: if fn in table is None then must have ss > 0 and (hs + ss) % 4 else (fn is not None) then @@ -776,9 +776,13 @@ class Matter: '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAO': Sizage(hs=4, ss=8, fs=12, ls=0), - '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), - '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), + '1AAO': Sizage(hs=4, ss=2, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), '5A': Sizage(hs=2, ss=2, fs=None, ls=1), '6A': Sizage(hs=2, ss=2, fs=None, ls=2), @@ -937,7 +941,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._size = size # int of soft part value, None when fs != None self._raw = bytes(raw) # crypto ops require bytes not bytearray elif soft and code: # special when raw None @@ -966,7 +969,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._size = size # int of soft part value, None when fs != None self._raw = b'' # empty raw when special elif qb64b is not None: @@ -999,9 +1001,10 @@ def _rawSize(cls, code): """ hs, ss, fs, ls = cls.Sizes[code] # get sizes cs = hs + ss # both hard + soft code size + # assumes .Sizes only has valid entries if fs is None: raise InvalidCodeSizeError(f"Non-fixed raw size code {code}.") - return (((fs - cs) * 3 // 4) - ls) + return (((fs - cs) * 3 // 4) - ls) # assumes cs % 4 != 3 and fs % 4 == 0 @classmethod @@ -1071,18 +1074,14 @@ def size(self): Returns: size(int | None): Number of variably sized b64 quadlets/b2 triplets in primitive when varibly sized - None when not variably sized (fs==None) - - Getter for ._size. Makes ._size read only + None when not variably sized when (fs!=None) Number of quadlets/triplets of chars/bytes of variable sized material or None when not variably sized. - Converted qb64 value to int of ss portion of full text code when variably - sized primitive material (fs = None). - + Converted qb64 value to int of soft ss portion of full text code + when variably sized primitive material (fs == None). """ - #return self._size return (b64ToInt(self.soft) if self.soft else None) @@ -1092,7 +1091,7 @@ def both(self): Returns: both (str): hard + soft parts of full text code """ - _, ss, _, _ = self.Sizes[self.code] + #_, ss, _, _ = self.Sizes[self.code] #if self.size is not None: #return (f"{self.code}{intToB64(self.size, l=ss)}") @@ -1208,37 +1207,35 @@ def _infil(self): """ code = self.code # hard part of full code == codex value - soft = self.soft # soft part of full code may be empty - size = self.size # size if variable length, None otherwise - raw = self.raw # bytes or bytearray + both = self.both # code + soft, soft may be empty + raw = self.raw # bytes or bytearray, raw may be empty - ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes hs, ss, fs, ls = self.Sizes[code] + cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries - if not fs: # variable sized, compute code ss value from .size - cs = hs + ss # both hard + soft size - - if size < 0 or size > (64 ** ss - 1): - raise InvalidVarSizeError("Invalid size={} for code={}." - "".format(size, code)) - # both is hard code + size converted to ss B64 chars - both = f"{code}{intToB64(size, l=ss)}" - - if len(both) % 4 != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - # prepad, convert, and prepend + + if cs != len(both): + InvalidCodeSizeError(f"Invalid full code={both} for sizes {hs=} and" + f" {ss=}.") + + ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes + if (cs % 4) != ps - ls: # adjusted pad provided by full code given lead bytes + raise InvalidCodeSizeError(f"Invalid code={both} for converted " + f"raw pad size={ps} and lead size={ls}.") + + if not fs: # variable sized + # prepad, convert, and prepend. When ls and ps then ls accounts for + # which ensures encodeB64 does not have trailing B64 pad chars return (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) else: # fixed size so prepad but lead ls may not be zero - both = code + soft - cs = len(both) - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") # prepad, convert, and replace upfront - # when fixed and ls != 0 then cs % 4 is zero and ps==ls - # otherwise when fixed and ls == 0 then cs % 4 == ps + # When fixed fs, raw may have ps != 0 when ls = 0 which hs+ss must + # provide so encodeB64 may have trailing B64 pad chars so need to + # strip these off. When fixed and ls != 0 then ps == ls + # When fixed and ls != 0 then cs % 4 is zero and ps==ls so using + # ps instead of ls for prepad works. Otherwise when fixed and + # ls == 0 then cs % 4 == ps so strip compensates for prepad. return (both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[cs % 4:]) @@ -1248,47 +1245,36 @@ def _binfil(self): self.code converted to Base2 + self.raw left shifted with pad bits equivalent of Base64 decode of .qb64 into .qb2 """ - code = self.code # codex value - soft = self.soft - size = self.size # optional size if variable length - raw = self.raw # bytes or bytearray + code = self.code # hard part of full code == codex value + both = self.both # code + soft, soft may be empty + raw = self.raw # bytes or bytearray may be empty hs, ss, fs, ls = self.Sizes[code] cs = hs + ss # assumes unit tests on Matter.Sizes ensure valid size entries - if not fs: # compute both and fs from size - - if size < 0 or size > (64 ** ss - 1): - raise InvalidVarSizeError("Invalid size={} for code={}." - "".format(size, code)) - # both is hard code + converted index - both = f"{code}{intToB64(size, l=ss)}" - fs = hs + ss + (size * 4) - else: - both = code + soft - - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(code))) - n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code # convert code both to right align b2 int then left shift in pad bits # then convert to bytes bcode = (b64ToInt(both) << (2 * (cs % 4))).to_bytes(n, 'big') - full = bcode + bytes([0] * ls) + raw + full = bcode + bytes([0] * ls) + raw # includes lead bytes + bfs = len(full) + if not fs: # compute fs + fs = hs + ss + (len(raw) + ls) * 4 // 3 # hs + ss + (size * 4) if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") - + raise InvalidCodeSizeError(f"Invalid full code={both} for raw size" + f"={len(raw)}.") return full def _exfil(self, qb64b): """ - Extracts self.code and self.raw from qualified base64 bytes qb64b + Extracts self.code and self.raw from qualified base64 str or bytes qb64b + Detects is str and converts to bytes + + Parameters: + qb64b (str | bytes | bytearray): fully qualified base64 from stream - cs = hs + ss - fs = (size * 4) + cs """ if not qb64b: # empty need more bytes raise ShortageError("Empty material.") @@ -1326,10 +1312,9 @@ def _exfil(self, qb64b): if hasattr(soft, "decode"): soft = soft.decode("utf-8") - size = None - if not fs: # compute fs from size chars in ss part of code - size = b64ToInt(soft) # compute variable size int may have value 0 - fs = (size * 4) + cs + if not fs: # compute fs from soft from ss part which provides size B64 + # compute variable size as int may have value 0 + fs = (b64ToInt(soft) * 4) + cs if len(qb64b) < fs: # need more bytes raise ShortageError(f"Need {fs - len(qb64b)} more chars.") @@ -1339,7 +1324,7 @@ def _exfil(self, qb64b): qb64b = qb64b.encode("utf-8") # check for non-zeroed pad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 + ps = cs % 4 # code pad size ps = cs mod 4 assumes cs mod 4 != 3 pbs = 2 * (ps if ps else ls) # pad bit size in bits if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero @@ -1368,7 +1353,6 @@ def _exfil(self, qb64b): self._code = hard # hard only self._soft = soft # soft only - self._size = size # variable size or None self._raw = raw # ensure bytes for crypto ops, may be empty @@ -1415,14 +1399,12 @@ def _bexfil(self, qb2): both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code soft = both[hs:hs + ss] # get soft may be empty - size = None if not fs: # compute fs from size chars in ss part of code - if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - size = b64ToInt(soft) # get size from soft - fs = (size * 4) + cs # compute fs + # compute size as int from soft part given by ss B64 chars + fs = (b64ToInt(soft) * 4) + cs # compute fs bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets if len(qb2) < bfs: # need more bytes @@ -1454,7 +1436,6 @@ def _bexfil(self, qb2): self._code = hard # hard only self._soft = soft # soft only may be empty - self._size = size # variable size or None self._raw = bytes(raw) # ensure bytes for crypto ops may be empty diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 75a709ee5..401d23db2 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -36,7 +36,7 @@ def sceil(r): """ Symmetric ceiling function Returns: - sc (int): value that is symmetric ceiling of r away from zero + sceil (int): value that is symmetric ceiling of r away from zero Because int() provides a symmetric floor towards zero, just inc int(r) by: 1 when r - int(r) > 0 (r positive) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 7011892b4..1c8d9275c 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -23,6 +23,12 @@ from cryptography.hazmat.primitives import hashes from cryptography import exceptions +from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, + ShortageError, InvalidCodeSizeError, InvalidVarIndexError, + InvalidValueError, DeserializeError, ValidationError, + InvalidVarRawSizeError) +from keri.kering import Version, Versionage, VersionError + from keri.core import coring from keri.core import eventing from keri.core.coring import (Ilkage, Ilks, Saids, Protocols, Protocolage, @@ -37,14 +43,10 @@ Diger, Prefixer, Cipher, Encrypter, Decrypter) from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN from keri.core.coring import generateSigners, generatePrivates -from keri.help.helping import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, - B64_CHARS, Reb64, nabSextets) + from keri.help import helping -from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, - ShortageError, InvalidCodeSizeError, InvalidVarIndexError, - InvalidValueError, DeserializeError, ValidationError, - InvalidVarRawSizeError) -from keri.kering import Version, Versionage, VersionError +from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, + B64_CHARS, Reb64, nabSextets) @@ -111,8 +113,12 @@ def test_matter_class(): 'Yes': '1AAM', 'Tag4': '1AAN', 'Tag8': '1AAO', - 'TBD1': '2AAA', - 'TBD2': '3AAA', + 'TBD0S': '1__-', + 'TBD0': '1___', + 'TBD1S': '2__-', + 'TBD1': '2___', + 'TBD2S': '3__-', + 'TBD2': '3___', 'StrB64_L0': '4A', 'StrB64_L1': '5A', 'StrB64_L2': '6A', @@ -218,9 +224,13 @@ def test_matter_class(): '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), '1AAN': Sizage(hs=4, ss=4, fs=8, ls=0), - '1AAO': Sizage(hs=4, ss=8, fs=12, ls=0), - '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), - '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), + '1AAO': Sizage(hs=4, ss=2, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, fs=8, ls=2), '4A': Sizage(hs=2, ss=2, fs=None, ls=0), '5A': Sizage(hs=2, ss=2, fs=None, ls=1), '6A': Sizage(hs=2, ss=2, fs=None, ls=2), @@ -262,24 +272,51 @@ def test_matter_class(): # verify all Codes for code, val in Matter.Sizes.items(): # hard code - assert (isinstance(val.hs, int) and isinstance(val.ss, int) and - isinstance(val.ls, int)) - assert val.hs > 0 and val.ss >= 0 and val.ls >= 0 - if val.fs is not None: # fixed sized - assert isinstance(val.fs, int) and val.fs > 0 and not val.fs % 4 - assert val.fs >= (val.hs + val.ss) - if val.ss > 0: # special soft value - assert val.fs == val.hs + val.ss # raw must be empty - assert val.ls == 0 # no lead - else: - assert val.ss == 0 - else: # variable sized - assert val.ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + hs = val.hs + ss = val.ss + fs = val.fs + ls = val.ls + cs = hs + ss + + assert (isinstance(hs, int) and isinstance(ss, int) and + isinstance(ls, int)) + assert hs > 0 and ss >= 0 and ls in (0, 1, 2) + assert len(code) == hs + + if fs is None: # variable sized + assert ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + + else: # fixed size + assert isinstance(fs, int) and fs > 0 and not fs % 4 + assert fs >= cs + assert cs % 4 != 3 # prevent ambiguous conversion + if ss > 0 and fs == cs: # special soft value with raw empty + assert ls == 0 # no lead + rs = (fs - cs) * 3 // 4 - ls # raw size bytes sans lead + ps = (3 - (rs + ls) % 3) % 3 # pad size given raw + lead + assert ps == (cs % 4) # cs % 4 is pad size given cs % 4 != 3 + assert sceil((rs + ls) * 4 / 3) + cs == fs # sextets add up + + + if code[0] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz': + assert len(code) == 1 + elif code[0] in '0': + assert len(code) == 2 + elif code[0] in '1': + assert len(code) == 4 and ls == 0 + elif code[0] in '2': + assert len(code) == 4 and ls == 1 + elif code[0] in '3': + assert len(code) == 4 and ls == 2 + else: + assert code[0] not in '-_' # count or op code + if code[0] in coring.SmallVrzDex: # small variable sized code - assert val.hs == 2 and val.ss == 2 and val.fs == None + assert val.hs == 2 and val.ss == 2 and val.fs is None assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + elif code[0] in coring.LargeVrzDex: # large veriable sized code - assert val.hs == 4 and val.ss == 4 and val.fs == None + assert val.hs == 4 and val.ss == 4 and val.fs is None assert code[0] == astuple(coring.LargeVrzDex)[val.ls] # Test .Hards @@ -295,6 +332,8 @@ def test_matter_class(): assert Matter._rawSize(MtrDex.Ed25519) == 32 assert Matter._leadSize(MtrDex.Ed25519) == 0 + assert not Matter._special(MtrDex.Ed25519) + assert Matter._special(MtrDex.Tag3) @@ -546,14 +585,14 @@ def test_matter(): assert ims == extra # stripped not include extra # test fix sized with leader 1 - # TBD1 = '2AAA' # Testing purposes only fixed with lead size 1 + # TBD1 = '2___' # Testing purposes only fixed with lead size 1 code = MtrDex.TBD1 # '2AAA' assert Matter._rawSize(code) == 2 assert Matter._leadSize(code) == 1 raw = b'ab' - qb64 = '2AAAAGFi' # '2AAA' + encodeB64(b'\x00ab').decode("utf-8") - qb2 = decodeB64(qb64) # b'\xd8\x00\x00\x00ab' + qb64 = '2___AGFi' # '2AAA' + encodeB64(b'\x00ab').decode("utf-8") + qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code @@ -616,7 +655,7 @@ def test_matter(): assert matter.prefixive == False # test with bad pad or lead - badqb64 = '2AAA_2Fi' # '2AAA' + encodeB64(b'\xffab').decode("utf-8") + badqb64 = '2____2Fi' # '2___' + encodeB64(b'\xffab').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xd8\x00\x00\xffab' with pytest.raises(ValueError) as ex: @@ -629,13 +668,13 @@ def test_matter(): # test fix sized with leader 2 - # TBD2 = '3AAA' # Testing purposes only of fixed with lead size 2 + # TBD2 = '3___' # Testing purposes only of fixed with lead size 2 code = MtrDex.TBD2 # '3AAA' assert Matter._rawSize(code) == 1 assert Matter._leadSize(code) == 2 raw = b'z' - qb64 = '3AAAAAB6' - qb2 = b'\xdc\x00\x00\x00\x00z' + qb64 = '3___AAB6' + qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code @@ -685,7 +724,7 @@ def test_matter(): assert matter.prefixive == False # test with bad pad or lead - badqb64 = '3AAA__96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") + badqb64 = '3_____96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") badqb2 = decodeB64(badqb64) #b'\xdc\x00\x00\xff\xffz' with pytest.raises(ValueError) as ex: From 4158da4b7b0f189dd614443dfea078cfb7561ea7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 16:46:48 -0600 Subject: [PATCH 5/6] clean up Matter code given new functionality, refinement and optimization get rid of unneccessary checks assuming unit text validates Sizes table thouroughly run time --- src/keri/core/coring.py | 82 +++++++++++++++++++-------------------- tests/core/test_coring.py | 19 +++++---- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 244333f89..2e006b26c 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -846,13 +846,10 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, .raw and .code and .size and .rsize """ - size = None # variable raw binary size including leader in quadlets - soft = soft # soft portion of code, if raw is not None: # raw provided but may be empty if not code: raise EmptyMaterialError(f"Improper initialization need either " f"(raw not None and code) or " - f"(raw empty and code and soft) or " f"(code and soft) or " f"qb64b or qb64 or qb2.") @@ -862,7 +859,12 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, if code not in self.Sizes: raise InvalidCodeError(f"Unsupported {code=}.") - if code[0] in SmallVrzDex or code[0] in LargeVrzDex: # dynamic size + hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + + if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex + #if not (code[0] in SmallVrzDex or code[0] in LargeVrzDex): + #raise InvalidCodeError(f"Non-variable size {code=}.") + if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " @@ -903,32 +905,27 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, f"{code=}.") soft = intToB64(size, ss) - else: - hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # invalid must not be variable size - raise InvalidVarSizeError(f"Unsupported {code=} for " - f"variable size {fs=}.") - + else: # fixed size but raw may be empty and/or special soft rize = Matter._rawSize(code) - if ss == 0 and soft: - raise InvalidSoftError(f"Non-empty {soft=} part when not" - f" special.") - - if fs == hs + ss and ss > 0: # special soft size - if ls != 0: # lead must be zero - raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" - f" {soft=} when special.") + #if ss == 0 and soft: + #raise InvalidSoftError(f"Non-empty {soft=} part when not" + #f" special.") - if rize: # raw must be empty - raise RawMaterialError(f"Nonzero raw size {rize=} when " - f" special {code=} {soft=}.") + if ss > 0: # special soft size, so soft must be provided + #if fs == hs + ss: + #if ls != 0: # lead must be zero + #raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" + #f" {soft=} when special.") - if not soft or len(soft) < ss: - raise SoftMaterialError(f"Not enough chars in {code=} " - f"{soft=} with {ss=}.") + #if rize: # raw must be empty + #raise RawMaterialError(f"Nonzero raw size {rize=} when " + #f" special {code=} {soft=}.") soft = soft[:ss] + if len(soft) != ss: + raise SoftMaterialError(f"Not enough chars in {soft=} " + f"with {ss=} for {code=}.") if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") @@ -943,33 +940,37 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, self._soft = soft # str soft part of code, empty when ss=0 self._raw = bytes(raw) # crypto ops require bytes not bytearray - elif soft and code: # special when raw None - hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # - raise InvalidSoftError(f"Unsupported {code=} {fs=} for special" - f" soft.") + elif soft and code: # fixed size and special when raw None + hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + if not fs: # variable sized code so can't be soft + raise InvalidSoftError(f"Unsupported variable sized {code=} " + f" with {fs=} for special {soft=}.") - if fs != hs + ss or ss == 0 or ls != 0: # not special soft code - raise InvalidSoftError("Invalid {code=} {fs=} or lead={ls} " - f" when special soft.") + if not ss > 0 or not ls == 0 or not fs == hs + ss: # not special soft + raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " + f" or {code=} {fs=} when special soft.") - rize = Matter._rawSize(code) - if rize: - raise InvalidSizeError(f"Nonzero raw size {rize=} when special" - f" {code=}.") + #rize = Matter._rawSize(code) + #if rize: + #raise InvalidSizeError(f"Nonzero raw size {rize=} when special" + #f" soft {code=}.") - if not soft or len(soft) < ss: - raise SoftMaterialError(f"Not enough chars in {code=} " - f"{soft=} with {ss=}.") + #if not soft or len(soft) < ss: + #raise SoftMaterialError(f"Not enough chars in {soft=} with " + #f"{ss=} for {code=}.") soft = soft[:ss] + if len(soft) != ss: + raise SoftMaterialError(f"Not enough chars in {soft=} " + f"with {ss=} for {code=}.") + if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.") self._code = code # str hard part of code self._soft = soft # str soft part of code, empty when ss=0 - self._raw = b'' # empty raw when special + self._raw = b'' # make raw empty when None and when special soft elif qb64b is not None: self._exfil(qb64b) @@ -987,7 +988,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, else: raise EmptyMaterialError(f"Improper initialization need either " f"(raw not None and code) or " - f"(raw empty and code and soft) or " f"(code and soft) or " f"qb64b or qb64 or qb2.") diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 1c8d9275c..e94bf0e71 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -285,13 +285,25 @@ def test_matter_class(): if fs is None: # variable sized assert ss > 0 and not ((val.hs + val.ss) % 4) # i.e. cs % 4 is 0 + assert code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex + + if code[0] in coring.SmallVrzDex: # small variable sized code + assert val.hs == 2 and val.ss == 2 and val.fs is None + assert code[0] == astuple(coring.SmallVrzDex)[val.ls] + + elif code[0] in coring.LargeVrzDex: # large veriable sized code + assert val.hs == 4 and val.ss == 4 and val.fs is None + assert code[0] == astuple(coring.LargeVrzDex)[val.ls] else: # fixed size + assert not (code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex) assert isinstance(fs, int) and fs > 0 and not fs % 4 assert fs >= cs assert cs % 4 != 3 # prevent ambiguous conversion if ss > 0 and fs == cs: # special soft value with raw empty assert ls == 0 # no lead + assert Matter._rawSize(code) == 0 + rs = (fs - cs) * 3 // 4 - ls # raw size bytes sans lead ps = (3 - (rs + ls) % 3) % 3 # pad size given raw + lead assert ps == (cs % 4) # cs % 4 is pad size given cs % 4 != 3 @@ -311,13 +323,6 @@ def test_matter_class(): else: assert code[0] not in '-_' # count or op code - if code[0] in coring.SmallVrzDex: # small variable sized code - assert val.hs == 2 and val.ss == 2 and val.fs is None - assert code[0] == astuple(coring.SmallVrzDex)[val.ls] - - elif code[0] in coring.LargeVrzDex: # large veriable sized code - assert val.hs == 4 and val.ss == 4 and val.fs is None - assert code[0] == astuple(coring.LargeVrzDex)[val.ls] # Test .Hards # verify first hs Sizes matches hs in Codes for same first char From 5f6120c58a729bd83f2b0ebd92da0dc679ea2c1a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 29 Mar 2024 16:52:52 -0600 Subject: [PATCH 6/6] remove commented out code --- src/keri/core/coring.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 2e006b26c..bd65d3a6d 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -862,9 +862,6 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, hs, ss, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex - #if not (code[0] in SmallVrzDex or code[0] in LargeVrzDex): - #raise InvalidCodeError(f"Non-variable size {code=}.") - if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " @@ -906,22 +903,9 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, soft = intToB64(size, ss) else: # fixed size but raw may be empty and/or special soft - rize = Matter._rawSize(code) - - #if ss == 0 and soft: - #raise InvalidSoftError(f"Non-empty {soft=} part when not" - #f" special.") + rize = Matter._rawSize(code) # get raw size from Sizes for code if ss > 0: # special soft size, so soft must be provided - #if fs == hs + ss: - #if ls != 0: # lead must be zero - #raise InvalidSoftError(f"Nonzero lead(ls)) for {code=}" - #f" {soft=} when special.") - - #if rize: # raw must be empty - #raise RawMaterialError(f"Nonzero raw size {rize=} when " - #f" special {code=} {soft=}.") - soft = soft[:ss] if len(soft) != ss: raise SoftMaterialError(f"Not enough chars in {soft=} " @@ -950,21 +934,11 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " f" or {code=} {fs=} when special soft.") - #rize = Matter._rawSize(code) - #if rize: - #raise InvalidSizeError(f"Nonzero raw size {rize=} when special" - #f" soft {code=}.") - - #if not soft or len(soft) < ss: - #raise SoftMaterialError(f"Not enough chars in {soft=} with " - #f"{ss=} for {code=}.") - soft = soft[:ss] if len(soft) != ss: raise SoftMaterialError(f"Not enough chars in {soft=} " f"with {ss=} for {code=}.") - if not Reb64.match(soft.encode("utf-8")): raise InvalidSoftError(f"Non Base64 chars in {soft=}.")