From f3ace16688e482154c92dd96ccdfaf5b074f058d Mon Sep 17 00:00:00 2001 From: Rodolfo Date: Fri, 4 Oct 2024 15:45:08 -0300 Subject: [PATCH 01/11] Missing sn in CloneCred after fix in PR #682 (#854) * Missing sn in CloneCred in PR 682 * correct tel sn * add rev and rev anchor --- src/keri/vdr/viring.py | 33 +++++++++++++++++++++++++++------ tests/vdr/test_verifying.py | 14 +++++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index c2fccfda..74782081 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -402,11 +402,19 @@ def cloneCreds(self, saids, db): atc = bytearray(signing.serialize(creder, prefixer, seqner, saider)) del atc[0:creder.size] - iss = bytearray(self.cloneTvtAt(creder.said)) + regk = creder.regi + status = self.tevers[regk].vcState(saider.qb64) + schemer = db.schema.get(creder.schema) + + iss = bytearray(self.cloneTvtAt(creder.said, sn=0)) iserder = serdering.SerderKERI(raw=iss) issatc = bytes(iss[iserder.size:]) - del iss[0:iserder.size] + if status.et in [coring.Ilks.rev, coring.Ilks.brv]: + rev = bytearray(self.cloneTvtAt(creder.said, sn=1)) + rserder = serdering.SerderKERI(raw=rev) + revatc = bytes(rev[rserder.size:]) + del rev[0:rserder.size] chainSaids = [] for k, p in (creder.edge.items() if creder.edge is not None else {}): @@ -419,15 +427,13 @@ def cloneCreds(self, saids, db): chainSaids.append(coring.Saider(qb64=p["n"])) chains = self.cloneCreds(chainSaids, db) - regk = creder.regi - status = self.tevers[regk].vcState(saider.qb64) - schemer = db.schema.get(creder.schema) - cred = dict( sad=creder.sad, atc=atc.decode("utf-8"), iss=iserder.sad, issatc=issatc.decode("utf-8"), + rev=rserder.sad if status.et in [coring.Ilks.rev, coring.Ilks.brv] else None, + revatc=revatc.decode("utf-8") if status.et in [coring.Ilks.rev, coring.Ilks.brv] else None, pre=creder.issuer, schema=schemer.sed, chains=chains, @@ -452,6 +458,21 @@ def cloneCreds(self, saids, db): ancatc = bytes(anc[aserder.size:]) cred['anc'] = aserder.sad cred['ancatc'] = ancatc.decode("utf-8"), + + if status.et in [coring.Ilks.rev, coring.Ilks.brv]: + ctr = core.Counter(qb64b=rev, strip=True, gvrsn=kering.Vrsn_1_0) + if ctr.code == counting.CtrDex_1_0.AttachmentGroup: + ctr = core.Counter(qb64b=rev, strip=True, gvrsn=kering.Vrsn_1_0) + + if ctr.code == counting.CtrDex_1_0.SealSourceCouples: + coring.Seqner(qb64b=rev, strip=True) + saider = coring.Saider(qb64b=rev) + + anc = db.cloneEvtMsg(pre=creder.issuer, fn=0, dig=saider.qb64b) + aserder = serdering.SerderKERI(raw=anc) + ancatc = bytes(anc[aserder.size:]) + cred['revanc'] = aserder.sad + cred['revancatc'] = ancatc.decode("utf-8"), creds.append(cred) diff --git a/tests/vdr/test_verifying.py b/tests/vdr/test_verifying.py index 6cd16154..48b24cf9 100644 --- a/tests/vdr/test_verifying.py +++ b/tests/vdr/test_verifying.py @@ -112,8 +112,9 @@ def test_verifier(seeder): # also try it via the cloneCreds function creds = regery.reger.cloneCreds(saids=saider, db=hab.db) - for idx, cred in enumerate(creds): + for cred in creds: assert dcre.sad == cred["sad"] + assert cred['rev'] is None with pytest.raises(kering.MissingEntryError): regery.reger.cloneCred(said="nonexistantsaid") @@ -658,5 +659,16 @@ def test_verifier_chained_credential(seeder): with pytest.raises(kering.RevokedChainError): vicverfer.processCredential(vLeiCreder, prefixer=ian.kever.prefixer, seqner=seqner, saider=coring.Saider(qb64=ian.kever.serder.said)) + + creds = ronreg.reger.cloneCreds(saids=[coring.Saider(qb64=creder.said)], db=ronHby.db) + for cred in creds: + assert cred['status']['et'] == 'rev' + assert cred['rev'] is not None + assert cred['rev']['i'] == creder.said + assert cred['revatc'] is not None + assert cred['revanc'] is not None + assert cred['revanc']['s'] == '3' + assert cred['revanc']['a'][0]['s'] == '1' + assert cred['revancatc'] is not None """End Test""" From cd328128334bde23034486b18c5efe85c0a56d24 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 4 Oct 2024 12:48:19 -0600 Subject: [PATCH 02/11] chore: remove unneeded migration (#869) This migration is specific to Provenant and was a one-off migration and can be deleted. --- src/keri/app/cli/commands/migrate.py | 162 --------------------------- 1 file changed, 162 deletions(-) delete mode 100644 src/keri/app/cli/commands/migrate.py diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py deleted file mode 100644 index 87858779..00000000 --- a/src/keri/app/cli/commands/migrate.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -KERI -keri.kli.commands module - -""" -import argparse - -from hio import help -from hio.base import doing - -from keri import kering -from keri.app.cli.common import existing -from keri.core import coring, serdering -from keri.db import koming, subing, dbing -from keri.db.basing import KeyStateRecord, StateEERecord -from keri.kering import ConfigurationError, Version -from keri.vdr import viring - -logger = help.ogler.getLogger() - -parser = argparse.ArgumentParser(description='View status of a local AID') -parser.set_defaults(handler=lambda args: handler(args), - transferable=True) -parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) -parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', - required=False, default="") -parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', - dest="bran", default=None) # passcode => bran -parser.add_argument('--force', action="store_true", required=False, - help='True means perform migration without prompting the user') - - -def handler(args): - if not args.force: - print() - print("This command will migrate your datastore to the next version of KERIpy and is not reversible.") - print("After this command, you will not be able to access your data store with this version.") - print() - yn = input("Are you sure you want to continue? [y|N]: ") - - if yn not in ("y", "Y"): - print("...exiting") - return [] - - kwa = dict(args=args) - return [doing.doify(migrate, **kwa)] - - -def migrate(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - base = args.base - bran = args.bran - - try: - with dbing.openLMDB(name=name, base=base, bran=bran, temp=False) as db: - print(db.path) - states = koming.Komer(db=db, - schema=dict, - subkey='stts.') - nstates = koming.Komer(db=db, - schema=KeyStateRecord, - subkey='stts.') - - for keys, sad in states.getItemIter(): - ksr = KeyStateRecord( - vn=Version, # version number as list [major, minor] - i=sad['i'], # qb64 prefix - s=sad['s'], # lowercase hex string no leading zeros - p=sad['p'], - d=sad['d'], - f=sad['f'], # lowercase hex string no leading zeros - dt=sad['dt'], - et=sad['et'], - kt=sad['kt'], - k=sad['k'], - nt=sad['nt'], - n=sad['n'], - bt=sad['bt'], - b=sad['b'], - c=sad['c'], - ee=StateEERecord._fromdict(sad['ee']), # latest est event dict - di=sad['di'] if sad['di'] else None - ) - - nstates.pin(keys=keys, val=ksr) - - with existing.existingHby(name=name, base=base, bran=bran) as hby: - rgy = viring.Reger(name=name, base=base, db=hby.db, temp=False, - reopen=True) - - rstates = koming.Komer(db=rgy, - schema=dict, - subkey='stts.') - - for _, sad in rstates.getItemIter(): - rsr = viring.RegStateRecord( - vn=list(Version), # version number as list [major, minor] - i=sad['i'], # qb64 registry SAID - s=sad['s'], # lowercase hex string no leading zeros - d=sad['d'], - ii=sad['ii'], - dt=sad['dt'], - et=sad['et'], - bt=sad['bt'], # hex string no leading zeros lowercase - b=sad['b'], # list of qb64 may be empty - c=sad['c'], - ) - # ksr = stateFromKever(kever) - rgy.states.pin(sad['i'], val=rsr) - - for (said,), _ in rgy.creds.getItemIter(): - snkey = dbing.snKey(said, 0) - dig = rgy.getTel(key=snkey) - - prefixer = coring.Prefixer(qb64=said) - seqner = coring.Seqner(sn=0) - saider = coring.Saider(qb64b=bytes(dig)) - rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) - - migrateKeys(hby.db) - - # clear escrows - print("clearing escrows") - hby.db.gpwe.trim() - hby.db.gdee.trim() - hby.db.dpwe.trim() - hby.db.gpse.trim() - hby.db.epse.trim() - hby.db.dune.trim() - - except ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - - -def migrateKeys(db): - # public keys mapped to the AID and event seq no they appeared in - pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in - digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - - for pre, fn, dig in db.getFelItemAllPreIter(): - dgkey = dbing.dgKey(pre, dig) # get message - if not (raw := db.getEvt(key=dgkey)): - raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) - serder = serdering.SerderKERI(raw=bytes(raw)) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - verfers = serder.verfers or [] - for verfer in verfers: - pubs.add(keys=(verfer.qb64,), val=val) - ndigers = serder.ndigers or [] - for diger in ndigers: - digs.add(keys=(diger.qb64,), val=val) From e7bcfeed1403d257da76596484c0bd0a307b405e Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Fri, 4 Oct 2024 12:48:48 -0600 Subject: [PATCH 03/11] docs: show libsodium install instructions (#870) --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index a6569235..68f60f94 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,15 @@ python 3.12.1+ libsodium 1.0.18+ +##### Libsodium installation + +After installing with Homebrew you need to link the library to the default location that the `ctypes.find_library()` looks in, `/usr/local/lib`, as shown below: + +```shell +sudo ln -s /opt/homebrew/lib /usr/local/lib +``` + + #### python packages lmdb 0.98+ pysodium 0.7.5+ From d41831fa6e8134ae8dcbda55ddd39768bb3c566b Mon Sep 17 00:00:00 2001 From: Kevin Griffin Date: Fri, 4 Oct 2024 15:21:08 -0400 Subject: [PATCH 04/11] adds escrow clearing (#864) * adds escrow clearing, moves original escrow command to 'escrow list' Signed-off-by: Kevin Griffin * fix: add qnfs database to clear escrow --------- Signed-off-by: Kevin Griffin Co-authored-by: Kent Bull --- src/keri/app/cli/commands/clean.py | 8 +- src/keri/app/cli/commands/escrow/__init__.py | 3 + src/keri/app/cli/commands/escrow/clear.py | 52 ++++++++ .../commands/{escrow.py => escrow/list.py} | 2 +- src/keri/db/basing.py | 26 ++++ tests/app/cli/test_kli_commands.py | 2 +- tests/db/test_basing.py | 113 +++++++++++++++--- 7 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 src/keri/app/cli/commands/escrow/__init__.py create mode 100644 src/keri/app/cli/commands/escrow/clear.py rename src/keri/app/cli/commands/{escrow.py => escrow/list.py} (99%) diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py index 63c3b490..e3e365b3 100644 --- a/src/keri/app/cli/commands/clean.py +++ b/src/keri/app/cli/commands/clean.py @@ -47,9 +47,13 @@ def __init__(self, args): super(CleanDoer, self).__init__() def recur(self, tyme): - hby = existing.setupHby(name=self.args.name, base=self.args.base, bran=self.args.bran, temp=self.args.temp) + + print("Clearing escrows...") + hby.db.clearEscrows() + print("Finished") + print("Migrating...") hby.db.migrate() print("Finished") @@ -59,6 +63,6 @@ def recur(self, tyme): print("Database open, performing clean...") hby.db.clean() - print("Finished.") + print("Finished") return True diff --git a/src/keri/app/cli/commands/escrow/__init__.py b/src/keri/app/cli/commands/escrow/__init__.py new file mode 100644 index 00000000..defc725e --- /dev/null +++ b/src/keri/app/cli/commands/escrow/__init__.py @@ -0,0 +1,3 @@ +import argparse + +parser = argparse.ArgumentParser(description="A collection of escrow operations") \ No newline at end of file diff --git a/src/keri/app/cli/commands/escrow/clear.py b/src/keri/app/cli/commands/escrow/clear.py new file mode 100644 index 00000000..ae5e03d7 --- /dev/null +++ b/src/keri/app/cli/commands/escrow/clear.py @@ -0,0 +1,52 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.escrow module + +""" +import argparse + +from hio import help +from hio.base import doing +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Clear escrows') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--force', action="store_true", required=False, + help='True means perform clear without prompting the user') + + +def handler(args): + if not args.force: + print() + print("This command will clear all escrows and is not reversible.") + print() + yn = input("Are you sure you want to continue? [y|N]: ") + + if yn not in ("y", "Y"): + print("...exiting") + return [] + + kwa = dict(args=args) + return [doing.doify(clear, **kwa)] + + +def clear(tymth, tock=0.0, **opts): + """ Command line clear handler + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + hby.db.clearEscrows() diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow/list.py similarity index 99% rename from src/keri/app/cli/commands/escrow.py rename to src/keri/app/cli/commands/escrow/list.py index 277ad4b7..e1aae8ab 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow/list.py @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- """ KERI -keri.kli.commands module +keri.kli.commands.escrow module """ import argparse diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 14e9bfb5..32077072 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1367,6 +1367,32 @@ def migrate(self): self.version = keri.__version__ + def clearEscrows(self): + """ + Clear all escrows + """ + for (k, _) in self.getUreItemIter(): + self.delUres(key=k) + for (k, _) in self.getVreItemIter(): + self.delVres(key=k) + for (k, _) in self.getPseItemIter(): + self.delPses(key=k) + for (k, _) in self.getPweItemIter(): + self.delPwes(key=k) + for (k, _) in self.getUweItemIter(): + self.delUwes(key=k) + for (k, _) in self.getOoeItemIter(): + self.delOoes(key=k) + for (k, _) in self.getLdeItemIter(): + self.delLdes(key=k) + for (pre, said), edig in self.qnfs.getItemIter(): + self.qnfs.rem(keys=(pre, said)) + + + for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi, + self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]: + escrow.trim() + @property def current(self): """ Current property determines if we are at the current database migration state. diff --git a/tests/app/cli/test_kli_commands.py b/tests/app/cli/test_kli_commands.py index 94d96042..ec30511d 100644 --- a/tests/app/cli/test_kli_commands.py +++ b/tests/app/cli/test_kli_commands.py @@ -239,7 +239,7 @@ def test_standalone_kli_commands(helpers, capsys): '\t3. DEMwUl3u8mJ-cWxSnReA0rQesIgZ8SFoHp0U2WyiZjRt\n' '\n') - args = parser.parse_args(["escrow", "--name", "test"]) + args = parser.parse_args(["escrow", "list", "--name", "test"]) assert args.handler is not None doers = args.handler(args) directing.runController(doers=doers) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 659403c2..ad4a17ee 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -7,28 +7,23 @@ import os from dataclasses import dataclass, asdict -import lmdb import pytest -from hio.base import doing - -from keri.help.helping import datify, dictify +import lmdb +from hio.base import doing from keri import core +from keri.app import habbing from keri.core import coring, eventing, serdering - -from keri.core.coring import Kinds, versify - +from keri.core.coring import Kinds, versify, Seqner from keri.core.eventing import incept, rotate, interact, Kever - -from keri.app import habbing - +from keri.core.serdering import Serder from keri.db import basing from keri.db import dbing from keri.db import subing -from keri.db.basing import openDB, Baser, KeyStateRecord +from keri.db.basing import openDB, Baser, KeyStateRecord, OobiRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB - +from keri.help.helping import datify, dictify # this breaks when running as __main__ better to do a custom import call to # walk the directory tree and import explicity rather than depend on it # being a known package. Works with pytest because pytest contructs a path @@ -185,9 +180,6 @@ def test_baser(): assert actual.local == record.local assert db.esrs.get(key) == record - - - # test first seen event log .fels sub db preA = b'BAKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' @@ -1824,6 +1816,97 @@ def test_KERI_BASER_MAP_SIZE_handles_bad_values(caplog): os.environ.pop("KERI_BASER_MAP_SIZE") +def test_clear_escrows(): + with openDB() as db: + key = b'A' + vals = [b"z", b"m", b"x", b"a"] + + db.putUres(key, vals) + db.putVres(key, vals) + db.putPses(key, vals) + db.putPwes(key, vals) + db.putUwes(key, vals) + db.putOoes(key, vals) + db.putLdes(key, vals) + + pre = b'k' + snh = b'snh' + saidb = b'saidb' + db.qnfs.add(keys=(pre, saidb), val=b"z") + assert db.qnfs.cnt(keys=(pre, saidb)) == 1 + + db.misfits.add(keys=(pre, snh), val=saidb) + assert db.misfits.cnt(keys=(pre, snh)) == 1 + + db.delegables.add(snKey(pre, 0), saidb) + assert db.delegables.cnt(keys=snKey(pre, 0)) == 1 + + db.pdes.addOn(keys=pre, on=0, val=saidb) + assert db.pdes.cnt(keys=snKey(pre, 0)) == 1 + + udesKey = dgKey('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8"), + 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8")) + db.udes.put(keys=udesKey, val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), + coring.Saider(qb64b=b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E'))) + assert db.udes.get(keys=udesKey) is not None + + saider = coring.Saider(qb64b='EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') + db.rpes.put(keys=('route',), vals=[saider]) + assert db.rpes.cnt(keys=('route',)) == 1 + + db.epsd.put(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',), val=coring.Dater()) + assert db.epsd.get(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',)) is not None + + db.eoobi.pin(keys=('url',), val=OobiRecord()) + assert db.eoobi.cntAll() == 1 + + serder = Serder(raw=b'{"v":"KERI10JSON0000cb_","t":"ixn","d":"EG8WAmM29ZBdoXbnb87yiPxQw4Y7gcQjqZS74vBAKsRm","i":"DApYGFaqnrALTyejaJaGAVhNpSCtqyerPqWVK9ZBNZk0","s":"4","p":"EAskHI462CuIMS_gNkcl_QewzrRSKH2p9zHQIO132Z30","a":[]}') + db.dpub.put(keys=(pre, 'said'), val=serder) + assert db.dpub.get(keys=(pre, 'said')) is not None + + db.gpwe.add(keys=(pre,), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) + assert db.gpwe.cnt(keys=(pre,)) == 1 + + db.gdee.add(keys=(pre,), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) + assert db.gdee.cnt(keys=(pre,)) == 1 + + db.dpwe.pin(keys=(pre, 'said'), val=serder) + assert db.dpwe.get(keys=(pre, 'said')) is not None + + db.gpse.add(keys=('qb64',), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) + assert db.gpse.cnt(keys=('qb64',)) == 1 + + db.epse.put(keys=('dig',), val=serder) + assert db.epse.get(keys=('dig',)) is not None + + db.dune.pin(keys=(pre, 'said'), val=serder) + assert db.dune.get(keys=(pre, 'said')) is not None + + db.clearEscrows() + + assert db.getUres(key) == [] + assert db.getVres(key) == [] + assert db.getPses(key) == [] + assert db.getPwes(key) == [] + assert db.getUwes(key) == [] + assert db.getOoes(key) == [] + assert db.getLdes(key) == [] + assert db.qnfs.cnt(keys=(pre, saidb)) == 0 + assert db.misfits.cnt(keys=(pre, snh)) == 0 + assert db.delegables.cnt(keys=snKey(pre, 0)) == 0 + assert db.pdes.cnt(keys=snKey(pre, 0)) == 0 + assert db.udes.get(keys=udesKey) is None + assert db.rpes.cnt(keys=('route',)) == 0 + assert db.epsd.get(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',)) is None + assert db.eoobi.cntAll() == 0 + assert db.dpub.get(keys=(pre, 'said')) is None + assert db.gpwe.cnt(keys=(pre,)) == 0 + assert db.gdee.cnt(keys=(pre,)) == 0 + assert db.dpwe.get(keys=(pre, 'said')) is None + assert db.gpse.cnt(keys=('qb64',)) == 0 + assert db.epse.get(keys=('dig',)) is None + assert db.dune.get(keys=(pre, 'said')) is None + if __name__ == "__main__": test_baser() test_clean_baser() From c712ba6b878e4001d12ea15a8b6371c6bc6d8472 Mon Sep 17 00:00:00 2001 From: Philip Feairheller Date: Fri, 11 Oct 2024 12:37:32 -0700 Subject: [PATCH 05/11] Two command line changes: (#871) (#876) - OOBI resolve now waits until the AID is in Kevers before completing - Witness authenticate includes the CESR http header. Fix OOBI generation to favor HTTPS over HTTP and to honor the full URL of the service endpoint. Use the "Last" variant of the method for finding event seals in all locations. Signed-off-by: pfeairheller --- src/keri/app/cli/commands/did/generate.py | 16 +++++++------- src/keri/app/cli/commands/ipex/grant.py | 4 ++-- src/keri/app/cli/commands/oobi/generate.py | 22 +++++++++---------- src/keri/app/cli/commands/oobi/resolve.py | 5 +++++ .../app/cli/commands/witness/authenticate.py | 4 +++- src/keri/app/delegating.py | 2 +- src/keri/app/grouping.py | 4 ++-- src/keri/app/oobiing.py | 11 +++++----- src/keri/app/querying.py | 3 +-- src/keri/core/eventing.py | 18 +++++++-------- src/keri/db/basing.py | 1 - src/keri/end/ending.py | 2 +- 12 files changed, 47 insertions(+), 45 deletions(-) diff --git a/src/keri/app/cli/commands/did/generate.py b/src/keri/app/cli/commands/did/generate.py index 0200d40b..0aca604c 100644 --- a/src/keri/app/cli/commands/did/generate.py +++ b/src/keri/app/cli/commands/did/generate.py @@ -73,20 +73,20 @@ def generate(tymth, tock=0.0, **opts): sys.exit(-1) wit = random.choice(hab.kever.wits) - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise kering.ConfigurationError(f"unable to query witness {wit}, no http endpoint") - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - enc = urllib.parse.quote_plus(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + enc = urllib.parse.quote_plus(f"{url.rstrip("/")}/oobi/{hab.pre}/witness") print(f"did:keri:{hab.pre}?oobi={enc}") elif role in (kering.Roles.controller,): - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: print(f"{alias} identifier {hab.pre} does not have any controller endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - enc = urllib.parse.quote_plus(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + enc = urllib.parse.quote_plus(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") print(f"did:keri:{hab.pre}?oobi={enc}") diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index cc766664..5787089f 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0): iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) - serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'], - seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) + serder = self.hby.db.fetchLastSealingEventByEventSeal(creder.sad['i'], + seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc, diff --git a/src/keri/app/cli/commands/oobi/generate.py b/src/keri/app/cli/commands/oobi/generate.py index e35a5335..406d5f08 100644 --- a/src/keri/app/cli/commands/oobi/generate.py +++ b/src/keri/app/cli/commands/oobi/generate.py @@ -4,7 +4,6 @@ """ import argparse -from urllib.parse import urlparse import sys from hio import help @@ -66,21 +65,21 @@ def generate(tymth, tock=0.0, **opts): sys.exit(-1) for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise kering.ConfigurationError(f"unable to query witness {wit}, no http endpoint") - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/witness") elif role in (kering.Roles.controller,): - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: print(f"{alias} identifier {hab.pre} does not have any controller endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") elif role in (kering.Roles.mailbox,): for (_, _, eid), end in hab.db.ends.getItemIter(keys=(hab.pre, kering.Roles.mailbox, )): if not (end.allowed and end.enabled is not False): @@ -91,6 +90,5 @@ def generate(tymth, tock=0.0, **opts): if not urls: print(f"{alias} identifier {hab.pre} does not have any mailbox endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/mailbox/{eid}") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/mailbox/{eid}") diff --git a/src/keri/app/cli/commands/oobi/resolve.py b/src/keri/app/cli/commands/oobi/resolve.py index dc6e62dc..747c67c5 100644 --- a/src/keri/app/cli/commands/oobi/resolve.py +++ b/src/keri/app/cli/commands/oobi/resolve.py @@ -107,6 +107,11 @@ def waitDo(self, tymth, tock=0.0): yield 0.25 obr = self.obi.hby.db.roobi.get(keys=(self.oobi,)) + if self.force: + while obr.cid not in self.hby.kevers: + self.hby.kvy.processEscrows() + yield 0.25 + print(self.oobi, obr.state) self.remove([self.hbyDoer, *self.obi.doers, *self.authn.doers]) diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py index f0a2b419..53241536 100644 --- a/src/keri/app/cli/commands/witness/authenticate.py +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -16,6 +16,7 @@ from keri.app import httping, connecting from keri.app.agenting import httpClient from keri.app.cli.common import existing +from keri.app.httping import CESR_DESTINATION_HEADER from keri.core import coring logger = help.ogler.getLogger() @@ -107,7 +108,8 @@ def authDo(self, tymth, tock=0.0): fargs['delkel'] = delkel.decode("utf-8") headers = (Hict([ - ("Content-Type", "multipart/form-data") + ("Content-Type", "multipart/form-data"), + (CESR_DESTINATION_HEADER, self.witness) ])) client, clientDoer = httpClient(self.hab, self.witness) diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index bddb7dd4..1e09e659 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -138,7 +138,7 @@ def processUnanchoredEscrow(self): dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): + if dserder := self.hby.db.fetchLastSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): seqner = coring.Seqner(sn=dserder.sn) couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 096798f7..f87ef249 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -171,8 +171,8 @@ def processDelegateEscrow(self): self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre, - seal=anchor): + if serder := self.hby.db.fetchLastSealingEventByEventSeal(kever.delpre, + seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index cb8b41d3..7228b918 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -115,15 +115,15 @@ def on_get_alias(self, req, rep, alias=None): if role in (kering.Roles.witness,): # Fetch URL OOBIs for all witnesses oobis = [] for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: rep.status = falcon.HTTP_404 rep.text = f"unable to query witness {wit}, no http endpoint" return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness/{wit}") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + oobis.append(f"{url.rstrip("/")}/oobi/{hab.pre}/witness/{wit}") res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] @@ -134,8 +134,7 @@ def on_get_alias(self, req, rep, alias=None): rep.text = f"unable to query controller {hab.pre}, no http endpoint" return url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + oobis.append(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") res["oobis"] = oobis else: rep.status = falcon.HTTP_404 diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index b6a401ad..21ff0129 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -140,8 +140,7 @@ def recur(self, tyme, deeds=None): if self.pre not in self.hab.kevers: return False - kever = self.hab.kevers[self.pre] - if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor): + if self.hby.db.fetchLastSealingEventByEventSeal(self.pre, seal=self.anchor): self.remove([self.witq]) return True diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index b244af68..3e8ba393 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3020,22 +3020,22 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): dserder = serdering.SerderKERI(raw=bytes(raw)) return dserder - elif eager: #missing aes but try to find seal by walking delegator's KEL + elif eager: # missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict if original: # search all events in delegator's kel not just last - if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, - seal=seal)): + if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): # database broken this should never happen so do not validate # since original must have been validated so it must have # all its delegation chain. raise ValidationError(f"Missing delegation source seal for {serder.ked}") - else: # only search last events in delegator's kel - if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, - seal=seal)): + else: # only search last events in delegator's kel + if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): # superseding delegation may not have happened yet so escrow # ToDo XXXX need to cue up to get latest events in # delegator's kel. - #raise ValidationError(f"Missing delegation source seal for {serder.ked}") + # raise ValidationError(f"Missing delegation source seal for {serder.ked}") return None # Only repair .aess when found delegation is for delegated event that @@ -3788,7 +3788,7 @@ def processEvent(self, serder, sigers, *, wigers=None, else: # not inception so can't verify sigs etc, add to out-of-order escrow self.escrowOOEvent(serder=serder, sigers=sigers, - seqner=delseqner, saider=delsaider, wigers=wigers) + seqner=delseqner, saider=delsaider, wigers=wigers, local=local) raise OutOfOrderError("Out-of-order event={}.".format(ked)) else: # already accepted inception event for pre so already first seen @@ -3829,7 +3829,7 @@ def processEvent(self, serder, sigers, *, wigers=None, if sn > sno: # sn later than sno so out of order escrow # escrow out-of-order event self.escrowOOEvent(serder=serder, sigers=sigers, - seqner=delseqner, saider=delsaider, wigers=wigers) + seqner=delseqner, saider=delsaider, wigers=wigers, local=local) raise OutOfOrderError("Out-of-order event={}.".format(ked)) elif ((sn == sno) or # inorder event (ixn, rot, drt) or diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 32077072..35fc6362 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1726,7 +1726,6 @@ def fetchAllSealingEventByEventSeal(self, pre, seal, sn=0): # use alias here until can change everywhere for backwards compatibility findAnchoringSealEvent = fetchAllSealingEventByEventSeal # alias - def fetchLastSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the last event at any sn but that contains a diff --git a/src/keri/end/ending.py b/src/keri/end/ending.py index 6ff62ef6..72729316 100644 --- a/src/keri/end/ending.py +++ b/src/keri/end/ending.py @@ -275,7 +275,7 @@ def siginput(name, method, path, headers, fields, hab=None, signers=None, expire expires (str): iso8601 formated date string indicating exiration of header signature signers (list): Optional signer objects used to sign the values hab (Hab): Optional Hab used to sign the values. One of signers or Hab is required - fields (str): Fields in request to sign. Includes special fields as well as Header fields + fields (list): Fields in request to sign. Includes special fields as well as Header fields headers (dict): HTTP request headers path (str): HTTP request path method (str): HTTP request method (POST, GET, PUT, etc) From 47282727ff30281e826c76f323767aab2f3c6115 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Thu, 24 Oct 2024 08:56:24 -0600 Subject: [PATCH 06/11] docs: fix Sphinx build (#880) * docs: fix Sphinx build * fix: add ReadTheDocs theme reqs --- docs/Makefile | 14 +++++++++++++- docs/_static/.gitkeep | 0 docs/conf.py | 3 +-- docs/index.rst | 7 ++++--- docs/keri_app.rst | 16 ++-------------- docs/keri_core.rst | 30 ++++++++++++++++++++++++++++++ docs/keri_db.rst | 6 ++++++ docs/keri_demo.rst | 8 ++++++++ docs/keri_help.rst | 2 +- docs/keri_peer.rst | 2 +- docs/keri_vc.rst | 4 ++-- docs/naming.md | 4 ++-- docs/requirements.txt | 13 +++++++++++-- 13 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 docs/_static/.gitkeep create mode 100644 docs/keri_demo.rst diff --git a/docs/Makefile b/docs/Makefile index d4bb2cbb..882f3418 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -12,9 +12,21 @@ BUILDDIR = _build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help Makefile +.PHONY: help Makefile clean html # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(SOURCEDIR) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + @echo + +open: + open _build/html/index.html \ No newline at end of file diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/conf.py b/docs/conf.py index 5763f28a..33703327 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ sphinx_rtd_theme = None project = 'keri' -copyright = '2022, Dr. Samuel Smith and contributors' +copyright = '2022 - 2024, Dr. Samuel Smith and contributors' author = 'Dr. Samuel Smith' version = release = keri.__version__ @@ -62,7 +62,6 @@ # if sphinx_rtd_theme: html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] else: html_theme = "default" diff --git a/docs/index.rst b/docs/index.rst index 32395b9b..b759a821 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,6 @@ -================================================== -Python Implementation of the KERI Core Libraries -================================================== +========================================================== +Python Reference Implementation of the KERI Core Libraries +========================================================== .. image:: https://img.shields.io/pypi/v/keri.svg :target: https://pypi.org/project/keri/ @@ -32,6 +32,7 @@ API Reference .. toctree:: :maxdepth: 1 + naming keri_app keri_core keri_db diff --git a/docs/keri_app.rst b/docs/keri_app.rst index e609031d..027ac267 100644 --- a/docs/keri_app.rst +++ b/docs/keri_app.rst @@ -13,12 +13,6 @@ keri.app.apping .. automodule:: keri.app.apping :members: -keri.app.booting ----------------- - -.. automodule:: keri.app.booting - :members: - keri.app.challenging -------------------- @@ -32,7 +26,7 @@ keri.app.configing :members: keri.app.connecting ------------------- +------------------- .. automodule:: keri.app.connecting :members: @@ -85,12 +79,6 @@ keri.app.keeping .. automodule:: keri.app.keeping :members: -keri.app.kiwiing ----------------- - -.. automodule:: keri.app.kiwiing - :members: - keri.app.notifying ------------------ @@ -100,7 +88,7 @@ keri.app.notifying keri.app.oobing --------------- -.. automodule:: keri.app.oobing +.. automodule:: keri.app.oobiing :members: keri.app.signaling diff --git a/docs/keri_core.rst b/docs/keri_core.rst index dd9313b8..310ddfa2 100644 --- a/docs/keri_core.rst +++ b/docs/keri_core.rst @@ -7,12 +7,24 @@ keri.core.coring .. automodule:: keri.core.coring :members: +keri.core.counting +------------------ + +.. automodule:: keri.core.counting + :members: + keri.core.eventing ------------------ .. automodule:: keri.core.eventing :members: +ker.core.indexing +----------------- + +.. automodule:: keri.core.indexing + :members: + keri.core.parsing ----------------- @@ -31,3 +43,21 @@ keri.core.scheming .. automodule:: keri.core.scheming :members: +keri.core.signing +----------------- + +.. automodule:: keri.core.signing + :members: + +keri.core.streaming +------------------- + +.. automodule:: keri.core.streaming + :members: + +keri.core.structing +------------------- + +.. automodule:: keri.core.structing + :members: + diff --git a/docs/keri_db.rst b/docs/keri_db.rst index e2354b17..cb970433 100644 --- a/docs/keri_db.rst +++ b/docs/keri_db.rst @@ -25,6 +25,12 @@ keri.db.koming .. automodule:: keri.db.koming :members: +keri.db.migrations +------------------ + +.. automodule:: keri.db.migrations + :members: + keri.db.subing -------------- diff --git a/docs/keri_demo.rst b/docs/keri_demo.rst new file mode 100644 index 00000000..00825bae --- /dev/null +++ b/docs/keri_demo.rst @@ -0,0 +1,8 @@ +KERI Demo API +============= + +keri.demo.demoing +----------------- + +.. automodule:: keri.demo.demoing + :members: \ No newline at end of file diff --git a/docs/keri_help.rst b/docs/keri_help.rst index 81e41d82..c8965d2d 100644 --- a/docs/keri_help.rst +++ b/docs/keri_help.rst @@ -1,5 +1,5 @@ KERI Help API -============ +============= keri.help.helping ----------------- diff --git a/docs/keri_peer.rst b/docs/keri_peer.rst index 8947447b..506a447f 100644 --- a/docs/keri_peer.rst +++ b/docs/keri_peer.rst @@ -1,5 +1,5 @@ KERI Peer API -============ +============= keri.peer.exchanging -------------------- diff --git a/docs/keri_vc.rst b/docs/keri_vc.rst index 5166211c..c99dd258 100644 --- a/docs/keri_vc.rst +++ b/docs/keri_vc.rst @@ -1,8 +1,8 @@ KERI Verificable Credential API -============= +=============================== keri.vc.protocoling ----------------- +------------------- .. automodule:: keri.vc.protocoling :members: diff --git a/docs/naming.md b/docs/naming.md index d2580995..e5eba714 100644 --- a/docs/naming.md +++ b/docs/naming.md @@ -1,5 +1,5 @@ - -# Python Style Guide for keripy +# Naming and Style guide +## Python Style Guide for KERIpy The Python PEPs on style have many options or allowed variants. The purpose of this document is to select a single preferred style diff --git a/docs/requirements.txt b/docs/requirements.txt index 9ac0edb6..89d1c0d0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,11 @@ -myst-parser >= 0.16.1 -Sphinx >= 4.3.2 \ No newline at end of file +myst-parser >= 4.0.0 +Sphinx >= 8.1.3 + +sphinx-rtd-theme==3.0.1 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jquery==4.1 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 From 676d519766cc671b1ddf8f8b3642cafd3ef3a33e Mon Sep 17 00:00:00 2001 From: Charles Lanahan Date: Mon, 18 Nov 2024 10:02:38 -0500 Subject: [PATCH 07/11] Removed --alias from verify command. (#867) * Adjusted log message to clarify which SAID errored as SADs may contain multiple SAIDs * Adjusted workflow to get right image to run * Adjusted help descriptions to more accurately describe what that command does in keripy * Restored these files to what they are in origin/main * Removed --prefix from verify command. Prefixes are 1-1 with aliases which are already required for this command and thus redundant. Removed for ease of use. * Reverted to remove alias rather than prefix from verify command --- scripts/demo/basic/demo-script.sh | 8 ++++---- src/keri/app/cli/commands/verify.py | 6 ++---- tests/app/cli/test_kli_commands.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/demo/basic/demo-script.sh b/scripts/demo/basic/demo-script.sh index 1cdbd619..b5bb43ef 100755 --- a/scripts/demo/basic/demo-script.sh +++ b/scripts/demo/basic/demo-script.sh @@ -46,16 +46,16 @@ isSuccess kli sign --name test --base "${KERI_TEMP_DIR}" --alias trans --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json isSuccess -kli verify --name test --base "${KERI_TEMP_DIR}" --alias trans --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature AABuQervAG8QLyvho99362U8TScec_4zYDVF1pUMWfWfKbl5thR6QakpvSvVMdcPcGYxi0McgNtW-Z3EhpC01I4A +kli verify --name test --base "${KERI_TEMP_DIR}" --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature AABuQervAG8QLyvho99362U8TScec_4zYDVF1pUMWfWfKbl5thR6QakpvSvVMdcPcGYxi0McgNtW-Z3EhpC01I4A isSuccess -kli verify --name test --base "${KERI_TEMP_DIR}" --alias trans --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature ABBJcS2ZbcHEdEEnaJE1CFUxdsKqkoU5TS34CImGh3s0cs-k3cNcy2PJxQ8WjIvAot1-cZ71o1E-WkqZ-tCTNZsJ +kli verify --name test --base "${KERI_TEMP_DIR}" --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature ABBJcS2ZbcHEdEEnaJE1CFUxdsKqkoU5TS34CImGh3s0cs-k3cNcy2PJxQ8WjIvAot1-cZ71o1E-WkqZ-tCTNZsJ isSuccess -kli verify --name test --base "${KERI_TEMP_DIR}" --alias trans --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature ACBieMVQBXvIxeqQuHc4Db_-GUoFE9e37TW8t6DomwXdMcSxBiHJpDp3EJH1Dcz9lHKbFuCRqoNo4wywZGtUpOcB +kli verify --name test --base "${KERI_TEMP_DIR}" --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature ACBieMVQBXvIxeqQuHc4Db_-GUoFE9e37TW8t6DomwXdMcSxBiHJpDp3EJH1Dcz9lHKbFuCRqoNo4wywZGtUpOcB isSuccess -kli verify --name test --base "${KERI_TEMP_DIR}" --alias trans --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature ACSHdal6kHAAjbW_frH83sDDCoBHw_nNKFysW5Dj8PSsnwVPePCNw-kFmF6Z8H87q7D3abw_5u2i4jmzdnWFsRDz +kli verify --name test --base "${KERI_TEMP_DIR}" --prefix EIryzWYlZ9bQr7EhMAoBXk4r2h-OgaEqERid7-AHNp6o --text @${KERI_DEMO_SCRIPT_DIR}/data/anchor.json --signature ACSHdal6kHAAjbW_frH83sDDCoBHw_nNKFysW5Dj8PSsnwVPePCNw-kFmF6Z8H87q7D3abw_5u2i4jmzdnWFsRDz ret=$? if [ $ret -eq 0 ]; then echo "Testing invalid signature should fail $ret" diff --git a/src/keri/app/cli/commands/verify.py b/src/keri/app/cli/commands/verify.py index 17033bb2..2df449e8 100644 --- a/src/keri/app/cli/commands/verify.py +++ b/src/keri/app/cli/commands/verify.py @@ -16,7 +16,6 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran @@ -45,16 +44,15 @@ def verify(tymth, tock=0.0, **opts): args = opts["args"] name = args.name - alias = args.alias base = args.base bran = args.bran sigers = [indexing.Siger(qb64=sig) for sig in args.signature] try: - with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (_, hab): + with existing.existingHby(name=name, base=base, bran=bran) as hby: - kever = hab.kevers[args.prefix] + kever = hby.kevers[args.prefix] txt = args.text if txt.startswith("@"): diff --git a/tests/app/cli/test_kli_commands.py b/tests/app/cli/test_kli_commands.py index ec30511d..4db45e43 100644 --- a/tests/app/cli/test_kli_commands.py +++ b/tests/app/cli/test_kli_commands.py @@ -204,7 +204,7 @@ def test_standalone_kli_commands(helpers, capsys): '3. ' 'ACCLl9pVv7OM4Y261GZkpPWQu__1mw8ffzcFY1lJ62CGjiEh3mvESu_N7a01YOCKqicqEe5TOXSf0j_8qBxPKxwO\n') - args = parser.parse_args(["verify", "--name", "test", "--alias", "trans", + args = parser.parse_args(["verify", "--name", "test", "--prefix", 'EF0bnfg4smFm9Q_OKlKUYRRQctGhTBWUU3rXf7zuA9GU', "--text", From bce08087c2d988c31a0f2f2b3c2abaffaeb1ac3a Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Mon, 18 Nov 2024 22:04:54 +0700 Subject: [PATCH 08/11] feat: migrates from 0.6.7 to 1.2.0 (#878) This includes migrations 0.6.8, 1.0.0, and changes the 1.1.0 rekey_habs to 1.2.0 for clarity --- src/keri/app/cli/commands/migrate/list.py | 6 +- src/keri/app/cli/commands/migrate/run.py | 16 +- src/keri/db/basing.py | 44 ++++-- .../add_key_and_reg_state_schemas.py | 147 ++++++++++++++++++ src/keri/db/migrations/hab_data_rename.py | 109 +++++++++++++ src/keri/db/migrations/rekey_habs.py | 32 +++- 6 files changed, 321 insertions(+), 33 deletions(-) create mode 100644 src/keri/db/migrations/add_key_and_reg_state_schemas.py create mode 100644 src/keri/db/migrations/hab_data_rename.py diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index 104fa8ba..f3f8c13c 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- """ -keri.kli.commands module +keri.kli.commands.migrate.list module """ import argparse @@ -16,7 +16,7 @@ def handler(args): """ - Launch KERI database initialization + List local LMDB database migrations and their completion status Args: args(Namespace): arguments object from command line @@ -25,7 +25,7 @@ def handler(args): return [lister] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Lists the local LMDB migrations and their completion status') parser.set_defaults(handler=handler, transferable=True) diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index 9a9e4c81..e87ca15f 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -1,16 +1,14 @@ # -*- encoding: utf-8 -*- """ -keri.kli.commands module +keri.kli.commands.migrate.run module """ import argparse -import keri from hio import help from hio.base import doing from keri import kering -from keri.app.cli.common import existing from keri.db import basing logger = help.ogler.getLogger() @@ -18,16 +16,16 @@ def handler(args): """ - Launch KERI database initialization + Launch KERI database migrator Args: args(Namespace): arguments object from command line """ - clean = MigrateDoer(args) - return [clean] + migrator = MigrateDoer(args) + return [migrator] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Migrates a database and keystore') parser.set_defaults(handler=handler, transferable=True) @@ -60,8 +58,8 @@ def recur(self, tyme): except kering.DatabaseError: pass - print("Migrating...") + print(f"Migrating {self.args.name}...") db.migrate() - print("Finished") + print(f"Finished migrating {self.args.name}") return True diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 35fc6362..25e0bc6c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -49,7 +49,9 @@ MIGRATIONS = [ - ("1.1.0", ["rekey_habs"]) + ("0.6.8", ["hab_data_rename"]), + ("1.0.0", ["add_key_and_reg_state_schemas"]), + ("1.2.0", ["rekey_habs"]) ] @@ -1312,7 +1314,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise if not self.current: - raise kering.DatabaseError("Database migrations must be run.") + raise kering.DatabaseError(f"Database migrations must be run. DB version {self.version}; current {keri.__version__}") removes = [] for keys, data in self.habs.getItemIter(): @@ -1346,10 +1348,18 @@ def migrate(self): """ for (version, migrations) in MIGRATIONS: - # Check to see if this is for an older version + # Only run migration if current source code version is at or below the migration version + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(version, str(ver_no_prerelease)) > 0: + print( + f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") + continue + # Skip migrations already run - where version less than (-1) or equal to (0) database version if self.version is not None and semver.compare(version, self.version) != 1: continue + print(f"Migrating database v{self.version} --> v{version}") for migration in migrations: modName = f"keri.db.migrations.{migration}" if self.migs.get(keys=(migration,)) is not None: @@ -1360,11 +1370,14 @@ def migrate(self): print(f"running migration {modName}") mod.migrate(self) except Exception as e: - print(f"\nAbandoning migration {migration} with error: {e}") + print(f"\nAbandoning migration {migration} at version {version} with error: {e}") return self.migs.pin(keys=(migration,), val=coring.Dater()) + # update database version after successful migration + self.version = version + self.version = keri.__version__ def clearEscrows(self): @@ -1407,14 +1420,16 @@ def current(self): if self.version == keri.__version__: return True - # If database version is ahead of library version, throw exception - if self.version is not None and semver.compare(self.version, keri.__version__) == 1: + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(self.version, str(ver_no_prerelease)) == 1: raise kering.ConfigurationError( f"Database version={self.version} is ahead of library version={keri.__version__}") last = MIGRATIONS[-1] - # If we aren't at latest version, but there are no outstanding migrations, reset version to latest - if self.migs.get(keys=(last[1][0],)) is not None: + # If we aren't at latest version, but there are no outstanding migrations, + # reset version to latest (rightmost (-1) migration is latest) + if self.migs.get(keys=(last[1][-1],)) is not None: return True # We have migrations to run @@ -1433,12 +1448,15 @@ def complete(self, name=None): migrations = [] if not name: for version, migs in MIGRATIONS: - for mig in migs: - dater = self.migs.get(keys=(mig,)) - migrations.append((mig, dater)) + # Print entries only for migrations that have been run + if self.version is not None and semver.compare(version, self.version) <= 0: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) else: - if name not in MIGRATIONS or not self.migs.get(keys=(name,)): - raise ValueError(f"No migration named {name}") + for version, migs in MIGRATIONS: # check all migrations for each version + if name not in migs or not self.migs.get(keys=(name,)): + raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) return migrations diff --git a/src/keri/db/migrations/add_key_and_reg_state_schemas.py b/src/keri/db/migrations/add_key_and_reg_state_schemas.py new file mode 100644 index 00000000..3a632f46 --- /dev/null +++ b/src/keri/db/migrations/add_key_and_reg_state_schemas.py @@ -0,0 +1,147 @@ +from keri import help +from keri.core import coring, serdering +from keri.db import koming, subing, dbing +from keri.db.basing import StateEERecord, KeyStateRecord +from keri.db.dbing import dgKey, splitKey +from keri.kering import ConfigurationError, Version +from keri.vdr import viring + +logger = help.ogler.getLogger() + +def _check_if_needed(db): + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + first = next(states.getItemIter(), None) + if first is None: + return False + keys, sad = first + if 'vn' in sad: + return False + return True + +def migrate(db): + """Adds schema for KeyStateRecord, RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., + and hby.db.digs. to be up to date as of 2022-??-?? + This migration performs the following: + - hby.db -> "stts." schema from dict -> KeyStateRecord + - rgy -> "stts." schema from dict -> RegStateRecord + - rgy -> "cancs." reset to (ACDC SAID, SN 0, TEL evt 0 digest) + - hby.db -> "pubs." and + hby.db -> "digs." + that don't exist are populated with verification keys and event digests for the first seen events and + Keys: + "pubs." Verfer of each Verfer for each FEL event + "digs." Diger of next Diger (ndiger) of each FEL event + Value: (prefix, sn) of each event + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + try: + logger.debug(f"Migrating keystate and regstate dict to schema for {db.path}") + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + nstates = koming.Komer(db=db, + schema=KeyStateRecord, + subkey='stts.') + + for keys, sad in states.getItemIter(): + ksr = KeyStateRecord( + vn=Version, # version number as list [major, minor] + i=sad['i'], # qb64 prefix + s=sad['s'], # lowercase hex string no leading zeros + p=sad['p'], + d=sad['d'], + f=sad['f'], # lowercase hex string no leading zeros + dt=sad['dt'], + et=sad['et'], + kt=sad['kt'], + k=sad['k'], + nt=sad['nt'], + n=sad['n'], + bt=sad['bt'], + b=sad['b'], + c=sad['c'], + ee=StateEERecord._fromdict(sad['ee']), # latest est event dict + di=sad['di'] if sad['di'] else None + ) + + nstates.pin(keys=keys, val=ksr) + + rgy = viring.Reger(name=db.name, base=db.base, db=db, temp=db.temp, reopen=True) + + rstates = koming.Komer(db=rgy, + schema=dict, + subkey='stts.') + + for _, sad in rstates.getItemIter(): + rsr = viring.RegStateRecord( + vn=list(Version), # version number as list [major, minor] + i=sad['i'], # qb64 registry SAID + s=sad['s'], # lowercase hex string no leading zeros + d=sad['d'], + ii=sad['ii'], + dt=sad['dt'], + et=sad['et'], + bt=sad['bt'], # hex string no leading zeros lowercase + b=sad['b'], # list of qb64 may be empty + c=sad['c'], + ) + # ksr = stateFromKever(kever) + rgy.states.pin(sad['i'], val=rsr) + + for (said,), _ in rgy.saved.getItemIter(): + snkey = dbing.snKey(said, 0) + dig = rgy.getTel(key=snkey) + + prefixer = coring.Prefixer(qb64=said) + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64b=bytes(dig)) + rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) + + migrateKeys(db) + + # clear escrows + logger.info("clearing escrows") + db.gpwe.trim() + db.gdee.trim() + db.dpwe.trim() + db.gpse.trim() + db.epse.trim() + db.dune.trim() + db.qnfs.trim() + + except ConfigurationError: + logger.error(f"identifier prefix for {db.name} does not exist, incept must be run first", ) + return -1 + + +def migrateKeys(db): + # public keys mapped to the AID and event seq no they appeared in + pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", + klas=(coring.Prefixer, coring.Seqner)) + + # next key digests mapped to the AID and event seq no they appeared in + digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", + klas=(coring.Prefixer, coring.Seqner)) + + for pre, fn, dig in db.getFelItemAllPreIter(): + dgkey = dbing.dgKey(pre, dig) # get message + if not (raw := db.getEvt(key=dgkey)): + logger.info(f"Migrate keys: missing event for dig={dig}, skipped.") + continue + serder = serdering.SerderKERI(raw=bytes(raw)) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) + verfers = serder.verfers or [] + for verfer in verfers: + pubs.add(keys=(verfer.qb64,), val=val) + ndigers = serder.ndigers or [] + for diger in ndigers: + digs.add(keys=(diger.qb64,), val=val) \ No newline at end of file diff --git a/src/keri/db/migrations/hab_data_rename.py b/src/keri/db/migrations/hab_data_rename.py new file mode 100644 index 00000000..26ab8a6c --- /dev/null +++ b/src/keri/db/migrations/hab_data_rename.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass, field, asdict +from typing import Optional + +from keri.db import koming, basing +from keri.db.basing import HabitatRecord, Baser +from keri.vdr.viring import Reger + + +@dataclass +class HabitatRecordV0_6_7: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + prefix (str): identifier prefix of hab qb64 + pid (str | None): group member identifier qb64 when hid is group + aids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + """ + prefix: str # aid qb64 + pid: Optional[str] # participant aid of group aid + aids: Optional[list] # all identifiers participating in the group identity + + watchers: list[str] = field(default_factory=list) # aids qb64 of watchers + +@dataclass +class HabitatRecordV0_6_8: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + +def _check_if_needed(db): + """ + Check if the migration is needed + Parameters: + db(Baser): Baser database object on which to run the migration + Returns: + bool: True if the migration is needed, False otherwise + """ + habs = koming.Komer(db=db, subkey='habs.', schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'prefix' in habord: + return True + return False + +def migrate(db): + """Rename data in HabitatRecord from the old labels to the new labels as of 2022-10-17 + + This migration performs the following: + 1. rename prefix -> hid + 2. rename pid -> mid + 3. rename aids -> smids, rmids + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_7, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + existing = asdict(habord) + habord_0_6_7 = HabitatRecordV0_6_7(**existing) + habord_0_6_8 = HabitatRecordV0_6_8( + hid=habord_0_6_7.prefix, + mid=habord_0_6_7.pid, + smids=habord_0_6_7.aids, + rmids=habord_0_6_7.aids, + sid=None, + watchers=habord_0_6_7.watchers + ) + habords[name] = habord_0_6_8 + + habs.trim() # remove existing records + + # Add in the renamed records + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_8, ) + + for name, habord in habords.items(): + name, = name + habs.pin(keys=(name,), val=habord) diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py index e21dfdb9..5149d4d1 100644 --- a/src/keri/db/migrations/rekey_habs.py +++ b/src/keri/db/migrations/rekey_habs.py @@ -4,7 +4,7 @@ @dataclass -class OldHabitatRecord: # baser.habs +class HabitatRecordV0_6_8: # baser.habs """ Habitat application state information keyed by habitat name (baser.habs) @@ -24,6 +24,17 @@ class OldHabitatRecord: # baser.habs sid: str | None = None # Signify identifier qb64 when hid is Signify watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers +def _check_if_needed(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'domain' in habord: + return False + return True def migrate(db): """ Re-key habs migration for changing the key for .habs and introducing the .names database @@ -36,24 +47,29 @@ def migrate(db): Parameters: db(Baser): Baser database object on which to run the migration - """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + habs = koming.Komer(db=db, subkey='habs.', - schema=OldHabitatRecord, ) + schema=dict, ) # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix nmsp = koming.Komer(db=db, subkey='nmsp.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) habords = dict() # Update Hab records from .habs with name for name, habord in habs.getItemIter(): name = ".".join(name) # detupleize the database key name - nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord = basing.HabitatRecord(**habord) nhabord.name = name - habords[habord.hid] = nhabord + habords[habord['hid']] = nhabord habs.trim() @@ -64,7 +80,7 @@ def migrate(db): nhabord = basing.HabitatRecord(**asdict(habord)) nhabord.name = name nhabord.domain = ns - habords[habord.hid] = nhabord + habords[habord['hid']] = nhabord nmsp.trim() # remove existing records @@ -72,4 +88,4 @@ def migrate(db): for pre, habord in habords.items(): db.habs.pin(keys=(pre,), val=habord) ns = "" if habord.domain is None else habord.domain - db.names.pin(keys=(ns, habord.name), val=pre) + db.names.pin(keys=(ns, habord.name), val=pre) \ No newline at end of file From c173a629bbf965f82d02a8332331d98dc4eae241 Mon Sep 17 00:00:00 2001 From: Fergal Date: Mon, 18 Nov 2024 12:07:43 -0300 Subject: [PATCH 09/11] feat: de-couple contact creation from OOBI resolution (#882) --- scripts/demo/basic/challenge.sh | 6 ++ .../multisig-grant-multisig-admit.sh | 16 +++++ .../multisig-issuer-interactive.sh | 7 +++ src/keri/app/cli/commands/contacts/list.py | 5 +- src/keri/app/cli/commands/contacts/replace.py | 61 +++++++++++++++++++ src/keri/app/oobiing.py | 5 -- 6 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 src/keri/app/cli/commands/contacts/replace.py diff --git a/scripts/demo/basic/challenge.sh b/scripts/demo/basic/challenge.sh index c3fcc6cc..b8419ebd 100755 --- a/scripts/demo/basic/challenge.sh +++ b/scripts/demo/basic/challenge.sh @@ -14,6 +14,12 @@ cha2_oobi="$(kli oobi generate --name cha2 --alias cha2 --role witness | sed -n kli oobi resolve --name cha1 --oobi-alias cha2 --oobi "${cha2_oobi}" kli oobi resolve --name cha2 --oobi-alias cha1 --oobi "${cha1_oobi}" +cha1_pre="$(kli aid --name cha1 --alias cha1)" +cha2_pre="$(kli aid --name cha2 --alias cha2)" + +kli contacts replace --name cha1 --prefix "${cha2_pre}" --alias cha2 +kli contacts replace --name cha2 --prefix "${cha1_pre}" --alias cha1 + words1="$(kli challenge generate --out string)" words2="$(kli challenge generate --out string)" diff --git a/scripts/demo/credentials/multisig-grant-multisig-admit.sh b/scripts/demo/credentials/multisig-grant-multisig-admit.sh index 713e9308..1cd4f00d 100755 --- a/scripts/demo/credentials/multisig-grant-multisig-admit.sh +++ b/scripts/demo/credentials/multisig-grant-multisig-admit.sh @@ -28,7 +28,9 @@ kli incept --name issuer2 --alias issuer2 --file ${KERI_DEMO_SCRIPT_DIR}/data/is # Exchange OOBIs between issuer group kli oobi resolve --name issuer1 --oobi-alias issuer2 --oobi http://127.0.0.1:5642/oobi/EFJtDtSoE6XOOqLoLvYoB7ctCzMtJDiAJltnXiK_EdlM/witness +kli contacts replace --name issuer1 --prefix EFJtDtSoE6XOOqLoLvYoB7ctCzMtJDiAJltnXiK_EdlM --alias issuer2 kli oobi resolve --name issuer2 --oobi-alias issuer1 --oobi http://127.0.0.1:5642/oobi/EEVlFHcMWAQNwezHjyKK5cKKzF6zgLlnrLyi_CcAEXCs/witness +kli contacts replace --name issuer2 --prefix EEVlFHcMWAQNwezHjyKK5cKKzF6zgLlnrLyi_CcAEXCs --alias issuer1 # Create the identifier to which the credential will be issued kli init --name issuee1 --salt 0ACDEyMzQ1Njc4OWxtbm9qWc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis @@ -40,19 +42,29 @@ kli incept --name issuee2 --alias issuee2 --file ${KERI_DEMO_SCRIPT_DIR}/data/is # Exchange OOBIs between issuee group kli oobi resolve --name issuee1 --oobi-alias issuee2 --oobi http://127.0.0.1:5642/oobi/EPw5WQAFcNXXSbg_pTKgh8-K_rfXnD1uDKS13OeNHkKE/witness +kli contacts replace --name issuee1 --prefix EPw5WQAFcNXXSbg_pTKgh8-K_rfXnD1uDKS13OeNHkKE --alias issuee2 kli oobi resolve --name issuee2 --oobi-alias issuee1 --oobi http://127.0.0.1:5642/oobi/EI0IoYyHxXc7_uQyaN2WocSC3lRZsvrDAPbREOw7fM0_/witness +kli contacts replace --name issuee2 --prefix EI0IoYyHxXc7_uQyaN2WocSC3lRZsvrDAPbREOw7fM0_ --alias issuee1 # Introduce issuer to issuee kli oobi resolve --name issuee1 --oobi-alias issuer1 --oobi http://127.0.0.1:5642/oobi/EEVlFHcMWAQNwezHjyKK5cKKzF6zgLlnrLyi_CcAEXCs/witness +kli contacts replace --name issuee1 --prefix EEVlFHcMWAQNwezHjyKK5cKKzF6zgLlnrLyi_CcAEXCs --alias issuer1 kli oobi resolve --name issuee2 --oobi-alias issuer1 --oobi http://127.0.0.1:5642/oobi/EEVlFHcMWAQNwezHjyKK5cKKzF6zgLlnrLyi_CcAEXCs/witness +kli contacts replace --name issuee2 --prefix EEVlFHcMWAQNwezHjyKK5cKKzF6zgLlnrLyi_CcAEXCs --alias issuer1 kli oobi resolve --name issuee1 --oobi-alias issuer2 --oobi http://127.0.0.1:5642/oobi/EFJtDtSoE6XOOqLoLvYoB7ctCzMtJDiAJltnXiK_EdlM/witness +kli contacts replace --name issuee1 --prefix EFJtDtSoE6XOOqLoLvYoB7ctCzMtJDiAJltnXiK_EdlM --alias issuer2 kli oobi resolve --name issuee2 --oobi-alias issuer2 --oobi http://127.0.0.1:5642/oobi/EFJtDtSoE6XOOqLoLvYoB7ctCzMtJDiAJltnXiK_EdlM/witness +kli contacts replace --name issuee2 --prefix EFJtDtSoE6XOOqLoLvYoB7ctCzMtJDiAJltnXiK_EdlM --alias issuer2 # Introduce the issuee to issuer kli oobi resolve --name issuer1 --oobi-alias issuee1 --oobi http://127.0.0.1:5642/oobi/EI0IoYyHxXc7_uQyaN2WocSC3lRZsvrDAPbREOw7fM0_/witness +kli contacts replace --name issuer1 --prefix EI0IoYyHxXc7_uQyaN2WocSC3lRZsvrDAPbREOw7fM0_ --alias issuee1 kli oobi resolve --name issuer2 --oobi-alias issuee1 --oobi http://127.0.0.1:5642/oobi/EI0IoYyHxXc7_uQyaN2WocSC3lRZsvrDAPbREOw7fM0_/witness +kli contacts replace --name issuer2 --prefix EI0IoYyHxXc7_uQyaN2WocSC3lRZsvrDAPbREOw7fM0_ --alias issuee1 kli oobi resolve --name issuer1 --oobi-alias issuee2 --oobi http://127.0.0.1:5642/oobi/EPw5WQAFcNXXSbg_pTKgh8-K_rfXnD1uDKS13OeNHkKE/witness +kli contacts replace --name issuer1 --prefix EPw5WQAFcNXXSbg_pTKgh8-K_rfXnD1uDKS13OeNHkKE --alias issuee2 kli oobi resolve --name issuer2 --oobi-alias issuee2 --oobi http://127.0.0.1:5642/oobi/EPw5WQAFcNXXSbg_pTKgh8-K_rfXnD1uDKS13OeNHkKE/witness +kli contacts replace --name issuer2 --prefix EPw5WQAFcNXXSbg_pTKgh8-K_rfXnD1uDKS13OeNHkKE --alias issuee2 ## Load Data OOBI for schema of credential to issue kli oobi resolve --name issuer1 --oobi-alias vc --oobi http://127.0.0.1:7723/oobi/EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao @@ -84,10 +96,14 @@ wait $PID_LIST # Introduce issuer issuer issuer to issuees kli oobi resolve --name issuer1 --oobi-alias issuee --oobi http://127.0.0.1:5642/oobi/ELkmm28zQEyxkryJZQ4WVT4fjukklM4dR91l2DQfQHZK/witness +kli contacts replace --name issuer1 --prefix ELkmm28zQEyxkryJZQ4WVT4fjukklM4dR91l2DQfQHZK --alias issuee kli oobi resolve --name issuer2 --oobi-alias issuee --oobi http://127.0.0.1:5642/oobi/ELkmm28zQEyxkryJZQ4WVT4fjukklM4dR91l2DQfQHZK/witness +kli contacts replace --name issuer2 --prefix ELkmm28zQEyxkryJZQ4WVT4fjukklM4dR91l2DQfQHZK --alias issuee kli oobi resolve --name issuee1 --oobi-alias issuer --oobi http://127.0.0.1:5642/oobi/ELrnb8aI_wy2q_sSbCAwkgy2kOdMpRI1urFrhQiMJGLW/witness +kli contacts replace --name issuee1 --prefix ELrnb8aI_wy2q_sSbCAwkgy2kOdMpRI1urFrhQiMJGLW --alias issuer kli oobi resolve --name issuee2 --oobi-alias issuer --oobi http://127.0.0.1:5642/oobi/ELrnb8aI_wy2q_sSbCAwkgy2kOdMpRI1urFrhQiMJGLW/witness +kli contacts replace --name issuee2 --prefix ELrnb8aI_wy2q_sSbCAwkgy2kOdMpRI1urFrhQiMJGLW --alias issuer # Create a credential registry owned by the issuer issuer kli vc registry incept --name issuer1 --alias issuer --registry-name vLEI --usage "Issue vLEIs" --nonce AHSNDV3ABI6U8OIgKaj3aky91ZpNL54I5_7-qwtC6q2s & diff --git a/scripts/demo/credentials/multisig-issuer-interactive.sh b/scripts/demo/credentials/multisig-issuer-interactive.sh index c26b265d..9b558f4d 100755 --- a/scripts/demo/credentials/multisig-issuer-interactive.sh +++ b/scripts/demo/credentials/multisig-issuer-interactive.sh @@ -15,7 +15,9 @@ kli incept --name multisig2 --alias multisig2 --file ${KERI_DEMO_SCRIPT_DIR}/dat # Exchange OOBIs between multisig group kli oobi resolve --name multisig1 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness +kli contacts replace --name multisig1 --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --alias multisig2 kli oobi resolve --name multisig2 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness +kli contacts replace --name multisig2 --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --alias multisig1 # Create the identifier to which the credential will be issued kli init --name holder --salt 0ACDEyMzQ1Njc4OWxtbm9qWc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis @@ -23,11 +25,15 @@ kli incept --name holder --alias holder --file ${KERI_DEMO_SCRIPT_DIR}/data/glei # Introduce multisig to Holder kli oobi resolve --name holder --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness +kli contacts replace --name holder --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --alias multisig2 kli oobi resolve --name holder --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness +kli contacts replace --name holder --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --alias multisig1 # Introduce the holder to all participants in the multisig group kli oobi resolve --name multisig1 --oobi-alias holder --oobi http://127.0.0.1:5642/oobi/ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k/witness +kli contacts replace --name multisig1 --prefix ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k --alias holder kli oobi resolve --name multisig2 --oobi-alias holder --oobi http://127.0.0.1:5642/oobi/ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k/witness +kli contacts replace --name multisig2 --prefix ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k --alias holder # Load Data OOBI for schema of credential to issue kli oobi resolve --name multisig1 --oobi-alias vc --oobi http://127.0.0.1:7723/oobi/EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao @@ -48,6 +54,7 @@ kli multisig join --name multisig2 wait $PID_LIST kli oobi resolve --name holder --oobi-alias multisig --oobi http://127.0.0.1:5642/oobi/EC61gZ9lCKmHAS7U5ehUfEbGId5rcY0D7MirFZHDQcE2/witness +kli contacts replace --name holder --prefix EC61gZ9lCKmHAS7U5ehUfEbGId5rcY0D7MirFZHDQcE2 --alias multisig # Create a credential registry owned by the multisig issuer kli vc registry incept --name multisig1 --alias multisig --registry-name vLEI --usage "Issue vLEIs" --nonce AHSNDV3ABI6U8OIgKaj3aky91ZpNL54I5_7-qwtC6q2s & diff --git a/src/keri/app/cli/commands/contacts/list.py b/src/keri/app/cli/commands/contacts/list.py index d4c626d3..0de4e9ac 100644 --- a/src/keri/app/cli/commands/contacts/list.py +++ b/src/keri/app/cli/commands/contacts/list.py @@ -1,11 +1,12 @@ # -*- encoding: utf-8 -*- """ KERI -keri.kli.commands module +keri.kli.commands.contacts module """ import argparse import json +import sys from hio import help from hio.base import doing @@ -76,4 +77,4 @@ def list(tymth, tock=0.0, **opts): except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 + sys.exit(-1) diff --git a/src/keri/app/cli/commands/contacts/replace.py b/src/keri/app/cli/commands/contacts/replace.py new file mode 100644 index 00000000..3c728310 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/replace.py @@ -0,0 +1,61 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts module + +""" +import argparse +import sys + +from hio import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +# Could be expanded to provide arbitrary data if desired +parser = argparse.ArgumentParser(description='Replace contact information for identifier prefix with alias information') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--prefix', '-o', help='identifier prefix to replace contact information for', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the contact', required=True) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(replace, **kwa)] + + +def replace(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + prefix = args.prefix + alias = args.alias + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + if prefix not in hby.kevers: + print(f"{prefix} is not a known identifier, oobi must be resolved first") + sys.exit(-1) + + org.replace(pre=prefix, data=dict(alias=alias)) + + except ConfigurationError: + print(f"identifier prefix for {name} does not exist, incept must be run first") + sys.exit(-1) diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index 7228b918..85bb5446 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -18,7 +18,6 @@ from . import httping from .. import help from .. import kering -from ..app import connecting from ..core import routing, eventing, parsing, scheming, serdering from ..db import basing from ..end import ending @@ -290,7 +289,6 @@ def __init__(self, hby, rvy=None, clienter=None, cues=None): self.registerReplyRoutes(self.rvy.rtr) self.clienter = clienter or httping.Clienter() - self.org = connecting.Organizer(hby=self.hby) # Set up a local parser for returned events from OOBI queries. rtr = routing.Router() @@ -502,9 +500,6 @@ def processClients(self): if ending.OOBI_AID_HEADER in response["headers"]: obr.cid = response["headers"][ending.OOBI_AID_HEADER] - if obr.oobialias is not None and obr.cid: - self.org.replace(pre=obr.cid, data=dict(alias=obr.oobialias, oobi=url)) - self.hby.db.coobi.rem(keys=(url,)) obr.state = Result.resolved self.hby.db.roobi.put(keys=(url,), val=obr) From 068893578286fc5e33deeead2c00aef5e0c2469c Mon Sep 17 00:00:00 2001 From: Arshdeep Singh Date: Mon, 18 Nov 2024 20:41:12 +0530 Subject: [PATCH 10/11] add null check for issuee in sendArtifacts for presenting untargeted credential. (#888) - Fix exception in sendArtifacts by adding a null check for issuee in case of presenting untargeted credential Signed-off-by: arshdeep singh --- src/keri/vdr/credentialing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 416bb0af..687fd22c 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -968,7 +968,7 @@ def sendArtifacts(hby, reger, postman, creder, recp): atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) - if isse != recp: + if isse is not None and isse != recp: ikever = hby.db.kevers[isse] for msg in hby.db.cloneDelegation(ikever): serder = serdering.SerderKERI(raw=msg) From 7737bfdb31c69e0b0995fbae6e8bfc48eaeb4cd7 Mon Sep 17 00:00:00 2001 From: Daniel Hardman Date: Tue, 3 Dec 2024 08:05:49 -0700 Subject: [PATCH 11/11] fix issue 892 (#893) * fix #892 Signed-off-by: Daniel Hardman * fix test that corresponds to previous change Signed-off-by: Daniel Hardman --------- Signed-off-by: Daniel Hardman --- src/keri/core/coring.py | 2 +- tests/core/test_coring.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 99fe7e18..25006d23 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -804,7 +804,7 @@ class Matter: '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), '1AAD': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), - '1AAE': Sizage(hs=4, ss=0, xs=0, fs=56, ls=0), + '1AAE': Sizage(hs=4, ss=0, xs=0, fs=156, ls=0), '1AAF': Sizage(hs=4, ss=4, xs=0, fs=8, ls=0), '1AAG': Sizage(hs=4, ss=0, xs=0, fs=36, ls=0), '1AAH': Sizage(hs=4, ss=0, xs=0, fs=100, ls=0), diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index e4118949..61e87e88 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -460,7 +460,7 @@ def test_matter_class(): '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), '1AAC': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), '1AAD': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), - '1AAE': Sizage(hs=4, ss=0, xs=0, fs=56, ls=0), + '1AAE': Sizage(hs=4, ss=0, xs=0, fs=156, ls=0), '1AAF': Sizage(hs=4, ss=4, xs=0, fs=8, ls=0), '1AAG': Sizage(hs=4, ss=0, xs=0, fs=36, ls=0), '1AAH': Sizage(hs=4, ss=0, xs=0, fs=100, ls=0),