From b50993d457d5d8591eca059151304142456f0c54 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Tue, 8 Oct 2024 16:18:38 -0600 Subject: [PATCH] feat: migrations for 0.6.7 to 1.2.0 --- src/keri/app/cli/commands/migrate.py | 162 ------------------ src/keri/app/cli/commands/migrate/list.py | 4 +- src/keri/app/cli/commands/migrate/run.py | 14 +- src/keri/db/basing.py | 33 ++-- .../add_key_and_reg_state_schemas.py | 132 ++++++++++++++ src/keri/db/migrations/hab_data_rename.py | 108 ++++++++++++ src/keri/db/migrations/rekey_habs.py | 12 +- 7 files changed, 279 insertions(+), 186 deletions(-) delete mode 100644 src/keri/app/cli/commands/migrate.py create mode 100644 src/keri/db/migrations/add_key_and_reg_state_schemas.py create mode 100644 src/keri/db/migrations/hab_data_rename.py diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py deleted file mode 100644 index 878587790..000000000 --- a/src/keri/app/cli/commands/migrate.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -KERI -keri.kli.commands module - -""" -import argparse - -from hio import help -from hio.base import doing - -from keri import kering -from keri.app.cli.common import existing -from keri.core import coring, serdering -from keri.db import koming, subing, dbing -from keri.db.basing import KeyStateRecord, StateEERecord -from keri.kering import ConfigurationError, Version -from keri.vdr import viring - -logger = help.ogler.getLogger() - -parser = argparse.ArgumentParser(description='View status of a local AID') -parser.set_defaults(handler=lambda args: handler(args), - transferable=True) -parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) -parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', - required=False, default="") -parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', - dest="bran", default=None) # passcode => bran -parser.add_argument('--force', action="store_true", required=False, - help='True means perform migration without prompting the user') - - -def handler(args): - if not args.force: - print() - print("This command will migrate your datastore to the next version of KERIpy and is not reversible.") - print("After this command, you will not be able to access your data store with this version.") - print() - yn = input("Are you sure you want to continue? [y|N]: ") - - if yn not in ("y", "Y"): - print("...exiting") - return [] - - kwa = dict(args=args) - return [doing.doify(migrate, **kwa)] - - -def migrate(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - base = args.base - bran = args.bran - - try: - with dbing.openLMDB(name=name, base=base, bran=bran, temp=False) as db: - print(db.path) - states = koming.Komer(db=db, - schema=dict, - subkey='stts.') - nstates = koming.Komer(db=db, - schema=KeyStateRecord, - subkey='stts.') - - for keys, sad in states.getItemIter(): - ksr = KeyStateRecord( - vn=Version, # version number as list [major, minor] - i=sad['i'], # qb64 prefix - s=sad['s'], # lowercase hex string no leading zeros - p=sad['p'], - d=sad['d'], - f=sad['f'], # lowercase hex string no leading zeros - dt=sad['dt'], - et=sad['et'], - kt=sad['kt'], - k=sad['k'], - nt=sad['nt'], - n=sad['n'], - bt=sad['bt'], - b=sad['b'], - c=sad['c'], - ee=StateEERecord._fromdict(sad['ee']), # latest est event dict - di=sad['di'] if sad['di'] else None - ) - - nstates.pin(keys=keys, val=ksr) - - with existing.existingHby(name=name, base=base, bran=bran) as hby: - rgy = viring.Reger(name=name, base=base, db=hby.db, temp=False, - reopen=True) - - rstates = koming.Komer(db=rgy, - schema=dict, - subkey='stts.') - - for _, sad in rstates.getItemIter(): - rsr = viring.RegStateRecord( - vn=list(Version), # version number as list [major, minor] - i=sad['i'], # qb64 registry SAID - s=sad['s'], # lowercase hex string no leading zeros - d=sad['d'], - ii=sad['ii'], - dt=sad['dt'], - et=sad['et'], - bt=sad['bt'], # hex string no leading zeros lowercase - b=sad['b'], # list of qb64 may be empty - c=sad['c'], - ) - # ksr = stateFromKever(kever) - rgy.states.pin(sad['i'], val=rsr) - - for (said,), _ in rgy.creds.getItemIter(): - snkey = dbing.snKey(said, 0) - dig = rgy.getTel(key=snkey) - - prefixer = coring.Prefixer(qb64=said) - seqner = coring.Seqner(sn=0) - saider = coring.Saider(qb64b=bytes(dig)) - rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) - - migrateKeys(hby.db) - - # clear escrows - print("clearing escrows") - hby.db.gpwe.trim() - hby.db.gdee.trim() - hby.db.dpwe.trim() - hby.db.gpse.trim() - hby.db.epse.trim() - hby.db.dune.trim() - - except ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - - -def migrateKeys(db): - # public keys mapped to the AID and event seq no they appeared in - pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in - digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - - for pre, fn, dig in db.getFelItemAllPreIter(): - dgkey = dbing.dgKey(pre, dig) # get message - if not (raw := db.getEvt(key=dgkey)): - raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) - serder = serdering.SerderKERI(raw=bytes(raw)) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - verfers = serder.verfers or [] - for verfer in verfers: - pubs.add(keys=(verfer.qb64,), val=val) - ndigers = serder.ndigers or [] - for diger in ndigers: - digs.add(keys=(diger.qb64,), val=val) diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index 104fa8baf..152850436 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -16,7 +16,7 @@ def handler(args): """ - Launch KERI database initialization + List local LMDB database migrations and their completion status Args: args(Namespace): arguments object from command line @@ -25,7 +25,7 @@ def handler(args): return [lister] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Lists the local LMDB migrations and their completion status') parser.set_defaults(handler=handler, transferable=True) diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index 9a9e4c818..9febc3046 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -18,16 +18,16 @@ def handler(args): """ - Launch KERI database initialization + Launch KERI database migrator Args: args(Namespace): arguments object from command line """ - clean = MigrateDoer(args) - return [clean] + migrator = MigrateDoer(args) + return [migrator] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore up to the latest source code version') parser.set_defaults(handler=handler, transferable=True) @@ -60,8 +60,8 @@ def recur(self, tyme): except kering.DatabaseError: pass - print("Migrating...") - db.migrate() - print("Finished") + print(f"Migrating {self.args.name}...") + db.migrate(name=self.args.name, base=self.args.base, temp=self.args.temp) + print(f"Finished migrating {self.args.name}") return True diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 14e9bfb53..8131b4e05 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -49,7 +49,9 @@ MIGRATIONS = [ - ("1.1.0", ["rekey_habs"]) + ("0.6.8", ["hab_data_rename"]), + ("1.0.0", ["hab_data_rename", "add_key_and_reg_state_schemas"]), + ("1.2.0", ["hab_data_rename", "add_key_and_reg_state_schemas", "rekey_habs"]) ] @@ -1335,7 +1337,7 @@ def reload(self): for keys in removes: # remove bare .habs records self.habs.rem(keys=keys) - def migrate(self): + def migrate(self, name, base, temp): """ Run all migrations required Run all migrations that are required from the current version of database up to the current version @@ -1346,10 +1348,18 @@ def migrate(self): """ for (version, migrations) in MIGRATIONS: - # Check to see if this is for an older version + # Only run migration if current source code version is at or below the migration version + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(version, ver_no_prerelease) > 0: + print( + f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") + continue + # Skip migrations already run - where version less than (-1) or equal to (0) database version if self.version is not None and semver.compare(version, self.version) != 1: continue + print(f"Migrating database v{self.version} --> v{version}") for migration in migrations: modName = f"keri.db.migrations.{migration}" if self.migs.get(keys=(migration,)) is not None: @@ -1357,13 +1367,13 @@ def migrate(self): mod = importlib.import_module(modName) try: - print(f"running migration {modName}") - mod.migrate(self) + mod.migrate(self, name=name, base=base, temp=temp) except Exception as e: print(f"\nAbandoning migration {migration} with error: {e}") return self.migs.pin(keys=(migration,), val=coring.Dater()) + self.version = version # update database version after successful migration self.version = keri.__version__ @@ -1407,12 +1417,15 @@ def complete(self, name=None): migrations = [] if not name: for version, migs in MIGRATIONS: - for mig in migs: - dater = self.migs.get(keys=(mig,)) - migrations.append((mig, dater)) + # Print entries only for migrations that have been run + if self.version is not None and semver.compare(version, self.version) <= 0: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) else: - if name not in MIGRATIONS or not self.migs.get(keys=(name,)): - raise ValueError(f"No migration named {name}") + for version, migs in MIGRATIONS: # check all migrations for each version + if name not in migs or not self.migs.get(keys=(name,)): + raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) return migrations diff --git a/src/keri/db/migrations/add_key_and_reg_state_schemas.py b/src/keri/db/migrations/add_key_and_reg_state_schemas.py new file mode 100644 index 000000000..5f0b79e75 --- /dev/null +++ b/src/keri/db/migrations/add_key_and_reg_state_schemas.py @@ -0,0 +1,132 @@ +from keri import help +from keri.core import coring, serdering +from keri.db import koming, subing, dbing +from keri.db.basing import StateEERecord, KeyStateRecord +from keri.db.dbing import dgKey, splitKey +from keri.kering import ConfigurationError, Version +from keri.vdr import viring + +logger = help.ogler.getLogger() + +def migrate(db, name, base, temp): + """Adds schema for KeyStateRecord, RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., + and hby.db.digs. to be up to date as of 2022-??-?? + This migration performs the following: + - hby.db -> "stts." schema from dict -> KeyStateRecord + - rgy -> "stts." schema from dict -> RegStateRecord + - rgy -> "cancs." reset to (ACDC SAID, SN 0, TEL evt 0 digest) + - hby.db -> "pubs." and + hby.db -> "digs." + that don't exist are populated with verification keys and event digests for the first seen events and + Keys: + "pubs." Verfer of each Verfer for each FEL event + "digs." Diger of next Diger (ndiger) of each FEL event + Value: (prefix, sn) of each event + Parameters: + db(Baser): Baser database object on which to run the migration + name(str): name of the keystore + base(str): additional optional prefix to file location of KERI keystore + temp(bool): create a temporary keystore, used for testing + """ + try: + logger.debug(f"Migrating keystate and regstate dict to schema for {db.path}") + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + nstates = koming.Komer(db=db, + schema=KeyStateRecord, + subkey='stts.') + + for keys, sad in states.getItemIter(): + ksr = KeyStateRecord( + vn=Version, # version number as list [major, minor] + i=sad['i'], # qb64 prefix + s=sad['s'], # lowercase hex string no leading zeros + p=sad['p'], + d=sad['d'], + f=sad['f'], # lowercase hex string no leading zeros + dt=sad['dt'], + et=sad['et'], + kt=sad['kt'], + k=sad['k'], + nt=sad['nt'], + n=sad['n'], + bt=sad['bt'], + b=sad['b'], + c=sad['c'], + ee=StateEERecord._fromdict(sad['ee']), # latest est event dict + di=sad['di'] if sad['di'] else None + ) + + nstates.pin(keys=keys, val=ksr) + + rgy = viring.Reger(name=name, base=base, db=db, temp=temp, reopen=True) + + rstates = koming.Komer(db=rgy, + schema=dict, + subkey='stts.') + + for _, sad in rstates.getItemIter(): + rsr = viring.RegStateRecord( + vn=list(Version), # version number as list [major, minor] + i=sad['i'], # qb64 registry SAID + s=sad['s'], # lowercase hex string no leading zeros + d=sad['d'], + ii=sad['ii'], + dt=sad['dt'], + et=sad['et'], + bt=sad['bt'], # hex string no leading zeros lowercase + b=sad['b'], # list of qb64 may be empty + c=sad['c'], + ) + # ksr = stateFromKever(kever) + rgy.states.pin(sad['i'], val=rsr) + + for (said,), _ in rgy.saved.getItemIter(): + snkey = dbing.snKey(said, 0) + dig = rgy.getTel(key=snkey) + + prefixer = coring.Prefixer(qb64=said) + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64b=bytes(dig)) + rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) + + migrateKeys(db) + + # clear escrows + logger.info("clearing escrows") + db.gpwe.trim() + db.gdee.trim() + db.dpwe.trim() + db.gpse.trim() + db.epse.trim() + db.dune.trim() + db.qnfs.trim() + + except ConfigurationError: + logger.error(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 + + +def migrateKeys(db): + # public keys mapped to the AID and event seq no they appeared in + pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", + klas=(coring.Prefixer, coring.Seqner)) + + # next key digests mapped to the AID and event seq no they appeared in + digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", + klas=(coring.Prefixer, coring.Seqner)) + + for pre, fn, dig in db.getFelItemAllPreIter(): + dgkey = dbing.dgKey(pre, dig) # get message + if not (raw := db.getEvt(key=dgkey)): + logger.info(f"Migrate keys: missing event for dig={dig}, skipped.") + continue + serder = serdering.SerderKERI(raw=bytes(raw)) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) + verfers = serder.verfers or [] + for verfer in verfers: + pubs.add(keys=(verfer.qb64,), val=val) + ndigers = serder.ndigers or [] + for diger in ndigers: + digs.add(keys=(diger.qb64,), val=val) \ No newline at end of file diff --git a/src/keri/db/migrations/hab_data_rename.py b/src/keri/db/migrations/hab_data_rename.py new file mode 100644 index 000000000..65e71d1ce --- /dev/null +++ b/src/keri/db/migrations/hab_data_rename.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass, field, asdict +from typing import Optional + +from keri.db import koming, basing +from keri.db.basing import HabitatRecord, Baser +from keri.vdr.viring import Reger + + +@dataclass +class HabitatRecordV0_6_7: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + prefix (str): identifier prefix of hab qb64 + pid (str | None): group member identifier qb64 when hid is group + aids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + """ + prefix: str # aid qb64 + pid: Optional[str] # participant aid of group aid + aids: Optional[list] # all identifiers participating in the group identity + + watchers: list[str] = field(default_factory=list) # aids qb64 of watchers + +@dataclass +class HabitatRecordV0_6_8: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + +def _check_if_needed(db): + """ + Check if the migration is needed + + Parameters: + db(Baser): Baser database object on which to run the migration + + Returns: + bool: True if the migration is needed, False otherwise + """ + habs = koming.Komer(db=db, subkey='habs.', schema=dict, ) + for name, habord in habs.getItemIter(): + if 'prefix' in habord: + return True + return False + +def migrate(db, name, base, temp): + """Rename data in HabitatRecord from the old labels to the new labels as of 2022-10-17 + + This migration performs the following: + 1. rename prefix -> hid + 2. rename pid -> mid + 3. rename aids -> smids, rmids + + Parameters: + db(Baser): Baser database object on which to run the migration + name(str): name of the keystore + base(str): additional optional prefix to file location of KERI keystore + temp(bool): create a temporary keystore, used for testing + """ + if not _check_if_needed(db): + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_7, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + existing = asdict(habord) + habord_0_6_7 = HabitatRecordV0_6_7(**existing) + habord_0_6_8 = HabitatRecordV0_6_8( + hid=habord_0_6_7.prefix, + mid=habord_0_6_7.pid, + smids=habord_0_6_7.aids, + rmids=habord_0_6_7.aids, + sid=None, + watchers=habord_0_6_7.watchers + ) + habords[habord_0_6_8.hid] = habord_0_6_8 + + habs.trim() # remove existing records + + # Add in the renamed records + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_8, ) + + for pre, habord in habords.items(): + habs.pin(keys=(pre,), val=habord) + diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py index e21dfdb9f..ede7b2657 100644 --- a/src/keri/db/migrations/rekey_habs.py +++ b/src/keri/db/migrations/rekey_habs.py @@ -4,7 +4,7 @@ @dataclass -class OldHabitatRecord: # baser.habs +class HabitatRecordV0_6_8: # baser.habs """ Habitat application state information keyed by habitat name (baser.habs) @@ -25,7 +25,7 @@ class OldHabitatRecord: # baser.habs watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers -def migrate(db): +def migrate(db, name, base, temp): """ Re-key habs migration for changing the key for .habs and introducing the .names database This migrations performs the following: @@ -36,16 +36,18 @@ def migrate(db): Parameters: db(Baser): Baser database object on which to run the migration - + name(str): name of the keystore + base(str): additional optional prefix to file location of KERI keystore + temp(bool): create a temporary keystore, used for testing """ habs = koming.Komer(db=db, subkey='habs.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix nmsp = koming.Komer(db=db, subkey='nmsp.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) habords = dict() # Update Hab records from .habs with name