From a0887acb303584c16cffa5b24cf7dccc0d2d8982 Mon Sep 17 00:00:00 2001 From: Kent Bull Date: Mon, 14 Oct 2024 12:01:54 -0600 Subject: [PATCH] feat: migrates from 0.6.7 to 1.2.0 This includes migrations 0.6.8, 1.0.0, and changes the 1.1.0 rekey_habs to 1.2.0 for clarity --- src/keri/app/cli/commands/migrate/list.py | 6 +- src/keri/app/cli/commands/migrate/run.py | 16 +- src/keri/db/basing.py | 44 ++++-- .../add_key_and_reg_state_schemas.py | 147 ++++++++++++++++++ src/keri/db/migrations/hab_data_rename.py | 108 +++++++++++++ src/keri/db/migrations/rekey_habs.py | 26 +++- 6 files changed, 317 insertions(+), 30 deletions(-) create mode 100644 src/keri/db/migrations/add_key_and_reg_state_schemas.py create mode 100644 src/keri/db/migrations/hab_data_rename.py diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index 104fa8baf..f3f8c13c1 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- """ -keri.kli.commands module +keri.kli.commands.migrate.list module """ import argparse @@ -16,7 +16,7 @@ def handler(args): """ - Launch KERI database initialization + List local LMDB database migrations and their completion status Args: args(Namespace): arguments object from command line @@ -25,7 +25,7 @@ def handler(args): return [lister] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Lists the local LMDB migrations and their completion status') parser.set_defaults(handler=handler, transferable=True) diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index 9a9e4c818..e87ca15f3 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -1,16 +1,14 @@ # -*- encoding: utf-8 -*- """ -keri.kli.commands module +keri.kli.commands.migrate.run module """ import argparse -import keri from hio import help from hio.base import doing from keri import kering -from keri.app.cli.common import existing from keri.db import basing logger = help.ogler.getLogger() @@ -18,16 +16,16 @@ def handler(args): """ - Launch KERI database initialization + Launch KERI database migrator Args: args(Namespace): arguments object from command line """ - clean = MigrateDoer(args) - return [clean] + migrator = MigrateDoer(args) + return [migrator] -parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser = argparse.ArgumentParser(description='Migrates a database and keystore') parser.set_defaults(handler=handler, transferable=True) @@ -60,8 +58,8 @@ def recur(self, tyme): except kering.DatabaseError: pass - print("Migrating...") + print(f"Migrating {self.args.name}...") db.migrate() - print("Finished") + print(f"Finished migrating {self.args.name}") return True diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 35fc63626..25e0bc6cf 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -49,7 +49,9 @@ MIGRATIONS = [ - ("1.1.0", ["rekey_habs"]) + ("0.6.8", ["hab_data_rename"]), + ("1.0.0", ["add_key_and_reg_state_schemas"]), + ("1.2.0", ["rekey_habs"]) ] @@ -1312,7 +1314,7 @@ def reload(self): """ # Check migrations to see if this database is up to date. Error otherwise if not self.current: - raise kering.DatabaseError("Database migrations must be run.") + raise kering.DatabaseError(f"Database migrations must be run. DB version {self.version}; current {keri.__version__}") removes = [] for keys, data in self.habs.getItemIter(): @@ -1346,10 +1348,18 @@ def migrate(self): """ for (version, migrations) in MIGRATIONS: - # Check to see if this is for an older version + # Only run migration if current source code version is at or below the migration version + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(version, str(ver_no_prerelease)) > 0: + print( + f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") + continue + # Skip migrations already run - where version less than (-1) or equal to (0) database version if self.version is not None and semver.compare(version, self.version) != 1: continue + print(f"Migrating database v{self.version} --> v{version}") for migration in migrations: modName = f"keri.db.migrations.{migration}" if self.migs.get(keys=(migration,)) is not None: @@ -1360,11 +1370,14 @@ def migrate(self): print(f"running migration {modName}") mod.migrate(self) except Exception as e: - print(f"\nAbandoning migration {migration} with error: {e}") + print(f"\nAbandoning migration {migration} at version {version} with error: {e}") return self.migs.pin(keys=(migration,), val=coring.Dater()) + # update database version after successful migration + self.version = version + self.version = keri.__version__ def clearEscrows(self): @@ -1407,14 +1420,16 @@ def current(self): if self.version == keri.__version__: return True - # If database version is ahead of library version, throw exception - if self.version is not None and semver.compare(self.version, keri.__version__) == 1: + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(self.version, str(ver_no_prerelease)) == 1: raise kering.ConfigurationError( f"Database version={self.version} is ahead of library version={keri.__version__}") last = MIGRATIONS[-1] - # If we aren't at latest version, but there are no outstanding migrations, reset version to latest - if self.migs.get(keys=(last[1][0],)) is not None: + # If we aren't at latest version, but there are no outstanding migrations, + # reset version to latest (rightmost (-1) migration is latest) + if self.migs.get(keys=(last[1][-1],)) is not None: return True # We have migrations to run @@ -1433,12 +1448,15 @@ def complete(self, name=None): migrations = [] if not name: for version, migs in MIGRATIONS: - for mig in migs: - dater = self.migs.get(keys=(mig,)) - migrations.append((mig, dater)) + # Print entries only for migrations that have been run + if self.version is not None and semver.compare(version, self.version) <= 0: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) else: - if name not in MIGRATIONS or not self.migs.get(keys=(name,)): - raise ValueError(f"No migration named {name}") + for version, migs in MIGRATIONS: # check all migrations for each version + if name not in migs or not self.migs.get(keys=(name,)): + raise ValueError(f"No migration named {name}") migrations.append((name, self.migs.get(keys=(name,)))) return migrations diff --git a/src/keri/db/migrations/add_key_and_reg_state_schemas.py b/src/keri/db/migrations/add_key_and_reg_state_schemas.py new file mode 100644 index 000000000..3a632f466 --- /dev/null +++ b/src/keri/db/migrations/add_key_and_reg_state_schemas.py @@ -0,0 +1,147 @@ +from keri import help +from keri.core import coring, serdering +from keri.db import koming, subing, dbing +from keri.db.basing import StateEERecord, KeyStateRecord +from keri.db.dbing import dgKey, splitKey +from keri.kering import ConfigurationError, Version +from keri.vdr import viring + +logger = help.ogler.getLogger() + +def _check_if_needed(db): + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + first = next(states.getItemIter(), None) + if first is None: + return False + keys, sad = first + if 'vn' in sad: + return False + return True + +def migrate(db): + """Adds schema for KeyStateRecord, RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., + and hby.db.digs. to be up to date as of 2022-??-?? + This migration performs the following: + - hby.db -> "stts." schema from dict -> KeyStateRecord + - rgy -> "stts." schema from dict -> RegStateRecord + - rgy -> "cancs." reset to (ACDC SAID, SN 0, TEL evt 0 digest) + - hby.db -> "pubs." and + hby.db -> "digs." + that don't exist are populated with verification keys and event digests for the first seen events and + Keys: + "pubs." Verfer of each Verfer for each FEL event + "digs." Diger of next Diger (ndiger) of each FEL event + Value: (prefix, sn) of each event + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + try: + logger.debug(f"Migrating keystate and regstate dict to schema for {db.path}") + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + nstates = koming.Komer(db=db, + schema=KeyStateRecord, + subkey='stts.') + + for keys, sad in states.getItemIter(): + ksr = KeyStateRecord( + vn=Version, # version number as list [major, minor] + i=sad['i'], # qb64 prefix + s=sad['s'], # lowercase hex string no leading zeros + p=sad['p'], + d=sad['d'], + f=sad['f'], # lowercase hex string no leading zeros + dt=sad['dt'], + et=sad['et'], + kt=sad['kt'], + k=sad['k'], + nt=sad['nt'], + n=sad['n'], + bt=sad['bt'], + b=sad['b'], + c=sad['c'], + ee=StateEERecord._fromdict(sad['ee']), # latest est event dict + di=sad['di'] if sad['di'] else None + ) + + nstates.pin(keys=keys, val=ksr) + + rgy = viring.Reger(name=db.name, base=db.base, db=db, temp=db.temp, reopen=True) + + rstates = koming.Komer(db=rgy, + schema=dict, + subkey='stts.') + + for _, sad in rstates.getItemIter(): + rsr = viring.RegStateRecord( + vn=list(Version), # version number as list [major, minor] + i=sad['i'], # qb64 registry SAID + s=sad['s'], # lowercase hex string no leading zeros + d=sad['d'], + ii=sad['ii'], + dt=sad['dt'], + et=sad['et'], + bt=sad['bt'], # hex string no leading zeros lowercase + b=sad['b'], # list of qb64 may be empty + c=sad['c'], + ) + # ksr = stateFromKever(kever) + rgy.states.pin(sad['i'], val=rsr) + + for (said,), _ in rgy.saved.getItemIter(): + snkey = dbing.snKey(said, 0) + dig = rgy.getTel(key=snkey) + + prefixer = coring.Prefixer(qb64=said) + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64b=bytes(dig)) + rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) + + migrateKeys(db) + + # clear escrows + logger.info("clearing escrows") + db.gpwe.trim() + db.gdee.trim() + db.dpwe.trim() + db.gpse.trim() + db.epse.trim() + db.dune.trim() + db.qnfs.trim() + + except ConfigurationError: + logger.error(f"identifier prefix for {db.name} does not exist, incept must be run first", ) + return -1 + + +def migrateKeys(db): + # public keys mapped to the AID and event seq no they appeared in + pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", + klas=(coring.Prefixer, coring.Seqner)) + + # next key digests mapped to the AID and event seq no they appeared in + digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", + klas=(coring.Prefixer, coring.Seqner)) + + for pre, fn, dig in db.getFelItemAllPreIter(): + dgkey = dbing.dgKey(pre, dig) # get message + if not (raw := db.getEvt(key=dgkey)): + logger.info(f"Migrate keys: missing event for dig={dig}, skipped.") + continue + serder = serdering.SerderKERI(raw=bytes(raw)) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) + verfers = serder.verfers or [] + for verfer in verfers: + pubs.add(keys=(verfer.qb64,), val=val) + ndigers = serder.ndigers or [] + for diger in ndigers: + digs.add(keys=(diger.qb64,), val=val) \ No newline at end of file diff --git a/src/keri/db/migrations/hab_data_rename.py b/src/keri/db/migrations/hab_data_rename.py new file mode 100644 index 000000000..8bc3d07f2 --- /dev/null +++ b/src/keri/db/migrations/hab_data_rename.py @@ -0,0 +1,108 @@ +from dataclasses import dataclass, field, asdict +from typing import Optional + +from keri.db import koming, basing +from keri.db.basing import HabitatRecord, Baser +from keri.vdr.viring import Reger + + +@dataclass +class HabitatRecordV0_6_7: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + prefix (str): identifier prefix of hab qb64 + pid (str | None): group member identifier qb64 when hid is group + aids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + """ + prefix: str # aid qb64 + pid: Optional[str] # participant aid of group aid + aids: Optional[list] # all identifiers participating in the group identity + + watchers: list[str] = field(default_factory=list) # aids qb64 of watchers + +@dataclass +class HabitatRecordV0_6_8: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + +def _check_if_needed(db): + """ + Check if the migration is needed + Parameters: + db(Baser): Baser database object on which to run the migration + Returns: + bool: True if the migration is needed, False otherwise + """ + habs = koming.Komer(db=db, subkey='habs.', schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'prefix' in habord: + return True + return False + +def migrate(db): + """Rename data in HabitatRecord from the old labels to the new labels as of 2022-10-17 + + This migration performs the following: + 1. rename prefix -> hid + 2. rename pid -> mid + 3. rename aids -> smids, rmids + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_7, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + existing = asdict(habord) + habord_0_6_7 = HabitatRecordV0_6_7(**existing) + habord_0_6_8 = HabitatRecordV0_6_8( + hid=habord_0_6_7.prefix, + mid=habord_0_6_7.pid, + smids=habord_0_6_7.aids, + rmids=habord_0_6_7.aids, + sid=None, + watchers=habord_0_6_7.watchers + ) + habords[habord_0_6_8.hid] = habord_0_6_8 + + habs.trim() # remove existing records + + # Add in the renamed records + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_8, ) + + for pre, habord in habords.items(): + habs.pin(keys=(pre,), val=habord) diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py index e21dfdb9f..6f1f54fc4 100644 --- a/src/keri/db/migrations/rekey_habs.py +++ b/src/keri/db/migrations/rekey_habs.py @@ -4,7 +4,7 @@ @dataclass -class OldHabitatRecord: # baser.habs +class HabitatRecordV0_6_8: # baser.habs """ Habitat application state information keyed by habitat name (baser.habs) @@ -24,6 +24,17 @@ class OldHabitatRecord: # baser.habs sid: str | None = None # Signify identifier qb64 when hid is Signify watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers +def _check_if_needed(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'domain' in habord: + return False + return True def migrate(db): """ Re-key habs migration for changing the key for .habs and introducing the .names database @@ -36,16 +47,21 @@ def migrate(db): Parameters: db(Baser): Baser database object on which to run the migration - """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + habs = koming.Komer(db=db, subkey='habs.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix nmsp = koming.Komer(db=db, subkey='nmsp.', - schema=OldHabitatRecord, ) + schema=HabitatRecordV0_6_8, ) habords = dict() # Update Hab records from .habs with name @@ -72,4 +88,4 @@ def migrate(db): for pre, habord in habords.items(): db.habs.pin(keys=(pre,), val=habord) ns = "" if habord.domain is None else habord.domain - db.names.pin(keys=(ns, habord.name), val=pre) + db.names.pin(keys=(ns, habord.name), val=pre) \ No newline at end of file