diff --git a/setup.py b/setup.py index 8dbbf6624..261634d5a 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,8 @@ 'mnemonic>=0.21', 'PrettyTable>=3.10.0', 'http_sfv>=0.9.9', - 'cryptography>=42.0.5' + 'cryptography>=42.0.5', + 'semver>=3.0.2' ], extras_require={ }, diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py deleted file mode 100644 index 775f957b9..000000000 --- a/src/keri/app/cli/commands/migrate.py +++ /dev/null @@ -1,163 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -KERI -keri.kli.commands module - -""" -import argparse - -from hio import help -from hio.base import doing - -from keri import kering -from keri.app.cli.common import existing -from keri.core import coring, serdering -from keri.db import koming, subing, dbing -from keri.db.basing import KeyStateRecord, StateEERecord -from keri.kering import ConfigurationError, Version -from keri.vdr import viring - -logger = help.ogler.getLogger() - -parser = argparse.ArgumentParser(description='View status of a local AID') -parser.set_defaults(handler=lambda args: handler(args), - transferable=True) -parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) -parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', - required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', - dest="bran", default=None) # passcode => bran -parser.add_argument('--force', action="store_true", required=False, - help='True means perform migration without prompting the user') - - -def handler(args): - if not args.force: - print() - print("This command will migrate your datastore to the next version of KERIpy and is not reversible.") - print("After this command, you will not be able to access your data store with this version.") - print() - yn = input("Are you sure you want to continue? [y|N]: ") - - if yn not in ("y", "Y"): - print("...exiting") - return [] - - kwa = dict(args=args) - return [doing.doify(migrate, **kwa)] - - -def migrate(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - base = args.base - bran = args.bran - - try: - with dbing.openLMDB(name=name, base=base, bran=bran, temp=False) as db: - print(db.path) - states = koming.Komer(db=db, - schema=dict, - subkey='stts.') - nstates = koming.Komer(db=db, - schema=KeyStateRecord, - subkey='stts.') - - for keys, sad in states.getItemIter(): - ksr = KeyStateRecord( - vn=Version, # version number as list [major, minor] - i=sad['i'], # qb64 prefix - s=sad['s'], # lowercase hex string no leading zeros - p=sad['p'], - d=sad['d'], - f=sad['f'], # lowercase hex string no leading zeros - dt=sad['dt'], - et=sad['et'], - kt=sad['kt'], - k=sad['k'], - nt=sad['nt'], - n=sad['n'], - bt=sad['bt'], - b=sad['b'], - c=sad['c'], - ee=StateEERecord._fromdict(sad['ee']), # latest est event dict - di=sad['di'] if sad['di'] else None - ) - - nstates.pin(keys=keys, val=ksr) - - with existing.existingHby(name=name, base=base, bran=bran) as hby: - rgy = viring.Reger(name=name, base=base, db=hby.db, temp=False, - reopen=True) - - rstates = koming.Komer(db=rgy, - schema=dict, - subkey='stts.') - - for _, sad in rstates.getItemIter(): - rsr = viring.RegStateRecord( - vn=list(Version), # version number as list [major, minor] - i=sad['i'], # qb64 registry SAID - s=sad['s'], # lowercase hex string no leading zeros - d=sad['d'], - ii=sad['ii'], - dt=sad['dt'], - et=sad['et'], - bt=sad['bt'], # hex string no leading zeros lowercase - b=sad['b'], # list of qb64 may be empty - c=sad['c'], - ) - # ksr = stateFromKever(kever) - rgy.states.pin(sad['i'], val=rsr) - - for (said,), _ in rgy.saved.getItemIter(): - snkey = dbing.snKey(said, 0) - dig = rgy.getTel(key=snkey) - - prefixer = coring.Prefixer(qb64=said) - seqner = coring.Seqner(sn=0) - saider = coring.Saider(qb64b=bytes(dig)) - rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) - - migrateKeys(hby.db) - - # clear escrows - print("clearing escrows") - hby.db.gpwe.trim() - hby.db.gdee.trim() - hby.db.dpwe.trim() - hby.db.gpse.trim() - hby.db.epse.trim() - hby.db.dune.trim() - - except ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - - -def migrateKeys(db): - # public keys mapped to the AID and event seq no they appeared in - pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in - digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - - for pre, fn, dig in db.getFelItemAllPreIter(key=b''): - dgkey = dbing.dgKey(pre, dig) # get message - if not (raw := db.getEvt(key=dgkey)): - print(f"Migrate keys: missing event for dig={dig}, skipped.") - continue - serder = serdering.SerderKERI(raw=bytes(raw)) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - verfers = serder.verfers or [] - for verfer in verfers: - pubs.add(keys=(verfer.qb64,), val=val) - ndigers = serder.ndigers or [] - for diger in ndigers: - digs.add(keys=(diger.qb64,), val=val) diff --git a/src/keri/app/cli/commands/migrate/__init__.py b/src/keri/app/cli/commands/migrate/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py new file mode 100644 index 000000000..0fac8de4a --- /dev/null +++ b/src/keri/app/cli/commands/migrate/list.py @@ -0,0 +1,64 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse +import logging + +from keri import help +from hio.base import doing +from prettytable import PrettyTable + +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + lister = ListDoer(args) + return [lister] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class ListDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(ListDoer, self).__init__() + + def recur(self, tyme): + tab = PrettyTable() + tab.field_names = ["Num", "Name", "Date Completed"] + tab.align["Name"] = "l" + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran if self.args.bran else None, temp=self.args.temp) + + for idx, (name, dater) in enumerate(hby.db.complete()): + print(name, dater) + date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" + tab.add_row((f"{idx + 1}", f"{name}", date)) + + print(tab) + return True diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py new file mode 100644 index 000000000..f57ac2570 --- /dev/null +++ b/src/keri/app/cli/commands/migrate/run.py @@ -0,0 +1,71 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse +import logging + +import keri +from keri import help +from hio.base import doing +from keri import kering + +from keri.app.cli.common import existing +from keri.db import basing +from keri.vdr import viring + +logger = help.ogler.getLogger("keri") + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = MigrateDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class MigrateDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(MigrateDoer, self).__init__() + + def recur(self, tyme): + name=self.args.name + base=self.args.base + temp=self.args.temp + hab_db = basing.Baser(name=name, + base=base, + temp=temp, + reopen=False) + + try: + hab_db.reopen() + except kering.DatabaseError as ex: + pass + + print(f"Migrating {name}...") + hab_db.migrate() + print(f"Finished migrating {name}") + + return True diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py new file mode 100644 index 000000000..1ff745815 --- /dev/null +++ b/src/keri/app/cli/commands/migrate/show.py @@ -0,0 +1,59 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio import help +from hio.base import doing + +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) +parser.add_argument('--migration', '-m', help='migration name', required=True) + + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + [(name, dater)] = hby.db.complete(name=self.args.migration) + date = dater.datetime.strftime("%Y-%m-%d %H:%M") if dater is not None else "Not Run" + + print(f"{self.args.migration} -> {date}") + + return True diff --git a/src/keri/app/cli/commands/version.py b/src/keri/app/cli/commands/version.py index 5ea6b775a..98cc81c5a 100644 --- a/src/keri/app/cli/commands/version.py +++ b/src/keri/app/cli/commands/version.py @@ -8,18 +8,35 @@ from hio.base import doing import keri +from keri.app.cli.common import existing parser = argparse.ArgumentParser(description='Print version of KLI') parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=False, + default=None) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran def handler(args): - return [doing.doify(version)] + kwa = dict(args=args) + return [doing.doify(version, **kwa)] -def version(tymth, tock=0.0): +def version(tymth, tock=0.0, **opts): """ Command line version handler """ _ = (yield tock) - print(keri.__version__) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + print(f"Library version: {keri.__version__}") + + if name is not None: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + print(f"Database version: {hby.db.version}") diff --git a/src/keri/app/cli/common/existing.py b/src/keri/app/cli/common/existing.py index 287d4e033..b2feba5e3 100644 --- a/src/keri/app/cli/common/existing.py +++ b/src/keri/app/cli/common/existing.py @@ -12,7 +12,7 @@ from keri.app import habbing, keeping -def setupHby(name, base="", bran=None, cf=None): +def setupHby(name, base="", bran=None, cf=None, temp=False): """ Create Habery off of existing directory Parameters: @@ -27,7 +27,7 @@ def setupHby(name, base="", bran=None, cf=None): """ ks = keeping.Keeper(name=name, base=base, - temp=False, + temp=temp, cf=cf, reopen=True) aeid = ks.gbls.get('aeid') diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0f808a8d4..ba2028a1c 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -18,7 +18,7 @@ So only need to set dupsort first time opened each other opening does not need to call it """ - +import importlib import os import shutil from collections import namedtuple @@ -30,10 +30,12 @@ import cbor2 as cbor import msgpack import lmdb +import semver from ordered_set import OrderedSet as oset from hio.base import doing +import keri from . import dbing, koming, subing from .. import kering @@ -45,6 +47,12 @@ logger = help.ogler.getLogger() +MIGRATIONS = [ + ("0.6.8", ["hab_data_rename"]), + ("1.0.0", ["add_key_and_reg_state_schemas"]), + ("1.2.0", ["rekey_habs"]) +] + class dbdict(dict): """ @@ -826,6 +834,10 @@ def reopen(self, **kwa): # events as ordered by first seen ordinals self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=coring.Seqner) + + self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) + self.vers = subing.Suber(db=self, subkey="vers.") + # Kever state made of KeyStateRecord key states self.states = koming.Komer(db=self, schema=KeyStateRecord, @@ -838,6 +850,9 @@ def reopen(self, **kwa): subkey='habs.', schema=HabitatRecord, ) + # habitat name database mapping (domain,name) as key to Prefixer + self.names = subing.Suber(db=self, subkey='names.', sep="^") + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix self.nmsp = koming.Komer(db=self, subkey='nmsp.', @@ -1060,6 +1075,10 @@ def reload(self): Reload stored prefixes and Kevers from .habs """ + # Check migrations to see if this database is up to date. Error otherwise + if not self.current: + raise kering.DatabaseError(f"Database migrations must be run. DB version {self.version}; current {keri.__version__}") + removes = [] for keys, data in self.habs.getItemIter(): if (ksr := self.states.get(keys=data.hid)) is not None: @@ -1099,6 +1118,99 @@ def reload(self): for keys in removes: # remove bare .habs records self.nmsp.rem(keys=keys) + def migrate(self, name, base, temp): + """ Run all migrations required + + Run all migrations that are required from the current version of database up to the current version + of the software that have not already been run. + + Sets the version of the database to the current version of the software after successful completion + of required migrations + + """ + for (version, migrations) in MIGRATIONS: + # Only run migration if current source code version is at or below the migration version + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(version, ver_no_prerelease) > 0: + print(f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") + continue + # Check to see if migration version is for an older database version + if self.version is not None and semver.compare(version, self.version) != 1: + continue + print(f"Migrating database v{self.version} --> v{version} ...") + + for migration in migrations: + modName = f"keri.db.migrations.{migration}" + if self.migs.get(keys=(migration,)) is not None: + continue + + mod = importlib.import_module(modName) + try: + mod.migrate(self) + except Exception as e: + print(f"\nAbandoning migration {migration} at version {version} with error: {e}") + return + + self.migs.pin(keys=(migration,), val=coring.Dater()) + self.version = version + + self.version = keri.__version__ + + @property + def current(self): + """ Current property determines if we are at the current database migration state. + + If the database version matches the library version return True + If the current database version is behind the current library version, check for migrations + - If there are migrations to run, return False + - If there are no migrations to run, reset database version to library version and return True + If the current database version is ahead of the current library version, raise exception + + """ + if self.version == keri.__version__: + return True + + # If database version is ahead of library version, throw exception + ver = semver.VersionInfo.parse(keri.__version__) + ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) + if self.version is not None and semver.compare(self.version, ver_no_prerelease) == 1: + raise kering.ConfigurationError( + f"Database version={self.version} is ahead of library version={keri.__version__}") + + last = MIGRATIONS[-1] + # If we aren't at latest version, but there are no outstanding migrations, + # reset version to latest (rightmost (-1) migration is latest) + if self.migs.get(keys=(last[1][-1],)) is not None: + return True + + # We have migrations to run + return False + + def complete(self, name=None): + """ Returns list of tuples of migrations completed with date of completion + + Parameters: + name(str): optional name of migration to check completeness + + Returns: + list: tuples of migration,date of completed migration names and the date of completion + + """ + migrations = [] + if not name: + for version, migs in MIGRATIONS: + # Only get migration completion dates for migrations that have been run + if self.version is not None and semver.compare(version, self.version) <= 0: + for mig in migs: + dater = self.migs.get(keys=(mig,)) + migrations.append((mig, dater)) + else: + if name not in MIGRATIONS or not self.migs.get(keys=(name,)): + raise ValueError(f"No migration named {name}") + migrations.append((name, self.migs.get(keys=(name,)))) + + return migrations def clean(self): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index d3831e214..bc8a9ac67 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -59,6 +59,7 @@ from hio.base import filing +import keri from ..help import helping ProemSize = 32 # does not include trailing separator @@ -345,6 +346,7 @@ def __init__(self, readonly=False, **kwa): """ self.env = None + self._version = None self.readonly = True if readonly else False super(LMDBer, self).__init__(**kwa) @@ -374,6 +376,7 @@ def reopen(self, readonly=False, **kwa): readonly (bool): True means open database in readonly mode False means open database in read/write mode """ + exists = self.exists(name=self.name, base=self.base) opened = super(LMDBer, self).reopen(**kwa) if readonly is not None: self.readonly = readonly @@ -382,10 +385,45 @@ def reopen(self, readonly=False, **kwa): # creates files data.mdb and lock.mdb in .dbDirPath self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=104857600, mode=self.perm, readonly=self.readonly) + self.opened = True if opened and self.env else False + + if self.opened and not self.readonly and (not exists or self.temp): + self.version = keri.__version__ + return self.opened + + @property + def version(self): + """ Return the version of database stored in __version__ key. + + This value is read through cached in memory + + Returns: + str: the version of the database or None if not set in the database + + """ + if self._version is None: + self._version = self.getVer() + + return self._version + + @version.setter + def version(self, val): + """ Set the version of the database in memory and in the __version__ key + + Parameters: + val (str): The new semver formatted version of the database + + """ + if hasattr(val, "decode"): + val = val.decode("utf-8") # convert bytes to str + + self._version = val + self.setVer(self._version) + def close(self, clear=False): """ Close lmdb at .env and if clear or .temp then remove lmdb directory at .path @@ -400,7 +438,33 @@ def close(self, clear=False): self.env = None - return(super(LMDBer, self).close(clear=clear)) + return (super(LMDBer, self).close(clear=clear)) + + def getVer(self): + """ Returns the value of the the semver formatted version in the __version__ key in this database + + Returns: + str: semver formatted version of the database + + """ + with self.env.begin() as txn: + cursor = txn.cursor() + version = cursor.get(b'__version__') + return version.decode("utf-8") if version is not None else None + + def setVer(self, val): + """ Set the version of the database in the __version__ key + + Parameters: + val (str): The new semver formatted version of the database + + """ + if hasattr(val, "encode"): + val = val.encode("utf-8") # convert str to bytes + + with self.env.begin(write=True) as txn: + cursor = txn.cursor() + cursor.replace(b'__version__', val) # For subdbs with no duplicate values allowed at each key. (dupsort==False) diff --git a/src/keri/db/migrations/__init__.py b/src/keri/db/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/keri/db/migrations/add_key_and_reg_state_schemas.py b/src/keri/db/migrations/add_key_and_reg_state_schemas.py new file mode 100644 index 000000000..c626bd2fb --- /dev/null +++ b/src/keri/db/migrations/add_key_and_reg_state_schemas.py @@ -0,0 +1,150 @@ +from keri import help +from keri.core import coring, serdering +from keri.db import koming, subing, dbing +from keri.db.basing import StateEERecord, KeyStateRecord +from keri.db.dbing import dgKey, splitKey +from keri.kering import ConfigurationError, Version +from keri.vdr import viring + +logger = help.ogler.getLogger() + +def _check_if_needed(db): + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + first = next(states.getItemIter(), None) + if first is None: + return False + keys, sad = first + if 'vn' in sad: + return False + return True + +def migrate(db): + """Adds schema for KeyStateRecord , RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., + and hby.db.digs. to be up to date as of 2022-??-?? + + This migration performs the following: + - hby.db -> "stts." schema from dict -> KeyStateRecord + - rgy -> "stts." schema from dict -> RegStateRecord + - rgy -> "cancs." reset to (ACDC SAID, SN 0, TEL evt 0 digest) + - hby.db -> "pubs." and + hby.db -> "digs." + that don't exist are populated with verification keys and event digests for the first seen events and + Keys: + "pubs." Verfer of each Verfer for each FEL event + "digs." Diger of next Diger (ndiger) of each FEL event + Value: (prefix, sn) of each event + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, already ran") + return + + try: + logger.debug(f"Migrating keystate and regstate dict to schema for {db.path}") + states = koming.Komer(db=db, + schema=dict, + subkey='stts.') + nstates = koming.Komer(db=db, + schema=KeyStateRecord, + subkey='stts.') + + for keys, sad in states.getItemIter(): + ksr = KeyStateRecord( + vn=Version, # version number as list [major, minor] + i=sad['i'], # qb64 prefix + s=sad['s'], # lowercase hex string no leading zeros + p=sad['p'], + d=sad['d'], + f=sad['f'], # lowercase hex string no leading zeros + dt=sad['dt'], + et=sad['et'], + kt=sad['kt'], + k=sad['k'], + nt=sad['nt'], + n=sad['n'], + bt=sad['bt'], + b=sad['b'], + c=sad['c'], + ee=StateEERecord._fromdict(sad['ee']), # latest est event dict + di=sad['di'] if sad['di'] else None + ) + + nstates.pin(keys=keys, val=ksr) + + rgy = viring.Reger(name=db.name, base=db.base, db=db, temp=db.temp, reopen=True) + + rstates = koming.Komer(db=rgy, + schema=dict, + subkey='stts.') + + for _, sad in rstates.getItemIter(): + rsr = viring.RegStateRecord( + vn=list(Version), # version number as list [major, minor] + i=sad['i'], # qb64 registry SAID + s=sad['s'], # lowercase hex string no leading zeros + d=sad['d'], + ii=sad['ii'], + dt=sad['dt'], + et=sad['et'], + bt=sad['bt'], # hex string no leading zeros lowercase + b=sad['b'], # list of qb64 may be empty + c=sad['c'], + ) + rgy.states.pin(sad['i'], val=rsr) + + for (said,), _ in rgy.saved.getItemIter(): + snkey = dbing.snKey(said, 0) + dig = rgy.getTel(key=snkey) + + prefixer = coring.Prefixer(qb64=said) + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64b=bytes(dig)) + rgy.cancs.pin(keys=said, val=[prefixer, seqner, saider]) + + migrateKeys(db) + + # clear escrows + logger.info("clearing escrows") + db.gpwe.trim() + db.gdee.trim() + db.dpwe.trim() + db.gpse.trim() + db.epse.trim() + db.dune.trim() + for ekey, edig in db.getQnfItemsNextIter(): + pre, _ = splitKey(ekey) + db.delQnf(dgKey(pre, edig), edig) + + except ConfigurationError: + logger.error(f"identifier prefix for {db.name} does not exist, incept must be run first", ) + return -1 + + +def migrateKeys(db): + # public keys mapped to the AID and event seq no they appeared in + pubs = subing.CatCesrIoSetSuber(db=db, subkey="pubs.", + klas=(coring.Prefixer, coring.Seqner)) + + # next key digests mapped to the AID and event seq no they appeared in + digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", + klas=(coring.Prefixer, coring.Seqner)) + + for pre, fn, dig in db.getFelItemAllPreIter(key=b''): + dgkey = dbing.dgKey(pre, dig) # get message + if not (raw := db.getEvt(key=dgkey)): + logger.info(f"Migrate keys: missing event for dig={dig}, skipped.") + continue + serder = serdering.SerderKERI(raw=bytes(raw)) + val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) + verfers = serder.verfers or [] + for verfer in verfers: + pubs.add(keys=(verfer.qb64,), val=val) + ndigers = serder.ndigers or [] + for diger in ndigers: + digs.add(keys=(diger.qb64,), val=val) diff --git a/src/keri/db/migrations/hab_data_rename.py b/src/keri/db/migrations/hab_data_rename.py new file mode 100644 index 000000000..6316b912f --- /dev/null +++ b/src/keri/db/migrations/hab_data_rename.py @@ -0,0 +1,87 @@ +from dataclasses import dataclass, field, asdict +from typing import Optional + +from keri.db import koming, basing +from keri.db.basing import HabitatRecord, Baser +from keri.vdr.viring import Reger + + +@dataclass +class HabitatRecordV0_6_7: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + prefix (str): identifier prefix of hab qb64 + pid (str | None): group member identifier qb64 when hid is group + aids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + """ + prefix: str # aid qb64 + pid: Optional[str] # participant aid of group aid + aids: Optional[list] # all identifiers participating in the group identity + + watchers: list[str] = field(default_factory=list) # aids qb64 of watchers + +def _check_if_needed(db): + """ + Check if the migration is needed + + Parameters: + db(Baser): Baser database object on which to run the migration + + Returns: + bool: True if the migration is needed, False otherwise + """ + habs = koming.Komer(db=db, subkey='habs.', schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'prefix' in habord: + return True + return False + +def migrate(db): + """Rename data in HabitatRecord from the old labels to the new labels as of 2022-10-17 + + This migration performs the following: + 1. rename prefix -> hid + 2. rename pid -> mid + 3. rename aids -> smids, rmids + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, already ran") + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_7, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + existing = asdict(habord) + habord_0_6_7 = HabitatRecordV0_6_7(**existing) + habord_0_6_8 = HabitatRecord( + hid=habord_0_6_7.prefix, + mid=habord_0_6_7.pid, + smids=habord_0_6_7.aids, + rmids=habord_0_6_7.aids, + sid=None, + watchers=habord_0_6_7.watchers + ) + habords[habord_0_6_8.hid] = habord_0_6_8 + + habs.trim() # remove existing records + + # Add in the renamed records + for pre, habord in habords.items(): + db.habs.pin(keys=(pre,), val=habord) + + diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py new file mode 100644 index 000000000..def6ca585 --- /dev/null +++ b/src/keri/db/migrations/rekey_habs.py @@ -0,0 +1,91 @@ +from dataclasses import dataclass, field, asdict + +from keri.db import koming, basing + + +@dataclass +class OldHabitatRecord: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + +def _check_if_needed(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'domain' in habord: + return False + return True + +def migrate(db): + """ Re-key habs migration for changing the key for .habs and introducing the .names database + + This migrations performs the following: + 1. Rekey .habs from name (alias) to the AID of the Hab + 2. Add Name and domain to the HabitatRecord for all Habs + 3. Populate the .names index as (ns, name) -> AID + 4. Remove the .nmsp namespaced Habs database (replaced within .habs and .names now) + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, already ran") + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=OldHabitatRecord, ) + + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix + nmsp = koming.Komer(db=db, + subkey='nmsp.', + schema=OldHabitatRecord, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + name = ".".join(name) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + habords[habord.hid] = nhabord + + habs.trim() + + # Update Hab records from .nmsp with name and domain (ns) + for keys, habord in nmsp.getItemIter(): + ns = keys[0] + name = ".".join(keys[1:]) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + nhabord.domain = ns + habords[habord.hid] = nhabord + + nmsp.trim() # remove existing records + + # Rekey .habs and create .names index + for pre, habord in habords.items(): + db.habs.pin(keys=(pre,), val=habord) + ns = "" if habord.domain is None else habord.domain + db.names.pin(keys=(ns, habord.name), val=pre)