From ed9d02225a43efa2cc89a57e86bdcbb731a164fe Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Wed, 23 Jan 2019 13:32:48 +0100 Subject: [PATCH 01/30] Validate that infered_from adn depends_on are not strings but sequences --- qcodes/dataset/param_spec.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qcodes/dataset/param_spec.py b/qcodes/dataset/param_spec.py index beb7183ec66..f425fc744a7 100644 --- a/qcodes/dataset/param_spec.py +++ b/qcodes/dataset/param_spec.py @@ -43,10 +43,18 @@ def __init__(self, name: str, inferred_from = [] if inferred_from is None else inferred_from depends_on = [] if depends_on is None else depends_on + if isinstance(inferred_from, str): + raise ValueError(f"ParamSpec {self.name} got " + f"string {inferred_from} it needs a " + f"Sequence of ParamSpecs or strings") self._inferred_from.extend( p.name if isinstance(p, ParamSpec) else p for p in inferred_from) + if isinstance(depends_on, str): + raise ValueError(f"ParamSpec {self.name} got " + f"string {depends_on} it needs a " + f"Sequence of ParamSpecs or strings") self._depends_on.extend( p.name if isinstance(p, ParamSpec) else p for p in depends_on) From 234843a9633f716632ef862836f1bc39ad4f506b Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Wed, 23 Jan 2019 13:48:17 +0100 Subject: [PATCH 02/30] Correct upgrade path --- qcodes/dataset/sqlite_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index f62bdac439a..c6e743cbe45 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -617,6 +617,8 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, # first possibility: another parameter depends on this parameter if layout_id in indeps: + inferred_from = inferred_from.split(', ') + paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, inferred_from=inferred_from) @@ -627,6 +629,7 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, setpoints = dependencies[layout_id] depends_on = [paramspecs[idp].name for idp in setpoints] + inferred_from = inferred_from.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, From c5108f5a40513d14e1c7d0b39a46b2f604ba5ea5 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Wed, 23 Jan 2019 14:34:34 +0100 Subject: [PATCH 03/30] Mypy fix --- qcodes/dataset/sqlite_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index c6e743cbe45..e317d93ab7c 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -617,11 +617,11 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, # first possibility: another parameter depends on this parameter if layout_id in indeps: - inferred_from = inferred_from.split(', ') + inferred_from_list = inferred_from.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, - inferred_from=inferred_from) + inferred_from=inferred_from_list) paramspecs[layout_id] = paramspec # second possibility: this parameter depends on another parameter @@ -629,13 +629,13 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, setpoints = dependencies[layout_id] depends_on = [paramspecs[idp].name for idp in setpoints] - inferred_from = inferred_from.split(', ') + inferred_from_list = inferred_from.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, depends_on=depends_on, - inferred_from=inferred_from) + inferred_from=inferred_from_list) paramspecs[layout_id] = paramspec # third possibility: no dependencies From 3d230ae636099718b45a006dfbacbaf722ca0e69 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Fri, 25 Jan 2019 15:52:58 +0100 Subject: [PATCH 04/30] Add another db to version 3 generation --- .../generate_version_3.py | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py index 7d67cd50628..f080819c647 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py +++ b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py @@ -29,6 +29,63 @@ def generate_empty_DB_file(): sqlite_base.connect(path) +def generate_DB_file_with_some_runs(): + """ + Generate a .db-file with a handful of runs with some interdependent + parameters + """ + + # This function will run often on CI and re-generate the .db-files + # That should ideally be a deterministic action + # (although this hopefully plays no role) + np.random.seed(0) + + v3fixturepath = os.path.join(fixturepath, 'version3') + os.makedirs(v3fixturepath, exist_ok=True) + path = os.path.join(v3fixturepath, 'some_runs.db') + + if os.path.exists(path): + os.remove(path) + + from qcodes.dataset.sqlite_base import connect + from qcodes.dataset.measurements import Measurement + from qcodes.dataset.experiment_container import Experiment + from qcodes import Parameter + + connect(path) + exp = Experiment(path_to_db=path, + name='experiment_1', + sample_name='no_sample_1') + + # Now make some parameters to use in measurements + params = [] + for n in range(5): + params.append(Parameter(f'p{n}', label=f'Parameter {n}', + unit=f'unit {n}', set_cmd=None, get_cmd=None)) + + # Set up an experiment + + meas = Measurement(exp) + meas.register_parameter(params[0]) + meas.register_parameter(params[1]) + meas.register_parameter(params[2], basis=(params[0],)) + meas.register_parameter(params[3], basis=(params[1],)) + meas.register_parameter(params[4], setpoints=(params[2], params[3])) + + # Make a number of identical runs + + for _ in range(10): + + with meas.run() as datasaver: + + for x in np.random.rand(10): + for y in np.random.rand(10): + z = np.random.rand() + datasaver.add_result((params[2], x), + (params[3], y), + (params[4], z)) + + def generate_DB_file_with_some_runs_having_not_run_descriptions(): """ Generate a .db-file with a handful of runs some of which lack run @@ -133,7 +190,8 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): if __name__ == '__main__': gens = (generate_empty_DB_file, - generate_DB_file_with_some_runs_having_not_run_descriptions) + generate_DB_file_with_some_runs_having_not_run_descriptions, + generate_DB_file_with_some_runs) # pylint: disable=E1101 utils.checkout_to_old_version_and_run_generators(version=3, gens=gens) From 37d89f093e6498389454fc536601c5d4c5fefe66 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 26 Jan 2019 18:07:57 +0100 Subject: [PATCH 05/30] Add logic to generate v3 db from upgraded v2 --- .../legacy_DB_generation/generate_version_3.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py index f080819c647..76822e67cc0 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py +++ b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py @@ -1,6 +1,7 @@ # Generate version 3 database files for qcodes' test suite to consume import os +import shutil import numpy as np # NB: it's important that we do not import anything from qcodes before we @@ -187,11 +188,26 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): conn.commit() # just to be sure +def generate_upgraded_v2_runs(): + """ + Generate some runs by upgradeing from v2 db. This + is needed since the bug we want to test against is in + the v2 to v3 upgrade and not in v3 it self. + This requires the v2 generation to be run before this one + """ + import qcodes.dataset.sqlite_base as sqlite_base + v2fixture_path = os.path.join(fixturepath, 'version2', 'some_runs.db') + v3fixturepath = os.path.join(fixturepath, 'version3', 'some_runs_upgraded_2.db') + shutil.copy2(v2fixture_path, v3fixturepath) + sqlite_base.connect(v3fixturepath) + + if __name__ == '__main__': gens = (generate_empty_DB_file, generate_DB_file_with_some_runs_having_not_run_descriptions, - generate_DB_file_with_some_runs) + generate_DB_file_with_some_runs, + generate_upgraded_v2_runs) # pylint: disable=E1101 utils.checkout_to_old_version_and_run_generators(version=3, gens=gens) From e382dffac4441de7d344311b171252eae244965c Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Fri, 25 Jan 2019 15:31:17 +0100 Subject: [PATCH 06/30] Actually test all parameters in db upgrade --- .../test_database_creation_and_upgrading.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index 5aac5f7d7af..5844c101db5 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -263,6 +263,24 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): assert p0.label == "Parameter 0" assert p0.unit == "unit 0" + p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] + assert p1.depends_on == '' + assert p1.inferred_from == '' + assert p1.label == "Parameter 1" + assert p1.unit == "unit 1" + + p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] + assert p2.depends_on == '' + assert p2.inferred_from == 'p0' + assert p2.label == "Parameter 2" + assert p2.unit == "unit 2" + + p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] + assert p3.depends_on == '' + assert p3.inferred_from == 'p1' + assert p3.label == "Parameter 3" + assert p3.unit == "unit 3" + p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] assert p4.depends_on == 'p2, p3' assert p4.inferred_from == '' From a18eab6ff820843ee8eea36ada915b24d89f67e8 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 26 Jan 2019 18:24:15 +0100 Subject: [PATCH 07/30] correct v3 hash --- qcodes/tests/dataset/legacy_DB_generation/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/utils.py b/qcodes/tests/dataset/legacy_DB_generation/utils.py index ba01ffa04ad..d5d5afac1d4 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/utils.py +++ b/qcodes/tests/dataset/legacy_DB_generation/utils.py @@ -24,10 +24,7 @@ GIT_HASHES: Dict[int, str] = {0: '78d42620fc245a975b5a615ed5e33061baac7846', 1: '056d59627e22fa3ca7aad4c265e9897c343f79cf', 2: '5202255924542dad6841dfe3d941a7f80c43956c', - # NOTE: commit hash associated with version 3 - # will need to be updated once we move to - # version 4 - 3: 'a5d8edae05898df0cbfb5d9048565df91efea692'} + 3: '01cb405d47b72b1a5bab227dcf4b69866ecebb0f'} gitrepopath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-5]) repo = Repo(gitrepopath) From 90132bafa257f97309003aaa20c4813d117d77cf Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 26 Jan 2019 18:39:17 +0100 Subject: [PATCH 08/30] add test that performing v3 to v4 Make sure that it fixes the issue observed in v3 --- .../test_database_creation_and_upgrading.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index 5844c101db5..b51ba291b34 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -28,6 +28,7 @@ perform_db_upgrade_0_to_1, perform_db_upgrade_1_to_2, perform_db_upgrade_2_to_3, + perform_db_upgrade_3_to_4, _latest_available_version) from qcodes.dataset.guids import parse_guid @@ -288,6 +289,103 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): assert p4.unit == "unit 4" +def test_perfrom_upgrade_v3_to_v4_fixes(): + + v3fixpath = os.path.join(fixturepath, 'db_files', 'version3') + + dbname_old = os.path.join(v3fixpath, 'some_runs_upgraded_2.db') + + if not os.path.exists(dbname_old): + pytest.skip("No db-file fixtures found. You can generate test db-files" + " using the scripts in the legacy_DB_generation folder") + + with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn: + + assert get_user_version(conn) == 3 + + sql = f""" + SELECT run_description + FROM runs + WHERE run_id == 1 + """ + c = atomic_transaction(conn, sql) + json_str = one(c, 'run_description') + + desc = RunDescriber.from_json(json_str) + idp = desc.interdeps + assert isinstance(idp, InterDependencies) + + p0 = [p for p in idp.paramspecs if p.name == 'p0'][0] + assert p0.depends_on == '' + assert p0.inferred_from == '' + assert p0.label == "Parameter 0" + assert p0.unit == "unit 0" + + p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] + assert p1.depends_on == '' + assert p1.inferred_from == '' + assert p1.label == "Parameter 1" + assert p1.unit == "unit 1" + + p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] + assert p2.depends_on == '' + assert p2.inferred_from == 'p, 0' + assert p2.label == "Parameter 2" + assert p2.unit == "unit 2" + + p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] + assert p3.depends_on == '' + assert p3.inferred_from == 'p, 1' + assert p3.label == "Parameter 3" + assert p3.unit == "unit 3" + + p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] + assert p4.depends_on == 'p2, p3' + assert p4.inferred_from == '' + assert p4.label == "Parameter 4" + assert p4.unit == "unit 4" + + perform_db_upgrade_3_to_4(conn) + + c = atomic_transaction(conn, sql) + json_str = one(c, 'run_description') + + desc = RunDescriber.from_json(json_str) + idp = desc.interdeps + assert isinstance(idp, InterDependencies) + + p0 = [p for p in idp.paramspecs if p.name == 'p0'][0] + assert p0.depends_on == '' + assert p0.inferred_from == '' + assert p0.label == "Parameter 0" + assert p0.unit == "unit 0" + + p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] + assert p1.depends_on == '' + assert p1.inferred_from == '' + assert p1.label == "Parameter 1" + assert p1.unit == "unit 1" + + p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] + assert p2.depends_on == '' + assert p2.inferred_from == 'p0' + assert p2.label == "Parameter 2" + assert p2.unit == "unit 2" + + p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] + assert p3.depends_on == '' + assert p3.inferred_from == 'p1' + assert p3.label == "Parameter 3" + assert p3.unit == "unit 3" + + p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] + assert p4.depends_on == 'p2, p3' + assert p4.inferred_from == '' + assert p4.label == "Parameter 4" + assert p4.unit == "unit 4" + + + @pytest.mark.usefixtures("empty_temp_db") def test_update_existing_guids(caplog): From 0ee04630647a5300f207b877fac5fb14675fa831 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 26 Jan 2019 18:42:11 +0100 Subject: [PATCH 09/30] Add stupid version of v3-v4 upgrade This should either be refactored to reduce duplication or simplified to only generate new desc when needed --- qcodes/dataset/sqlite_base.py | 68 +++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index e317d93ab7c..42017f6e42b 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -723,6 +723,74 @@ def perform_db_upgrade_2_to_3(conn: ConnectionPlus) -> None: log.debug(f"Upgrade in transition, run number {run_id}: OK") +@upgrader +def perform_db_upgrade_3_to_4(conn: ConnectionPlus) -> None: + """ + Perform the upgrade from version 3 to version 4. This really + repeats the version 3 upgrade as it originally had a bug in + the inferred annotation. This has since been fixes so rerun + the upgrade. + """ + + no_of_runs_query = "SELECT max(run_id) FROM runs" + no_of_runs = one(atomic_transaction(conn, no_of_runs_query), 'max(run_id)') + no_of_runs = no_of_runs or 0 + + # If one run fails, we want the whole upgrade to roll back, hence the + # entire upgrade is one atomic transaction + + with atomic(conn) as conn: + + result_tables = _2to3_get_result_tables(conn) + layout_ids_all = _2to3_get_layout_ids(conn) + indeps_all = _2to3_get_indeps(conn) + deps_all = _2to3_get_deps(conn) + layouts = _2to3_get_layouts(conn) + dependencies = _2to3_get_dependencies(conn) + + pbar = tqdm(range(1, no_of_runs+1)) + pbar.set_description("Upgrading database") + + for run_id in pbar: + + if run_id in layout_ids_all: + + result_table_name = result_tables[run_id] + layout_ids = list(layout_ids_all[run_id]) + if run_id in indeps_all: + independents = tuple(indeps_all[run_id]) + else: + independents = () + if run_id in deps_all: + dependents = tuple(deps_all[run_id]) + else: + dependents = () + + paramspecs = _2to3_get_paramspecs(conn, + layout_ids, + layouts, + dependencies, + dependents, + independents, + result_table_name) + + interdeps = InterDependencies(*paramspecs.values()) + desc = RunDescriber(interdeps=interdeps) + json_str = desc.to_json() + + else: + + json_str = RunDescriber(InterDependencies()).to_json() + + sql = f""" + UPDATE runs + SET run_description = ? + WHERE run_id == ? + """ + cur = conn.cursor() + cur.execute(sql, (json_str, run_id)) + log.debug(f"Upgrade in transition, run number {run_id}: OK") + def _latest_available_version() -> int: """Return latest available database schema version""" return len(_UPGRADE_ACTIONS) From f68c853add830be6fbfe952a7b1876a24b13e6a3 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 26 Jan 2019 19:11:11 +0100 Subject: [PATCH 10/30] adapt tests to new latest version --- qcodes/tests/dataset/test_database_creation_and_upgrading.py | 2 +- qcodes/tests/dataset/test_database_extract_runs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index b51ba291b34..8b05880ed0d 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -487,7 +487,7 @@ def test_cannot_connect_to_newer_db(): def test_latest_available_version(): - assert 3 == _latest_available_version() + assert 4 == _latest_available_version() @pytest.mark.parametrize('version', VERSIONS) diff --git a/qcodes/tests/dataset/test_database_extract_runs.py b/qcodes/tests/dataset/test_database_extract_runs.py index e0131649674..7977cf3cc7f 100644 --- a/qcodes/tests/dataset/test_database_extract_runs.py +++ b/qcodes/tests/dataset/test_database_extract_runs.py @@ -443,7 +443,7 @@ def test_old_versions_not_touched(two_empty_temp_db_connections, with pytest.warns(UserWarning) as warning: extract_runs_into_db(fixturepath, target_path, 1) expected_mssg = ('Source DB version is 2, but this ' - 'function needs it to be in version 3. ' + 'function needs it to be in version 4. ' 'Run this function again with ' 'upgrade_source_db=True to auto-upgrade ' 'the source DB file.') @@ -465,7 +465,7 @@ def test_old_versions_not_touched(two_empty_temp_db_connections, with pytest.warns(UserWarning) as warning: extract_runs_into_db(source_path, fixturepath, 1) expected_mssg = ('Target DB version is 2, but this ' - 'function needs it to be in version 3. ' + 'function needs it to be in version 4. ' 'Run this function again with ' 'upgrade_target_db=True to auto-upgrade ' 'the target DB file.') From c719e85c5d4c98a6c8349d6ca34bb2c904bb372d Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 11:20:29 +0100 Subject: [PATCH 11/30] Use test generators from other repo --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4a467ab86f5..84ba10709c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,11 +41,14 @@ script: - | pip uninstall -y qcodes pip install -e . - cd qcodes/tests/dataset/legacy_DB_generation + cd .. + git clone https://github.com/QCoDeS/qcodes_generate_test_db.git + cd qcodes_generate_test_db python generate_version_0.py python generate_version_1.py python generate_version_2.py python generate_version_3.py + cd $TRAVIS_BUILD_DIR cd ../../../.. pip uninstall -y qcodes pip install . From 70511e243c7b20bbe4af2299f48cba58e4525862 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 11:24:40 +0100 Subject: [PATCH 12/30] Correct path --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 84ba10709c6..00122bee005 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,6 @@ script: python generate_version_2.py python generate_version_3.py cd $TRAVIS_BUILD_DIR - cd ../../../.. pip uninstall -y qcodes pip install . - cd qcodes From 3fa7e6cfebe47c6c46a633a4c65594a840427ee8 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 12:25:38 +0100 Subject: [PATCH 13/30] Update azure build --- azure-pipelines.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 32969c49a28..ba15fad080d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,11 +29,13 @@ jobs: mypy qcodes displayName: "mypy" - script: | - cd qcodes - python tests\dataset\legacy_DB_generation\generate_version_0.py - python tests\dataset\legacy_DB_generation\generate_version_1.py - python tests\dataset\legacy_DB_generation\generate_version_2.py - python tests\dataset\legacy_DB_generation\generate_version_3.py + cd .. + git clone https://github.com/QCoDeS/qcodes_generate_test_db.git + cd qcodes_generate_test_db + python generate_version_0.py + python generate_version_1.py + python generate_version_2.py + python generate_version_3.py displayName: "Generate db fixtures" - script: | cd qcodes From 4abcd6e999b55ea5f363e665a3dc8600ccd67446 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 12:41:42 +0100 Subject: [PATCH 14/30] legacy generation has been moved to it's own repo --- .../dataset/legacy_DB_generation/README.md | 26 --- .../dataset/legacy_DB_generation/__init__.py | 0 .../generate_version_0.py | 37 --- .../generate_version_1.py | 36 --- .../generate_version_2.py | 146 ------------ .../generate_version_3.py | 213 ------------------ .../dataset/legacy_DB_generation/utils.py | 85 ------- 7 files changed, 543 deletions(-) delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/README.md delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/__init__.py delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/generate_version_0.py delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/generate_version_1.py delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/generate_version_2.py delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py delete mode 100644 qcodes/tests/dataset/legacy_DB_generation/utils.py diff --git a/qcodes/tests/dataset/legacy_DB_generation/README.md b/qcodes/tests/dataset/legacy_DB_generation/README.md deleted file mode 100644 index 1f34eed61cb..00000000000 --- a/qcodes/tests/dataset/legacy_DB_generation/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Legacy database file generation - -## Why have old database files? - -When the database schema (the database user version number) changes, the API in the core QCoDeS repository also changes to produce output matching the new version. That, in turn, makes the official API incapable of generating databases of older versions. To the end of testing the upgrade functions, we would like to be able to generate old version databases. The purpose of the `legacy_DB_generation` folder is contain scripts that generate old-version databases for the tests to consume. - -When a new corner-case that we want to test for is discovered, we extend the scripts to produce a .db-file covering that case. When a new version upgrade is launched, we make a new script to generate .db-files for that version upgrade to test itself on. - -## How to run the scripts? - -The scripts use `gitpython` to roll QCoDeS back to a relevant old commit and create database files with the ancient source code. - -The scripts should *not* be run as a part of the QCoDeS test suite, but prior to test execution in a different process. The scripts have some dependencies and will run in the normal QCoDeS environment **PROVIDED** that QCoDeS was installed with the editable flag (i.e. `pip install -e `). - -## How do I write my own script? - -First, please check if there is already a script producing .db-files of your desired version. If so, simply extend that script to produce a .db-file covering your particular test case. If not, follow this checklist: - - * Identify the database version of which you would like to create one or more .db-files. Let's call that "your version". - * Check the variable `GIT_HASHES` in `utils.py` to see if "your version" already has a recorded commit hash. - * If not, search through the `git log` of `master` to find the merge commit *just* before the merge commit that introduces the *next* version after "your version". Put that first commit into `GIT_HASHES` along with the version number of "your version". - * Make a script called `generate_version_.py`. Copy the general structure of `generate_version_0.py`. Make your generating functions take *ZERO* arguments and do all their imports inside their own scope. - -## Anything else? - -Remember to update the tests to use your newly generated fixtures. A test must **skip** (not fail) if the fixture is not present on disk. Also make sure that the CI runs your fixture-generating script. diff --git a/qcodes/tests/dataset/legacy_DB_generation/__init__.py b/qcodes/tests/dataset/legacy_DB_generation/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_0.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_0.py deleted file mode 100644 index 7692ef15336..00000000000 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_0.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generate version 0 database files for qcodes' test suite to consume - -import os - -# NB: it's important that we do not import anything from qcodes before we -# do the git magic (which we do below), hence the relative import here -import utils as utils - - -fixturepath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-2]) -fixturepath = os.path.join(fixturepath, 'fixtures', 'db_files') - - -def generate_empty_DB_file(): - """ - Generate the bare minimal DB file with no runs - """ - - import qcodes.dataset.sqlite_base as sqlite_base - - v0fixturepath = os.path.join(fixturepath, 'version0') - os.makedirs(v0fixturepath, exist_ok=True) - path = os.path.join(v0fixturepath, 'empty.db') - - if os.path.exists(path): - os.remove(path) - - conn = sqlite_base.connect(path) - sqlite_base.init_db(conn) - - -if __name__ == '__main__': - - gens = (generate_empty_DB_file,) - - # pylint: disable=E1101 - utils.checkout_to_old_version_and_run_generators(version=0, gens=gens) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_1.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_1.py deleted file mode 100644 index 1dd6a6d9d6c..00000000000 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_1.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generate version 1 database files for qcodes' test suite to consume - -import os - -# NB: it's important that we do not import anything from qcodes before we -# do the git magic (which we do below), hence the relative import here -import utils as utils - - -fixturepath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-2]) -fixturepath = os.path.join(fixturepath, 'fixtures', 'db_files') - - -def generate_empty_DB_file(): - """ - Generate the bare minimal DB file with no runs - """ - - import qcodes.dataset.sqlite_base as sqlite_base - - v0fixturepath = os.path.join(fixturepath, 'version1') - os.makedirs(v0fixturepath, exist_ok=True) - path = os.path.join(v0fixturepath, 'empty.db') - - if os.path.exists(path): - os.remove(path) - - sqlite_base.connect(path) - - -if __name__ == '__main__': - - gens = (generate_empty_DB_file,) - - # pylint: disable=E1101 - utils.checkout_to_old_version_and_run_generators(version=1, gens=gens) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_2.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_2.py deleted file mode 100644 index ac68dfadbed..00000000000 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_2.py +++ /dev/null @@ -1,146 +0,0 @@ -# Generate version 2 database files for qcodes' test suite to consume - -import os -import numpy as np - -# NB: it's important that we do not import anything from qcodes before we -# do the git magic (which we do below), hence the relative import here -import utils as utils - - -fixturepath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-2]) -fixturepath = os.path.join(fixturepath, 'fixtures', 'db_files') - - -def generate_empty_DB_file(): - """ - Generate the bare minimal DB file with no runs - """ - - import qcodes.dataset.sqlite_base as sqlite_base - - v2fixturepath = os.path.join(fixturepath, 'version2') - os.makedirs(v2fixturepath, exist_ok=True) - path = os.path.join(v2fixturepath, 'empty.db') - - if os.path.exists(path): - os.remove(path) - - sqlite_base.connect(path) - - -def generate_DB_file_with_some_runs(): - """ - Generate a .db-file with a handful of runs with some interdependent - parameters - """ - - # This function will run often on CI and re-generate the .db-files - # That should ideally be a deterministic action - # (although this hopefully plays no role) - np.random.seed(0) - - v2fixturepath = os.path.join(fixturepath, 'version2') - os.makedirs(v2fixturepath, exist_ok=True) - path = os.path.join(v2fixturepath, 'some_runs.db') - - if os.path.exists(path): - os.remove(path) - - from qcodes.dataset.sqlite_base import connect - from qcodes.dataset.measurements import Measurement - from qcodes.dataset.experiment_container import Experiment - from qcodes import Parameter - - connect(path) - exp = Experiment(path) - exp._new(name='experiment_1', sample_name='no_sample_1') - - # Now make some parameters to use in measurements - params = [] - for n in range(5): - params.append(Parameter(f'p{n}', label=f'Parameter {n}', - unit=f'unit {n}', set_cmd=None, get_cmd=None)) - - # Set up an experiment - - meas = Measurement(exp) - meas.register_parameter(params[0]) - meas.register_parameter(params[1]) - meas.register_parameter(params[2], basis=(params[0],)) - meas.register_parameter(params[3], basis=(params[1],)) - meas.register_parameter(params[4], setpoints=(params[2], params[3])) - - # Make a number of identical runs - - for _ in range(10): - - with meas.run() as datasaver: - - for x in np.random.rand(10): - for y in np.random.rand(10): - z = np.random.rand() - datasaver.add_result((params[2], x), - (params[3], y), - (params[4], z)) - - -def generate_DB_file_with_empty_runs(): - """ - Generate a DB file that holds empty runs and runs with no interdependencies - """ - - v2fixturepath = os.path.join(fixturepath, 'version2') - os.makedirs(v2fixturepath, exist_ok=True) - path = os.path.join(v2fixturepath, 'empty_runs.db') - - if os.path.exists(path): - os.remove(path) - - from qcodes.dataset.sqlite_base import connect - from qcodes.dataset.measurements import Measurement - from qcodes.dataset.experiment_container import Experiment - from qcodes import Parameter - from qcodes.dataset.data_set import DataSet - - conn = connect(path) - exp = Experiment(path) - exp._new(name='experiment_1', sample_name='no_sample_1') - - # Now make some parameters to use in measurements - params = [] - for n in range(5): - params.append(Parameter(f'p{n}', label=f'Parameter {n}', - unit=f'unit {n}', set_cmd=None, get_cmd=None)) - - # truly empty run, no layouts table, no nothing - dataset = DataSet(path, conn=conn) - dataset._new('empty_dataset', exp_id=exp.exp_id) - - # empty run - meas = Measurement(exp) - with meas.run() as datasaver: - pass - - # run with no interdeps - meas = Measurement(exp) - for param in params: - meas.register_parameter(param) - - with meas.run() as datasaver: - pass - - with meas.run() as datasaver: - for _ in range(10): - res = tuple((p, 0.0) for p in params) - datasaver.add_result(*res) - - -if __name__ == '__main__': - - gens = (generate_empty_DB_file, - generate_DB_file_with_some_runs, - generate_DB_file_with_empty_runs) - - # pylint: disable=E1101 - utils.checkout_to_old_version_and_run_generators(version=2, gens=gens) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py deleted file mode 100644 index 76822e67cc0..00000000000 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py +++ /dev/null @@ -1,213 +0,0 @@ -# Generate version 3 database files for qcodes' test suite to consume - -import os -import shutil -import numpy as np - -# NB: it's important that we do not import anything from qcodes before we -# do the git magic (which we do below), hence the relative import here -import utils as utils - - -fixturepath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-2]) -fixturepath = os.path.join(fixturepath, 'fixtures', 'db_files') - - -def generate_empty_DB_file(): - """ - Generate the bare minimal DB file with no runs - """ - - import qcodes.dataset.sqlite_base as sqlite_base - - v3fixturepath = os.path.join(fixturepath, 'version3') - os.makedirs(v3fixturepath, exist_ok=True) - path = os.path.join(v3fixturepath, 'empty.db') - - if os.path.exists(path): - os.remove(path) - - sqlite_base.connect(path) - - -def generate_DB_file_with_some_runs(): - """ - Generate a .db-file with a handful of runs with some interdependent - parameters - """ - - # This function will run often on CI and re-generate the .db-files - # That should ideally be a deterministic action - # (although this hopefully plays no role) - np.random.seed(0) - - v3fixturepath = os.path.join(fixturepath, 'version3') - os.makedirs(v3fixturepath, exist_ok=True) - path = os.path.join(v3fixturepath, 'some_runs.db') - - if os.path.exists(path): - os.remove(path) - - from qcodes.dataset.sqlite_base import connect - from qcodes.dataset.measurements import Measurement - from qcodes.dataset.experiment_container import Experiment - from qcodes import Parameter - - connect(path) - exp = Experiment(path_to_db=path, - name='experiment_1', - sample_name='no_sample_1') - - # Now make some parameters to use in measurements - params = [] - for n in range(5): - params.append(Parameter(f'p{n}', label=f'Parameter {n}', - unit=f'unit {n}', set_cmd=None, get_cmd=None)) - - # Set up an experiment - - meas = Measurement(exp) - meas.register_parameter(params[0]) - meas.register_parameter(params[1]) - meas.register_parameter(params[2], basis=(params[0],)) - meas.register_parameter(params[3], basis=(params[1],)) - meas.register_parameter(params[4], setpoints=(params[2], params[3])) - - # Make a number of identical runs - - for _ in range(10): - - with meas.run() as datasaver: - - for x in np.random.rand(10): - for y in np.random.rand(10): - z = np.random.rand() - datasaver.add_result((params[2], x), - (params[3], y), - (params[4], z)) - - -def generate_DB_file_with_some_runs_having_not_run_descriptions(): - """ - Generate a .db-file with a handful of runs some of which lack run - description or have it as empty object (based on a real case). - - Generated runs: - #1: run with parameters and correct run description - #2: run with parameters but run description is NULL - #3: run with parameters but run description is empty RunDescriber - #4: run without parameters but run description is NULL - """ - v3fixturepath = os.path.join(fixturepath, 'version3') - os.makedirs(v3fixturepath, exist_ok=True) - path = os.path.join(v3fixturepath, 'some_runs_without_run_description.db') - - if os.path.exists(path): - os.remove(path) - - from qcodes.dataset.measurements import Measurement - from qcodes.dataset.experiment_container import Experiment - from qcodes import Parameter - from qcodes.dataset.descriptions import RunDescriber - from qcodes.dataset.dependencies import InterDependencies - - exp = Experiment(path_to_db=path, - name='experiment_1', - sample_name='no_sample_1') - conn = exp.conn - - # Now make some parameters to use in measurements - params = [] - for n in range(5): - params.append(Parameter(f'p{n}', label=f'Parameter {n}', - unit=f'unit {n}', set_cmd=None, get_cmd=None)) - - # Set up a measurement - - meas = Measurement(exp) - meas.register_parameter(params[0]) - meas.register_parameter(params[1]) - meas.register_parameter(params[2], basis=(params[0],)) - meas.register_parameter(params[3], basis=(params[1],)) - meas.register_parameter(params[4], setpoints=(params[2], params[3])) - - # Initially make 3 correct runs - - run_ids = [] - - for _ in range(3): - - with meas.run() as datasaver: - - for x in np.random.rand(10): - for y in np.random.rand(10): - z = np.random.rand() - datasaver.add_result((params[2], x), - (params[3], y), - (params[4], z)) - - run_ids.append(datasaver.run_id) - - assert [1, 2, 3] == run_ids, 'Run ids of generated runs are not as ' \ - 'expected after generating runs #1-3' - - # Formulate SQL query for adjusting run_description column - - set_run_description_sql = f""" - UPDATE runs - SET run_description = ? - WHERE run_id == ? - """ - - # Make run_description of run #2 NULL - - conn.execute(set_run_description_sql, (None, run_ids[1])) - conn.commit() # just to be sure - - # Make run_description of run #3 equivalent to an empty RunDescriber - - empty_run_description = RunDescriber(InterDependencies()).to_json() - conn.execute(set_run_description_sql, (empty_run_description, run_ids[2])) - conn.commit() # just to be sure - - # Set up a measurement without parameters, and create run #4 out of it - - meas_no_params = Measurement(exp) - - with meas_no_params.run() as datasaver: - pass - - run_ids.append(datasaver.run_id) - - assert [1, 2, 3, 4] == run_ids, 'Run ids of generated runs are not as ' \ - 'expected after generating run #4' - - # Make run_description of run #4 NULL - - conn.execute(set_run_description_sql, (None, run_ids[3])) - conn.commit() # just to be sure - - -def generate_upgraded_v2_runs(): - """ - Generate some runs by upgradeing from v2 db. This - is needed since the bug we want to test against is in - the v2 to v3 upgrade and not in v3 it self. - This requires the v2 generation to be run before this one - """ - import qcodes.dataset.sqlite_base as sqlite_base - v2fixture_path = os.path.join(fixturepath, 'version2', 'some_runs.db') - v3fixturepath = os.path.join(fixturepath, 'version3', 'some_runs_upgraded_2.db') - shutil.copy2(v2fixture_path, v3fixturepath) - sqlite_base.connect(v3fixturepath) - - -if __name__ == '__main__': - - gens = (generate_empty_DB_file, - generate_DB_file_with_some_runs_having_not_run_descriptions, - generate_DB_file_with_some_runs, - generate_upgraded_v2_runs) - - # pylint: disable=E1101 - utils.checkout_to_old_version_and_run_generators(version=3, gens=gens) diff --git a/qcodes/tests/dataset/legacy_DB_generation/utils.py b/qcodes/tests/dataset/legacy_DB_generation/utils.py deleted file mode 100644 index d5d5afac1d4..00000000000 --- a/qcodes/tests/dataset/legacy_DB_generation/utils.py +++ /dev/null @@ -1,85 +0,0 @@ -# General utilities for the database generation and loading scheme -from typing import Dict, List, Tuple -from contextlib import contextmanager -import os - -from git import Repo - -# A brief overview of what each version introduces: -# -# Version 0: the original table schema, runs, experiments, layouts, -# dependencies, result-tables -# -# Version 1: a GUID column is added to the runs table -# -# Version 2: indices are added to runs; GUID and exp_id -# -# Version 3: run_description column is added to the runs table -# - -# NOTE that each hash is supposed to be representing a commit JUST before the -# "next" version is being introduced. -# - -GIT_HASHES: Dict[int, str] = {0: '78d42620fc245a975b5a615ed5e33061baac7846', - 1: '056d59627e22fa3ca7aad4c265e9897c343f79cf', - 2: '5202255924542dad6841dfe3d941a7f80c43956c', - 3: '01cb405d47b72b1a5bab227dcf4b69866ecebb0f'} - -gitrepopath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-5]) -repo = Repo(gitrepopath) - - -@contextmanager -def leave_untouched(repo): - """ - Leave a git repository untouched by whatever fiddling around we need to do - We support both the case of the repo being initially in a detached head - state (relevant for Travis) and the -hopefully- normal case for users of - being at the tip of a branch - """ - - if repo.is_dirty(): - raise ValueError('Git repository is dirty. Can not proceed.') - - was_detached = repo.head.is_detached - - if not was_detached: - current_branch = repo.active_branch - current_commit = repo.head.commit - - try: - yield - - finally: - repo.git.reset('--hard', current_commit) - if not was_detached: - repo.git.checkout(current_branch) - - -def checkout_to_old_version_and_run_generators(version: int, - gens: Tuple) -> None: - """ - Check out the repo to an older version and run the generating functions - supplied. - """ - - with leave_untouched(repo): - - repo.git.checkout(GIT_HASHES[version]) - - # If QCoDeS is not installed in editable mode, it makes no difference - # to do our git magic, since the import will be from site-packages in - # the environment folder, and not from the git-managed folder - import qcodes - qcpath = os.sep.join(qcodes.__file__.split(os.sep)[:-2]) - - # Windows and paths... There can be random un-capitalizations - if qcpath.lower() != gitrepopath.lower(): - raise ValueError('QCoDeS does not seem to be installed in editable' - ' mode, can not proceed. To use this script, ' - 'uninstall QCoDeS and reinstall it with pip ' - 'install -e ') - - for generator in gens: - generator() From 1c1ef1c01542b2b37dc18d0957a170dd9e3aff5e Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 12:53:45 +0100 Subject: [PATCH 15/30] update tests to reflect that there are now 2 inferred params --- .../tests/dataset/test_database_creation_and_upgrading.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index 8b05880ed0d..8050c52171c 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -278,7 +278,7 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] assert p3.depends_on == '' - assert p3.inferred_from == 'p1' + assert p3.inferred_from == 'p1, p0' assert p3.label == "Parameter 3" assert p3.unit == "unit 3" @@ -335,7 +335,7 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] assert p3.depends_on == '' - assert p3.inferred_from == 'p, 1' + assert p3.inferred_from == 'p, 1, ,, , p, 0' assert p3.label == "Parameter 3" assert p3.unit == "unit 3" @@ -374,7 +374,7 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] assert p3.depends_on == '' - assert p3.inferred_from == 'p1' + assert p3.inferred_from == 'p1, p0' assert p3.label == "Parameter 3" assert p3.unit == "unit 3" From bac7d43f1f58247851802ae7c79bc4930e540697 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 14:34:39 +0100 Subject: [PATCH 16/30] Handle empty inferred_from correctly --- qcodes/dataset/sqlite_base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 42017f6e42b..38c71ebaba1 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -615,9 +615,11 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, paramtype = row['type'] break + inferred_from_list: List[str] = [] # first possibility: another parameter depends on this parameter if layout_id in indeps: - inferred_from_list = inferred_from.split(', ') + if inferred_from != '': + inferred_from_list = inferred_from.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, @@ -629,7 +631,8 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, setpoints = dependencies[layout_id] depends_on = [paramspecs[idp].name for idp in setpoints] - inferred_from_list = inferred_from.split(', ') + if inferred_from != '': + inferred_from_list = inferred_from.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, @@ -644,7 +647,7 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, paramtype=paramtype, label=label, unit=unit, depends_on=[], - inferred_from=[]) + inferred_from=inferred_from_list) paramspecs[layout_id] = paramspec return paramspecs From a52d6ec0bad616452fd0474215470f53441fa258 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 14:39:39 +0100 Subject: [PATCH 17/30] Add more asserts to tests --- .../test_database_creation_and_upgrading.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index 8050c52171c..e4ebec039a6 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -260,36 +260,50 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): p0 = [p for p in idp.paramspecs if p.name == 'p0'][0] assert p0.depends_on == '' + assert p0.depends_on_ == [] assert p0.inferred_from == '' + assert p0.inferred_from_ == [] assert p0.label == "Parameter 0" assert p0.unit == "unit 0" p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] assert p1.depends_on == '' + assert p1.depends_on_ == [] assert p1.inferred_from == '' + assert p1.inferred_from_ == [] assert p1.label == "Parameter 1" assert p1.unit == "unit 1" p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] assert p2.depends_on == '' + assert p2.depends_on_ == [] assert p2.inferred_from == 'p0' + assert p2.inferred_from_ == ['p0'] assert p2.label == "Parameter 2" assert p2.unit == "unit 2" p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] assert p3.depends_on == '' + assert p3.depends_on_ == [] assert p3.inferred_from == 'p1, p0' + assert p3.inferred_from_ == ['p1', 'p0'] assert p3.label == "Parameter 3" assert p3.unit == "unit 3" p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] assert p4.depends_on == 'p2, p3' + assert p4.depends_on_ == ['p2', 'p3'] assert p4.inferred_from == '' + assert p4.inferred_from_ == [] assert p4.label == "Parameter 4" assert p4.unit == "unit 4" def test_perfrom_upgrade_v3_to_v4_fixes(): + """ + Test that a db that was upgraded from v2 to v3 with a buggy + version will be corrected when upgraded to v4. + """ v3fixpath = os.path.join(fixturepath, 'db_files', 'version3') @@ -317,31 +331,45 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): p0 = [p for p in idp.paramspecs if p.name == 'p0'][0] assert p0.depends_on == '' + assert p0.depends_on_ == [] assert p0.inferred_from == '' + assert p0.inferred_from_ == [] assert p0.label == "Parameter 0" assert p0.unit == "unit 0" p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] assert p1.depends_on == '' + assert p1.depends_on_ == [] assert p1.inferred_from == '' + assert p1.inferred_from_ == [] assert p1.label == "Parameter 1" assert p1.unit == "unit 1" p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] assert p2.depends_on == '' + assert p2.depends_on_ == [] + # the 2 lines below are wrong due to the incorrect upgrade from + # db version 2 to 3 assert p2.inferred_from == 'p, 0' + assert p2.inferred_from_ == ['p', '0'] assert p2.label == "Parameter 2" assert p2.unit == "unit 2" p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] assert p3.depends_on == '' + assert p3.depends_on_ == [] + # the 2 lines below are wrong due to the incorrect upgrade from + # db version 2 to 3 assert p3.inferred_from == 'p, 1, ,, , p, 0' + assert p3.inferred_from_ == ['p', '1', ',', ' ', 'p', '0'] assert p3.label == "Parameter 3" assert p3.unit == "unit 3" p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] assert p4.depends_on == 'p2, p3' + assert p4.depends_on_ == ['p2', 'p3'] assert p4.inferred_from == '' + assert p4.inferred_from_ == [] assert p4.label == "Parameter 4" assert p4.unit == "unit 4" @@ -356,31 +384,41 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): p0 = [p for p in idp.paramspecs if p.name == 'p0'][0] assert p0.depends_on == '' + assert p0.depends_on_ == [] assert p0.inferred_from == '' + assert p0.inferred_from_ == [] assert p0.label == "Parameter 0" assert p0.unit == "unit 0" p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] assert p1.depends_on == '' + assert p1.depends_on_ == [] assert p1.inferred_from == '' + assert p1.inferred_from_ == [] assert p1.label == "Parameter 1" assert p1.unit == "unit 1" p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] assert p2.depends_on == '' + assert p2.depends_on_ == [] assert p2.inferred_from == 'p0' + assert p2.inferred_from_ == ['p0'] assert p2.label == "Parameter 2" assert p2.unit == "unit 2" p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] assert p3.depends_on == '' + assert p3.depends_on_ == [] assert p3.inferred_from == 'p1, p0' + assert p3.inferred_from_ == ['p1', 'p0'] assert p3.label == "Parameter 3" assert p3.unit == "unit 3" p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] assert p4.depends_on == 'p2, p3' + assert p4.depends_on_ == ['p2', 'p3'] assert p4.inferred_from == '' + assert p4.inferred_from_ == [] assert p4.label == "Parameter 4" assert p4.unit == "unit 4" From 67545d1381edbe5a7dad3daf78cda000e4d25161 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 14:50:44 +0100 Subject: [PATCH 18/30] simplify upgrade logic --- qcodes/dataset/sqlite_base.py | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 38c71ebaba1..defde46e1d6 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -616,16 +616,12 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, break inferred_from_list: List[str] = [] + depends_on: List[str] = [] # first possibility: another parameter depends on this parameter if layout_id in indeps: if inferred_from != '': inferred_from_list = inferred_from.split(', ') - paramspec = ParamSpec(name=name, paramtype=paramtype, - label=label, unit=unit, - inferred_from=inferred_from_list) - paramspecs[layout_id] = paramspec - # second possibility: this parameter depends on another parameter elif layout_id in deps: @@ -633,22 +629,14 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, depends_on = [paramspecs[idp].name for idp in setpoints] if inferred_from != '': inferred_from_list = inferred_from.split(', ') - - paramspec = ParamSpec(name=name, - paramtype=paramtype, - label=label, unit=unit, - depends_on=depends_on, - inferred_from=inferred_from_list) - paramspecs[layout_id] = paramspec - # third possibility: no dependencies - else: - paramspec = ParamSpec(name=name, - paramtype=paramtype, - label=label, unit=unit, - depends_on=[], - inferred_from=inferred_from_list) - paramspecs[layout_id] = paramspec + + paramspec = ParamSpec(name=name, + paramtype=paramtype, + label=label, unit=unit, + depends_on=depends_on, + inferred_from=inferred_from_list) + paramspecs[layout_id] = paramspec return paramspecs From 6ea4cee6fc08644ef46212c081fac733c350378c Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 15:19:50 +0100 Subject: [PATCH 19/30] test param that is neither dep/dependent --- .../test_database_creation_and_upgrading.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index e4ebec039a6..6b4c1ffb9f5 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -298,6 +298,13 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): assert p4.label == "Parameter 4" assert p4.unit == "unit 4" + p5 = [p for p in idp.paramspecs if p.name == 'p5'][0] + assert p5.depends_on == '' + assert p5.depends_on_ == [] + assert p5.inferred_from == 'p0' + assert p5.inferred_from_ == ['p0'] + assert p5.label == "Parameter 5" + assert p5.unit == "unit 5" def test_perfrom_upgrade_v3_to_v4_fixes(): """ @@ -422,6 +429,14 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): assert p4.label == "Parameter 4" assert p4.unit == "unit 4" + p5 = [p for p in idp.paramspecs if p.name == 'p5'][0] + assert p5.depends_on == '' + assert p5.depends_on_ == [] + assert p5.inferred_from == 'p0' + assert p5.inferred_from_ == ['p0'] + assert p5.label == "Parameter 5" + assert p5.unit == "unit 5" + @pytest.mark.usefixtures("empty_temp_db") From 3fe5858d67a006747ad24dbe9bcb0f40def59ccb Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 15:25:31 +0100 Subject: [PATCH 20/30] test that this is wrong before upgrade --- .../dataset/test_database_creation_and_upgrading.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index 6b4c1ffb9f5..5dcb5999b8c 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -380,6 +380,16 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): assert p4.label == "Parameter 4" assert p4.unit == "unit 4" + p5 = [p for p in idp.paramspecs if p.name == 'p5'][0] + assert p5.depends_on == '' + assert p5.depends_on_ == [] + # the 2 lines below are wrong due to the incorrect upgrade from + # db version 2 to 3. Here the interdep is missing + assert p5.inferred_from == '' + assert p5.inferred_from_ == [] + assert p5.label == "Parameter 5" + assert p5.unit == "unit 5" + perform_db_upgrade_3_to_4(conn) c = atomic_transaction(conn, sql) From fefc025c314484bfe37ddb668ed46111966f50f5 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 15:26:22 +0100 Subject: [PATCH 21/30] also handle inferred_from when not dep/dependent --- qcodes/dataset/sqlite_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index defde46e1d6..3dcd568a397 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -630,7 +630,9 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, if inferred_from != '': inferred_from_list = inferred_from.split(', ') # third possibility: no dependencies - + else: + if inferred_from != '': + inferred_from_list = inferred_from.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, From 64af8e82543e5b1de4ec7de3e907898747d783b9 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 15:35:16 +0100 Subject: [PATCH 22/30] simplifty upgrade logic --- qcodes/dataset/sqlite_base.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 3dcd568a397..49c3239808f 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -617,22 +617,15 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, inferred_from_list: List[str] = [] depends_on: List[str] = [] - # first possibility: another parameter depends on this parameter - if layout_id in indeps: - if inferred_from != '': - inferred_from_list = inferred_from.split(', ') - - # second possibility: this parameter depends on another parameter - elif layout_id in deps: + # this parameter depends on another parameter + if layout_id in deps: setpoints = dependencies[layout_id] depends_on = [paramspecs[idp].name for idp in setpoints] - if inferred_from != '': - inferred_from_list = inferred_from.split(', ') - # third possibility: no dependencies - else: - if inferred_from != '': - inferred_from_list = inferred_from.split(', ') + + if inferred_from != '': + inferred_from_list = inferred_from.split(', ') + paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, From 959a79f4b22cfe67317e607a454369553f8557e2 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 15:58:29 +0100 Subject: [PATCH 23/30] docstring --- qcodes/dataset/sqlite_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 49c3239808f..87b04e4966a 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -713,7 +713,7 @@ def perform_db_upgrade_2_to_3(conn: ConnectionPlus) -> None: def perform_db_upgrade_3_to_4(conn: ConnectionPlus) -> None: """ Perform the upgrade from version 3 to version 4. This really - repeats the version 3 upgrade as it originally had a bug in + repeats the version 3 upgrade as it originally had two bugs in the inferred annotation. This has since been fixes so rerun the upgrade. """ From 62e416099a7c42d329ac0106cba17a9065fcd6c0 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 16:04:29 +0100 Subject: [PATCH 24/30] add a direct v3->v4 upgrade test --- .../test_database_creation_and_upgrading.py | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index 5dcb5999b8c..ce68d276e17 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -306,7 +306,8 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): assert p5.label == "Parameter 5" assert p5.unit == "unit 5" -def test_perfrom_upgrade_v3_to_v4_fixes(): + +def test_perform_upgrade_v2_v3_to_v4_fixes(): """ Test that a db that was upgraded from v2 to v3 with a buggy version will be corrected when upgraded to v4. @@ -448,6 +449,86 @@ def test_perfrom_upgrade_v3_to_v4_fixes(): assert p5.unit == "unit 5" +def test_perform_upgrade_v3_to_v4_fixes(): + """ + Test that a db that was upgraded from v2 to v3 with a buggy + version will be corrected when upgraded to v4. + """ + + v3fixpath = os.path.join(fixturepath, 'db_files', 'version3') + + dbname_old = os.path.join(v3fixpath, 'some_runs_upgraded_2.db') + + if not os.path.exists(dbname_old): + pytest.skip("No db-file fixtures found. You can generate test db-files" + " using the scripts in the legacy_DB_generation folder") + + with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn: + + assert get_user_version(conn) == 3 + + sql = f""" + SELECT run_description + FROM runs + WHERE run_id == 1 + """ + + perform_db_upgrade_3_to_4(conn) + + c = atomic_transaction(conn, sql) + json_str = one(c, 'run_description') + + desc = RunDescriber.from_json(json_str) + idp = desc.interdeps + assert isinstance(idp, InterDependencies) + + p0 = [p for p in idp.paramspecs if p.name == 'p0'][0] + assert p0.depends_on == '' + assert p0.depends_on_ == [] + assert p0.inferred_from == '' + assert p0.inferred_from_ == [] + assert p0.label == "Parameter 0" + assert p0.unit == "unit 0" + + p1 = [p for p in idp.paramspecs if p.name == 'p1'][0] + assert p1.depends_on == '' + assert p1.depends_on_ == [] + assert p1.inferred_from == '' + assert p1.inferred_from_ == [] + assert p1.label == "Parameter 1" + assert p1.unit == "unit 1" + + p2 = [p for p in idp.paramspecs if p.name == 'p2'][0] + assert p2.depends_on == '' + assert p2.depends_on_ == [] + assert p2.inferred_from == 'p0' + assert p2.inferred_from_ == ['p0'] + assert p2.label == "Parameter 2" + assert p2.unit == "unit 2" + + p3 = [p for p in idp.paramspecs if p.name == 'p3'][0] + assert p3.depends_on == '' + assert p3.depends_on_ == [] + assert p3.inferred_from == 'p1, p0' + assert p3.inferred_from_ == ['p1', 'p0'] + assert p3.label == "Parameter 3" + assert p3.unit == "unit 3" + + p4 = [p for p in idp.paramspecs if p.name == 'p4'][0] + assert p4.depends_on == 'p2, p3' + assert p4.depends_on_ == ['p2', 'p3'] + assert p4.inferred_from == '' + assert p4.inferred_from_ == [] + assert p4.label == "Parameter 4" + assert p4.unit == "unit 4" + + p5 = [p for p in idp.paramspecs if p.name == 'p5'][0] + assert p5.depends_on == '' + assert p5.depends_on_ == [] + assert p5.inferred_from == 'p0' + assert p5.inferred_from_ == ['p0'] + assert p5.label == "Parameter 5" + assert p5.unit == "unit 5" @pytest.mark.usefixtures("empty_temp_db") def test_update_existing_guids(caplog): From 82aeb96a7a17ce9210565b1f6a01661306fd3ff9 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 16:25:04 +0100 Subject: [PATCH 25/30] More clear error message --- qcodes/dataset/param_spec.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qcodes/dataset/param_spec.py b/qcodes/dataset/param_spec.py index f425fc744a7..3bd8c7132c9 100644 --- a/qcodes/dataset/param_spec.py +++ b/qcodes/dataset/param_spec.py @@ -45,7 +45,8 @@ def __init__(self, name: str, if isinstance(inferred_from, str): raise ValueError(f"ParamSpec {self.name} got " - f"string {inferred_from} it needs a " + f"string {inferred_from} as inferred_from. " + f"It needs a " f"Sequence of ParamSpecs or strings") self._inferred_from.extend( p.name if isinstance(p, ParamSpec) else p @@ -53,7 +54,7 @@ def __init__(self, name: str, if isinstance(depends_on, str): raise ValueError(f"ParamSpec {self.name} got " - f"string {depends_on} it needs a " + f"string {depends_on} as depends_on. It needs a " f"Sequence of ParamSpecs or strings") self._depends_on.extend( p.name if isinstance(p, ParamSpec) else p From 5e317fa439b2ae1a790c8d397da5fe52f55019ba Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Mon, 4 Feb 2019 16:28:42 +0100 Subject: [PATCH 26/30] Add tests for invalid paramspecs --- qcodes/tests/dataset/test_paramspec.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/dataset/test_paramspec.py b/qcodes/tests/dataset/test_paramspec.py index 74a0f5d7a34..30cbe2ba9eb 100644 --- a/qcodes/tests/dataset/test_paramspec.py +++ b/qcodes/tests/dataset/test_paramspec.py @@ -47,7 +47,7 @@ def version_0_serializations(): 'paramtype': 'array', 'label': 'My Array ParamSpec', 'unit': 'Ars', - 'inferred_from': ['p1', 'p2' ], + 'inferred_from': ['p1', 'p2'], 'depends_on': []}) return sers @@ -141,6 +141,11 @@ def test_depends_on(name1, name2, name3): assert ps1.depends_on == f"{ps2.name}, {ps3.name}, foo" assert ps1.depends_on_ == [ps2.name, ps3.name, "foo"] + with pytest.raises(ValueError, + match=f"ParamSpec {name1} got string foo as depends_on. " + "It needs a Sequence of ParamSpecs or strings"): + ParamSpec(name1, "numeric", depends_on='foo') + @given( name1=hst.text(min_size=4, alphabet=alphabet), @@ -156,6 +161,12 @@ def test_inferred_from(name1, name2, name3): assert ps1.inferred_from == f"{ps2.name}, {ps3.name}, bar" assert ps1.inferred_from_ == [ps2.name, ps3.name, "bar"] + with pytest.raises(ValueError, + match=f"ParamSpec {name1} got string foo as " + f"inferred_from. " + "It needs a Sequence of ParamSpecs or strings"): + ParamSpec(name1, "numeric", inferred_from='foo') + @given( name1=hst.text(min_size=4, alphabet=alphabet), From 24f3fe9b274f44d4bdef5da2e4a9c3abca5a64ee Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 5 Feb 2019 09:12:54 +0100 Subject: [PATCH 27/30] Document the errors fixed --- qcodes/dataset/sqlite_base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 87b04e4966a..6eaca9948c3 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -714,8 +714,11 @@ def perform_db_upgrade_3_to_4(conn: ConnectionPlus) -> None: """ Perform the upgrade from version 3 to version 4. This really repeats the version 3 upgrade as it originally had two bugs in - the inferred annotation. This has since been fixes so rerun - the upgrade. + the inferred annotation. inferred_from was passed incorrectly + resulting in the parameter being marked inferred_from for each char + in the inferred_from variable and inferred_from was not handled + correctly for parameters that were neither dependencies nor dependent on + other parameters. Both have since been fixes so rerun the upgrade. """ no_of_runs_query = "SELECT max(run_id) FROM runs" From 863de65011ed1c6280e76315c11bb4ca13a5c4ea Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Wed, 6 Feb 2019 13:23:25 +0100 Subject: [PATCH 28/30] Update test docstrings --- .../test_database_creation_and_upgrading.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/qcodes/tests/dataset/test_database_creation_and_upgrading.py b/qcodes/tests/dataset/test_database_creation_and_upgrading.py index ce68d276e17..d2cc5e2b76a 100644 --- a/qcodes/tests/dataset/test_database_creation_and_upgrading.py +++ b/qcodes/tests/dataset/test_database_creation_and_upgrading.py @@ -125,7 +125,8 @@ def test_perform_actual_upgrade_0_to_1(): if not os.path.exists(dbname_old): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the " + "https://github.com/QCoDeS/qcodes_generate_test_db/ repo") with temporarily_copied_DB(dbname_old, debug=False, version=0) as conn: @@ -183,7 +184,8 @@ def test_perform_actual_upgrade_2_to_3_empty(): if not os.path.exists(dbname_old): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the " + "https://github.com/QCoDeS/qcodes_generate_test_db/ repo") with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn: @@ -212,7 +214,8 @@ def test_perform_actual_upgrade_2_to_3_empty_runs(): if not os.path.exists(dbname_old): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the " + "https://github.com/QCoDeS/qcodes_generate_test_db/ repo") with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn: @@ -227,7 +230,8 @@ def test_perform_actual_upgrade_2_to_3_some_runs(): if not os.path.exists(dbname_old): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the" + "https://github.com/QCoDeS/qcodes_generate_test_db/ repo") with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn: @@ -319,7 +323,8 @@ def test_perform_upgrade_v2_v3_to_v4_fixes(): if not os.path.exists(dbname_old): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the" + " https://github.com/QCoDeS/qcodes_generate_test_db/ repo") with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn: @@ -449,10 +454,9 @@ def test_perform_upgrade_v2_v3_to_v4_fixes(): assert p5.unit == "unit 5" -def test_perform_upgrade_v3_to_v4_fixes(): +def test_perform_upgrade_v3_to_v4(): """ - Test that a db that was upgraded from v2 to v3 with a buggy - version will be corrected when upgraded to v4. + Test that a db upgrade from v2 to v4 works correctly. """ v3fixpath = os.path.join(fixturepath, 'db_files', 'version3') @@ -461,7 +465,8 @@ def test_perform_upgrade_v3_to_v4_fixes(): if not os.path.exists(dbname_old): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the " + "https://github.com/QCoDeS/qcodes_generate_test_db/ repo") with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn: @@ -530,6 +535,7 @@ def test_perform_upgrade_v3_to_v4_fixes(): assert p5.label == "Parameter 5" assert p5.unit == "unit 5" + @pytest.mark.usefixtures("empty_temp_db") def test_update_existing_guids(caplog): @@ -643,7 +649,8 @@ def test_getting_db_version(version): if not os.path.exists(dbname): pytest.skip("No db-file fixtures found. You can generate test db-files" - " using the scripts in the legacy_DB_generation folder") + " using the scripts in the " + "https://github.com/QCoDeS/qcodes_generate_test_db/ repo") (db_v, new_v) = get_db_version_and_newest_available_version(dbname) From 6824df549e753847b15a34dc4ee4496b0cf7b05e Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Wed, 6 Feb 2019 13:23:44 +0100 Subject: [PATCH 29/30] Correct docstring --- qcodes/dataset/sqlite_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 6eaca9948c3..8cf48f6c971 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -718,7 +718,7 @@ def perform_db_upgrade_3_to_4(conn: ConnectionPlus) -> None: resulting in the parameter being marked inferred_from for each char in the inferred_from variable and inferred_from was not handled correctly for parameters that were neither dependencies nor dependent on - other parameters. Both have since been fixes so rerun the upgrade. + other parameters. Both have since been fixed so rerun the upgrade. """ no_of_runs_query = "SELECT max(run_id) FROM runs" From 778de4dc1d83e7bd0142ad99f7aa4fadf03b3fec Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Wed, 6 Feb 2019 13:26:24 +0100 Subject: [PATCH 30/30] More consistent variable names --- qcodes/dataset/sqlite_base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index 8cf48f6c971..90d2721e8e6 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -606,7 +606,7 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, # depend, then the dependent ParamSpecs and finally the rest for layout_id in list(indeps) + list(deps) + list(the_rest): - (name, label, unit, inferred_from) = layouts[layout_id] + (name, label, unit, inferred_from_str) = layouts[layout_id] # get the data type sql = f'PRAGMA TABLE_INFO("{result_table_name}")' c = transaction(conn, sql) @@ -615,7 +615,7 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, paramtype = row['type'] break - inferred_from_list: List[str] = [] + inferred_from: List[str] = [] depends_on: List[str] = [] # this parameter depends on another parameter @@ -623,14 +623,14 @@ def _2to3_get_paramspecs(conn: ConnectionPlus, setpoints = dependencies[layout_id] depends_on = [paramspecs[idp].name for idp in setpoints] - if inferred_from != '': - inferred_from_list = inferred_from.split(', ') + if inferred_from_str != '': + inferred_from = inferred_from_str.split(', ') paramspec = ParamSpec(name=name, paramtype=paramtype, label=label, unit=unit, depends_on=depends_on, - inferred_from=inferred_from_list) + inferred_from=inferred_from) paramspecs[layout_id] = paramspec return paramspecs