From a850344913b6c0b755d6b671c5aee7e4c7ae9f26 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Tue, 28 Feb 2023 11:10:12 +0100 Subject: [PATCH 01/40] update requirement for neptune --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 04a444e..d5457d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ python = "^3.7" importlib-metadata = { version = "*", python = "<3.8" } # Base requirements -neptune-client = ">=0.16.17" +neptune = ">=1.0.0" kedro = ">=0.18.0" "ruamel.yaml" = "^0.17.0" From 10c49a2a7b9fa037ee21aa33310920656afcb737 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 17:39:04 +0100 Subject: [PATCH 02/40] move neptune to optional dev requirements --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d5457d7..b3fd7ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ python = "^3.7" importlib-metadata = { version = "*", python = "<3.8" } # Base requirements -neptune = ">=1.0.0" kedro = ">=0.18.0" "ruamel.yaml" = "^0.17.0" @@ -24,6 +23,7 @@ pre-commit = { version = "*", optional = true } pytest = { version = ">=5.0", optional = true } pytest-cov = { version = "2.10.1", optional = true } backoff = { version = "*", optional = true } +neptune = { version = ">=1.0.0", optional = true } [tool.poetry.extras] dev = [ @@ -31,6 +31,7 @@ dev = [ "pytest", "pytest-cov", "backoff", + "neptune", ] [tool.poetry] From d15c57dbc6a95f7df321b45eb8d04f9b87e44fbe Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 17:39:23 +0100 Subject: [PATCH 03/40] handle case when neptune not installed --- src/kedro_neptune/version.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/kedro_neptune/version.py b/src/kedro_neptune/version.py index bba75f3..0d15f45 100644 --- a/src/kedro_neptune/version.py +++ b/src/kedro_neptune/version.py @@ -1,6 +1,7 @@ __all__ = ["__version__"] import sys +from importlib.util import find_spec if sys.version_info >= (3, 8): from importlib.metadata import ( @@ -13,6 +14,16 @@ version, ) +if not (find_spec("neptune") or find_spec("neptune-client")): + msg = """ + The Neptune client library was not found. + + Install the neptune package with + `pip install neptune` + + Need help? -> https://docs.neptune.ai/setup/installation/""" + raise PackageNotFoundError(msg) + try: __version__ = version("kedro-neptune") except PackageNotFoundError: From 4d00428edbe5d9ef9ea37fad92feff7db00bc02c Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 17:39:43 +0100 Subject: [PATCH 04/40] fix imports --- src/kedro_neptune/__init__.py | 14 +++++++------- tests/kedro_neptune/utils/run_utils.py | 11 +++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/kedro_neptune/__init__.py b/src/kedro_neptune/__init__.py index aa495c3..2c50b89 100644 --- a/src/kedro_neptune/__init__.py +++ b/src/kedro_neptune/__init__.py @@ -51,19 +51,19 @@ from kedro_neptune.version import __version__ try: - # neptune-client=0.9.0+ package structure - import neptune.new as neptune - from neptune.new.handler import Handler - from neptune.new.integrations.utils import join_paths - from neptune.new.types import File - from neptune.new.utils import stringify_unsupported -except ImportError: # neptune-client>=1.0.0 package structure import neptune from neptune.handler import Handler from neptune.integrations.utils import join_paths from neptune.types import File from neptune.utils import stringify_unsupported +except ImportError: + # neptune-client=0.9.0+ package structure + import neptune.new as neptune + from neptune.new.handler import Handler + from neptune.new.integrations.utils import join_paths + from neptune.new.types import File + from neptune.new.utils import stringify_unsupported INTEGRATION_VERSION_KEY = "source_code/integrations/kedro-neptune" diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 28e8e9a..ee389be 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -24,8 +24,15 @@ ) import backoff -from neptune.new import init_run -from neptune.new.metadata_containers import Run + +try: + from neptune import ( + Run, + init_run, + ) +except ImportError: + from neptune.new import init_run + from neptune.new.metadata_containers import Run # It may take some time to refresh cache From e279423cf5b50d533e1b3aad44dc64c773cb0c1c Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 17:53:14 +0100 Subject: [PATCH 05/40] update changlog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd8f33e..57b2a21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ -## [UNRELEASED] 0.1.5 +## 0.1.5 ### Fixes - Fixed some of latest `neptune-client` warning messages ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58)) - Fixed failing run on latest MacOS 12 ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58)) ### Changes +- Removed `neptune` and `neptune-client` from base requirements for backward compatibility ([#62](https://github.com/neptune-ai/kedro-neptune/pull/62)) - Simplified example project and removed unrelated files ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58)) - Better life-time of the internally created Run in `NeptuneRunDataSet` ([#58](https://github.com/neptune-ai/kedro-neptune/pull/58)) - Updated the integration for compatibility with `neptune-client` `1.0.0`. ([#59](https://github.com/neptune-ai/kedro-neptune/pull/59)) From 2d6b71f53285fe8852cb0eacaa2e947795c69f13 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 18:31:34 +0100 Subject: [PATCH 06/40] temporarily remove macos tests --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73c753f..61cf98b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,8 @@ jobs: strategy: max-parallel: 4 matrix: - os: [macos-latest, ubuntu-latest] - python-version: [3.7, 3.8, 3.9, "3.10"] + os: [ubuntu-latest] + python-version: [3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 From d814866381bc47bc742ba6b92aeab54d9f5857e9 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 18:32:35 +0100 Subject: [PATCH 07/40] fix accidental typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61cf98b..37c073a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: max-parallel: 4 matrix: os: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2 From 2f4b9efa38ab2a80105391036abd564707f1abe8 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 19:44:26 +0100 Subject: [PATCH 08/40] temporarily disable parallel test --- .github/actions/e2e/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index ea7ccb9..3faa7ab 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -30,7 +30,7 @@ runs: run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" shell: bash - - name: Test parallel run - working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel - shell: bash +# - name: Test parallel run +# working-directory: ${{ inputs.working_directory }}/examples/planets +# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel +# shell: bash From 9bdecf5fe940614560628ea7c9ac19c762d153e3 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 19:50:46 +0100 Subject: [PATCH 09/40] change checkout action to v3 --- .github/actions/e2e/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 3faa7ab..988ede7 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -10,7 +10,7 @@ runs: using: "composite" steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: neptune-ai/kedro-neptune path: ${{ inputs.working_directory }} From 2e70ce8284eb93a8225134de83a272dbce370057 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 19:56:08 +0100 Subject: [PATCH 10/40] checkout v3 + changed neptune project --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37c073a..c6ee4ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: os: [ubuntu-latest] python-version: [3.7, 3.8, 3.9, "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-python@v2 with: @@ -18,7 +18,7 @@ jobs: - name: Test env: NEPTUNE_API_TOKEN: ${{ secrets.E2E_NEPTUNE_API_TOKEN }} - NEPTUNE_PROJECT: e2e-tests/integrations + NEPTUNE_PROJECT: e2e-tests/e2e uses: ./.github/actions/e2e publish: From fb44f000df617966ce12d7987f9dedb192b27deb Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:17:18 +0100 Subject: [PATCH 11/40] swap tests --- .github/actions/e2e/action.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 988ede7..87f2057 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -25,12 +25,12 @@ runs: pip list shell: bash - - name: Test standard - working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" - shell: bash - -# - name: Test parallel run +# - name: Test standard # working-directory: ${{ inputs.working_directory }}/examples/planets -# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel +# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" # shell: bash + + - name: Test parallel run + working-directory: ${{ inputs.working_directory }}/examples/planets + run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel + shell: bash From bf6bf7ea2ee3f149425337e2f8954df4eabb98c9 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:28:57 +0100 Subject: [PATCH 12/40] experiment with dummy test --- .github/actions/e2e/action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 87f2057..1da6159 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -30,7 +30,10 @@ runs: # run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" # shell: bash - - name: Test parallel run - working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel +# - name: Test parallel run +# working-directory: ${{ inputs.working_directory }}/examples/planets +# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel +# shell: bash + - name: Dumb test + run : pytest -v tests/kedro_neptune/test_utils.py shell: bash From 26a080de5c89ea4fec4432921221b9172b377951 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:34:20 +0100 Subject: [PATCH 13/40] uncomment and change tests --- .github/actions/e2e/action.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 1da6159..f818d86 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -25,15 +25,15 @@ runs: pip list shell: bash -# - name: Test standard -# working-directory: ${{ inputs.working_directory }}/examples/planets -# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" -# shell: bash + - name: Test standard + working-directory: ${{ inputs.working_directory }}/examples/planets + run: pytest -v -m "not parallel" + shell: bash -# - name: Test parallel run -# working-directory: ${{ inputs.working_directory }}/examples/planets -# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel -# shell: bash + - name: Test parallel run + working-directory: ${{ inputs.working_directory }}/examples/planets + run: pytest -v -m parallel + shell: bash - name: Dumb test run : pytest -v tests/kedro_neptune/test_utils.py shell: bash From 4c1ee8be9a0050d293a833f788e202b1894c840d Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:36:09 +0100 Subject: [PATCH 14/40] simplify commands --- .github/actions/e2e/action.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index f818d86..c535df4 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,13 +27,5 @@ runs: - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: pytest -v -m "not parallel" - shell: bash - - - name: Test parallel run - working-directory: ${{ inputs.working_directory }}/examples/planets - run: pytest -v -m parallel - shell: bash - - name: Dumb test - run : pytest -v tests/kedro_neptune/test_utils.py + run: pytest -v shell: bash From 976bd968f0f34700fd65ed88f2bbdc9c548882f8 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:38:22 +0100 Subject: [PATCH 15/40] change test directory --- .github/actions/e2e/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index c535df4..a772a45 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,5 +27,5 @@ runs: - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: pytest -v + run: pytest -v ../../tests shell: bash From a442f025e78d34c02a915ec8a493518f03bea2d5 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:51:14 +0100 Subject: [PATCH 16/40] fix paths, run as one test --- .github/actions/e2e/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index a772a45..3ebd3a9 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -10,7 +10,7 @@ runs: using: "composite" steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: repository: neptune-ai/kedro-neptune path: ${{ inputs.working_directory }} @@ -25,7 +25,7 @@ runs: pip list shell: bash - - name: Test standard + - name: test working-directory: ${{ inputs.working_directory }}/examples/planets - run: pytest -v ../../tests + run: export PYTHONPATH=$PWD/src; pytest -v ../../tests" shell: bash From 898dd483815c9f11668235c1d28c05cec7737115 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 20:53:59 +0100 Subject: [PATCH 17/40] fix typo --- .github/actions/e2e/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 3ebd3a9..b1fd674 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,5 +27,5 @@ runs: - name: test working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest -v ../../tests" + run: export PYTHONPATH=$PWD/src; pytest -v ../../tests shell: bash From c30c1af4fb58a09d247c00bffecdfb42d65d66f8 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 21:28:39 +0100 Subject: [PATCH 18/40] Revert "temporarily disable parallel test" This reverts commit 2f4b9efa38ab2a80105391036abd564707f1abe8. --- .github/actions/e2e/action.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index b1fd674..ea7ccb9 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -25,7 +25,12 @@ runs: pip list shell: bash - - name: test + - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest -v ../../tests + run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" + shell: bash + + - name: Test parallel run + working-directory: ${{ inputs.working_directory }}/examples/planets + run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel shell: bash From 9846f9300b56c0a81093ff1b64af26bab77c7f27 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 21:32:50 +0100 Subject: [PATCH 19/40] try again with no parallel --- .github/actions/e2e/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index ea7ccb9..3faa7ab 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -30,7 +30,7 @@ runs: run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" shell: bash - - name: Test parallel run - working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel - shell: bash +# - name: Test parallel run +# working-directory: ${{ inputs.working_directory }}/examples/planets +# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel +# shell: bash From d2d35eea6022c51de2b60a3f67fb630542a5f10d Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:01:44 +0100 Subject: [PATCH 20/40] try with max 1 parallel --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6ee4ef..f2d2c4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ jobs: test: runs-on: ${{ matrix.os }} strategy: - max-parallel: 4 + max-parallel: 1 matrix: os: [ubuntu-latest] python-version: [3.7, 3.8, 3.9, "3.10"] From c5e31999ab8d68049e33c26b23d8d3ac710a13c5 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:08:39 +0100 Subject: [PATCH 21/40] check if at least one test can run --- .github/actions/e2e/action.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 3faa7ab..1189a23 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,7 +27,7 @@ runs: - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" + run: export PYTHONPATH=$PWD/src; pytest ../../tests/kedro_neptune/test_utils.py shell: bash # - name: Test parallel run diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2d2c4f..c6ee4ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ jobs: test: runs-on: ${{ matrix.os }} strategy: - max-parallel: 1 + max-parallel: 4 matrix: os: [ubuntu-latest] python-version: [3.7, 3.8, 3.9, "3.10"] From 01d0dcb4b9d82083c742ffb218f32639eac70acf Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:13:34 +0100 Subject: [PATCH 22/40] check if after filename change still runs --- .github/actions/e2e/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 1189a23..67ebb1b 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,7 +27,7 @@ runs: - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests/kedro_neptune/test_utils.py + run: export PYTHONPATH=$PWD/src; pytest ../../tests/kedro_neptune/test_standard.py shell: bash # - name: Test parallel run From eaa8e0c0ccd278c469c48da9864c442c62c41791 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:18:38 +0100 Subject: [PATCH 23/40] check if assert_structure causes freeze --- tests/kedro_neptune/test_standard.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/kedro_neptune/test_standard.py b/tests/kedro_neptune/test_standard.py index c31665c..ed90812 100644 --- a/tests/kedro_neptune/test_standard.py +++ b/tests/kedro_neptune/test_standard.py @@ -14,14 +14,19 @@ # limitations under the License. # from tests.kedro_neptune.utils.kedro_utils import run_pipeline -from tests.kedro_neptune.utils.run_utils import assert_structure + +# from tests.kedro_neptune.utils.run_utils import assert_structure def test_run(): run_pipeline(project="planets") - assert_structure() + + +# assert_structure() def test_run_with_params(): run_pipeline(project="planets", session_params={"extra_params": {"travel_speed": 40000}}) - assert_structure(travel_speed=40000) + + +# assert_structure(travel_speed=40000) From e273bf803acda1f6abf776aa65f63d7734259ca0 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:23:06 +0100 Subject: [PATCH 24/40] check if removing backoff fixes tests --- tests/kedro_neptune/test_standard.py | 11 +++-------- tests/kedro_neptune/utils/run_utils.py | 3 --- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/kedro_neptune/test_standard.py b/tests/kedro_neptune/test_standard.py index ed90812..c31665c 100644 --- a/tests/kedro_neptune/test_standard.py +++ b/tests/kedro_neptune/test_standard.py @@ -14,19 +14,14 @@ # limitations under the License. # from tests.kedro_neptune.utils.kedro_utils import run_pipeline - -# from tests.kedro_neptune.utils.run_utils import assert_structure +from tests.kedro_neptune.utils.run_utils import assert_structure def test_run(): run_pipeline(project="planets") - - -# assert_structure() + assert_structure() def test_run_with_params(): run_pipeline(project="planets", session_params={"extra_params": {"travel_speed": 40000}}) - - -# assert_structure(travel_speed=40000) + assert_structure(travel_speed=40000) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index ee389be..f1efed1 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -23,8 +23,6 @@ Optional, ) -import backoff - try: from neptune import ( Run, @@ -36,7 +34,6 @@ # It may take some time to refresh cache -@backoff.on_exception(backoff.expo, AssertionError, max_value=10) def assert_structure(travel_speed: int = 10000): run = restore_run() From affce32e4be18ff24e03be6467f55c91037b6958 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:25:50 +0100 Subject: [PATCH 25/40] add sleep for cache to refresh --- tests/kedro_neptune/utils/run_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index f1efed1..1423d3f 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -17,6 +17,7 @@ import hashlib import os +import time from ast import literal_eval from typing import ( List, @@ -36,7 +37,7 @@ # It may take some time to refresh cache def assert_structure(travel_speed: int = 10000): run = restore_run() - + time.sleep(30) # Base run information assert run.exists("kedro") assert run.exists("kedro/catalog") From 9021b9e63fc231c7d46594c5e1b353267f46873a Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:29:12 +0100 Subject: [PATCH 26/40] increase sleep time and sync --- tests/kedro_neptune/utils/run_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 1423d3f..09befe3 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -37,7 +37,8 @@ # It may take some time to refresh cache def assert_structure(travel_speed: int = 10000): run = restore_run() - time.sleep(30) + run.sync() + time.sleep(60) # Base run information assert run.exists("kedro") assert run.exists("kedro/catalog") From 0f315a6085c8d552b7dcad48a67c1a44c67d7479 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:34:28 +0100 Subject: [PATCH 27/40] try with context manager and backoff --- tests/kedro_neptune/utils/run_utils.py | 171 +++++++++++++------------ 1 file changed, 90 insertions(+), 81 deletions(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 09befe3..5580267 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -17,13 +17,14 @@ import hashlib import os -import time from ast import literal_eval from typing import ( List, Optional, ) +import backoff + try: from neptune import ( Run, @@ -35,87 +36,95 @@ # It may take some time to refresh cache +@backoff.on_exception(backoff.expo, AssertionError, max_value=10) def assert_structure(travel_speed: int = 10000): - run = restore_run() - run.sync() - time.sleep(60) - # Base run information - assert run.exists("kedro") - assert run.exists("kedro/catalog") - assert run.exists("kedro/nodes") - assert run.exists("kedro/kedro_command") - assert run.exists("kedro/run_params") - assert run.exists("kedro/structure") - - # Data catalog - assert run.exists("kedro/catalog/datasets") - assert run.exists("kedro/catalog/files") - assert run.exists("kedro/catalog/parameters") - - assert run.exists("kedro/catalog/datasets/planets") - assert run["kedro/catalog/datasets/planets"].fetch() == { - "filepath": f"{os.getcwd()}/data/planets/planets.csv", - "name": "planets", - "protocol": "file", - "save_args": {"index": False}, - "type": "CSVDataSet", - "version": "None", - } - - assert run.exists("kedro/catalog/datasets/planets@neptune") - assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { - "extension": "csv", - "filepath": f"{os.getcwd()}/data/planets/planets.csv", - "name": "planets@neptune", - "protocol": "file", - "type": "NeptuneFileDataSet", - "version": "None", - } - assert run.exists("kedro/catalog/files/planets@neptune") - run["kedro/catalog/files/planets@neptune"].download("/tmp/file") - with open("/tmp/file", "rb") as handler: - file_content = handler.read() - assert hashlib.md5(file_content).hexdigest() == "af37712c8c80afc9690e4b70b7a590c5" - - assert run.exists("kedro/catalog/files/logo") - run["kedro/catalog/files/logo"].download("/tmp/file") - with open("/tmp/file", "rb") as handler: - file_content = handler.read() - assert hashlib.md5(file_content).hexdigest() == "85d289d3ed3f321557b6c428b7b35a67" - - assert run.exists("kedro/catalog/parameters/travel_speed") - assert run["kedro/catalog/parameters/travel_speed"].fetch() == travel_speed - - # Nodes data - check_node_metadata( - run=run, node_namespace="kedro/nodes/distances", inputs=["planets"], outputs=["distances_to_planets"] - ) - check_node_metadata( - run=run, - node_namespace="kedro/nodes/furthest", - inputs=["distances_to_planets"], - outputs=["furthest_planet_distance", "furthest_planet_name"], - ) - check_node_metadata(run=run, node_namespace="kedro/nodes/judge_model", inputs=["neptune_run", "dataset"]) - check_node_metadata(run=run, node_namespace="kedro/nodes/prepare_dataset", inputs=["planets"], outputs=["dataset"]) - check_node_metadata( - run=run, - node_namespace="kedro/nodes/travel_time", - inputs=["furthest_planet_distance", "furthest_planet_name", "params:travel_speed"], - outputs=["travel_hours"], - ) - assert run.exists("kedro/nodes/travel_time/parameters") - assert run.exists("kedro/nodes/travel_time/parameters/travel_speed") - assert run["kedro/nodes/travel_time/parameters/travel_speed"].fetch() == travel_speed - - # User defined data - assert run.exists("furthest_planet") - assert run.exists("furthest_planet/name") - assert run.exists("furthest_planet/travel_days") - assert run.exists("furthest_planet/travel_hours") - assert run.exists("furthest_planet/travel_months") - assert run.exists("furthest_planet/travel_years") - assert run["furthest_planet/name"].fetch() == "NEPTUNE" + with init_run( + capture_stderr=False, + capture_stdout=False, + capture_hardware_metrics=False, + capture_traceback=False, + source_files=[], + ) as run: + + # Base run information + assert run.exists("kedro") + assert run.exists("kedro/catalog") + assert run.exists("kedro/nodes") + assert run.exists("kedro/kedro_command") + assert run.exists("kedro/run_params") + assert run.exists("kedro/structure") + + # Data catalog + assert run.exists("kedro/catalog/datasets") + assert run.exists("kedro/catalog/files") + assert run.exists("kedro/catalog/parameters") + + assert run.exists("kedro/catalog/datasets/planets") + assert run["kedro/catalog/datasets/planets"].fetch() == { + "filepath": f"{os.getcwd()}/data/planets/planets.csv", + "name": "planets", + "protocol": "file", + "save_args": {"index": False}, + "type": "CSVDataSet", + "version": "None", + } + + assert run.exists("kedro/catalog/datasets/planets@neptune") + assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { + "extension": "csv", + "filepath": f"{os.getcwd()}/data/planets/planets.csv", + "name": "planets@neptune", + "protocol": "file", + "type": "NeptuneFileDataSet", + "version": "None", + } + assert run.exists("kedro/catalog/files/planets@neptune") + run["kedro/catalog/files/planets@neptune"].download("/tmp/file") + with open("/tmp/file", "rb") as handler: + file_content = handler.read() + assert hashlib.md5(file_content).hexdigest() == "af37712c8c80afc9690e4b70b7a590c5" + + assert run.exists("kedro/catalog/files/logo") + run["kedro/catalog/files/logo"].download("/tmp/file") + with open("/tmp/file", "rb") as handler: + file_content = handler.read() + assert hashlib.md5(file_content).hexdigest() == "85d289d3ed3f321557b6c428b7b35a67" + + assert run.exists("kedro/catalog/parameters/travel_speed") + assert run["kedro/catalog/parameters/travel_speed"].fetch() == travel_speed + + # Nodes data + check_node_metadata( + run=run, node_namespace="kedro/nodes/distances", inputs=["planets"], outputs=["distances_to_planets"] + ) + check_node_metadata( + run=run, + node_namespace="kedro/nodes/furthest", + inputs=["distances_to_planets"], + outputs=["furthest_planet_distance", "furthest_planet_name"], + ) + check_node_metadata(run=run, node_namespace="kedro/nodes/judge_model", inputs=["neptune_run", "dataset"]) + check_node_metadata( + run=run, node_namespace="kedro/nodes/prepare_dataset", inputs=["planets"], outputs=["dataset"] + ) + check_node_metadata( + run=run, + node_namespace="kedro/nodes/travel_time", + inputs=["furthest_planet_distance", "furthest_planet_name", "params:travel_speed"], + outputs=["travel_hours"], + ) + assert run.exists("kedro/nodes/travel_time/parameters") + assert run.exists("kedro/nodes/travel_time/parameters/travel_speed") + assert run["kedro/nodes/travel_time/parameters/travel_speed"].fetch() == travel_speed + + # User defined data + assert run.exists("furthest_planet") + assert run.exists("furthest_planet/name") + assert run.exists("furthest_planet/travel_days") + assert run.exists("furthest_planet/travel_hours") + assert run.exists("furthest_planet/travel_months") + assert run.exists("furthest_planet/travel_years") + assert run["furthest_planet/name"].fetch() == "NEPTUNE" def restore_run(): From b287369884d2b9b3648c7e262676d9fee5e3eba2 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:45:59 +0100 Subject: [PATCH 28/40] backoff only 1 retry --- tests/kedro_neptune/utils/run_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 5580267..f352fa1 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -36,7 +36,7 @@ # It may take some time to refresh cache -@backoff.on_exception(backoff.expo, AssertionError, max_value=10) +@backoff.on_exception(backoff.expo, AssertionError, max_value=1, max_time=60) def assert_structure(travel_speed: int = 10000): with init_run( capture_stderr=False, From f417bbd9dcd5cd7049d8ff426f929d094f6a5d62 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 22:49:07 +0100 Subject: [PATCH 29/40] add option to pytest to not capture stdout --- .github/actions/e2e/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 67ebb1b..dff86b6 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,7 +27,7 @@ runs: - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest ../../tests/kedro_neptune/test_standard.py + run: export PYTHONPATH=$PWD/src; pytest -s ../../tests/kedro_neptune/test_standard.py shell: bash # - name: Test parallel run From 0074629009d4eec06a58fa690715fd104fa366b3 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:04:31 +0100 Subject: [PATCH 30/40] try with sleep again --- tests/kedro_neptune/utils/run_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index f352fa1..870e360 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -17,14 +17,13 @@ import hashlib import os +import time from ast import literal_eval from typing import ( List, Optional, ) -import backoff - try: from neptune import ( Run, @@ -36,8 +35,9 @@ # It may take some time to refresh cache -@backoff.on_exception(backoff.expo, AssertionError, max_value=1, max_time=60) +# @backoff.on_exception(backoff.expo, AssertionError, max_value=1, max_time=60) def assert_structure(travel_speed: int = 10000): + time.sleep(120) with init_run( capture_stderr=False, capture_stdout=False, @@ -45,7 +45,7 @@ def assert_structure(travel_speed: int = 10000): capture_traceback=False, source_files=[], ) as run: - + run.sync(wait=True) # Base run information assert run.exists("kedro") assert run.exists("kedro/catalog") From 8cb879f8617d435cfe4e3b9de856537b89a094ab Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:34:51 +0100 Subject: [PATCH 31/40] temporarily disable dict comparison assertions --- tests/kedro_neptune/utils/run_utils.py | 35 +++++++++++++------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 870e360..a447721 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -16,7 +16,6 @@ __all__ = ["assert_structure"] import hashlib -import os import time from ast import literal_eval from typing import ( @@ -37,7 +36,7 @@ # It may take some time to refresh cache # @backoff.on_exception(backoff.expo, AssertionError, max_value=1, max_time=60) def assert_structure(travel_speed: int = 10000): - time.sleep(120) + time.sleep(30) with init_run( capture_stderr=False, capture_stdout=False, @@ -60,24 +59,24 @@ def assert_structure(travel_speed: int = 10000): assert run.exists("kedro/catalog/parameters") assert run.exists("kedro/catalog/datasets/planets") - assert run["kedro/catalog/datasets/planets"].fetch() == { - "filepath": f"{os.getcwd()}/data/planets/planets.csv", - "name": "planets", - "protocol": "file", - "save_args": {"index": False}, - "type": "CSVDataSet", - "version": "None", - } + # assert run["kedro/catalog/datasets/planets"].fetch() == { + # "filepath": f"{os.getcwd()}/data/planets/planets.csv", + # "name": "planets", + # "protocol": "file", + # "save_args": {"index": False}, + # "type": "CSVDataSet", + # "version": "None", + # } assert run.exists("kedro/catalog/datasets/planets@neptune") - assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { - "extension": "csv", - "filepath": f"{os.getcwd()}/data/planets/planets.csv", - "name": "planets@neptune", - "protocol": "file", - "type": "NeptuneFileDataSet", - "version": "None", - } + # assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { + # "extension": "csv", + # "filepath": f"{os.getcwd()}/data/planets/planets.csv", + # "name": "planets@neptune", + # "protocol": "file", + # "type": "NeptuneFileDataSet", + # "version": "None", + # } assert run.exists("kedro/catalog/files/planets@neptune") run["kedro/catalog/files/planets@neptune"].download("/tmp/file") with open("/tmp/file", "rb") as handler: From 1bef7b5be7df87e730d62f0d6dabf72157fcfc06 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:38:58 +0100 Subject: [PATCH 32/40] peek into cwd --- tests/kedro_neptune/utils/run_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index a447721..56d43e3 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -16,6 +16,7 @@ __all__ = ["assert_structure"] import hashlib +import os import time from ast import literal_eval from typing import ( @@ -59,6 +60,8 @@ def assert_structure(travel_speed: int = 10000): assert run.exists("kedro/catalog/parameters") assert run.exists("kedro/catalog/datasets/planets") + print("######################################################################") + print("OS CWD:", os.getcwd()) # assert run["kedro/catalog/datasets/planets"].fetch() == { # "filepath": f"{os.getcwd()}/data/planets/planets.csv", # "name": "planets", From d41acd35a2054f2e5fa96c88bd0d8948c9649047 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:41:52 +0100 Subject: [PATCH 33/40] peek into fetched dict --- tests/kedro_neptune/utils/run_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 56d43e3..d0c1ce4 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -62,6 +62,7 @@ def assert_structure(travel_speed: int = 10000): assert run.exists("kedro/catalog/datasets/planets") print("######################################################################") print("OS CWD:", os.getcwd()) + print(run["kedro/catalog/datasets/planets"].fetch()) # assert run["kedro/catalog/datasets/planets"].fetch() == { # "filepath": f"{os.getcwd()}/data/planets/planets.csv", # "name": "planets", From 52afeabee74099922eea4b2ac7a6f82e3c56593a Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:46:48 +0100 Subject: [PATCH 34/40] some ci debbuging --- tests/kedro_neptune/utils/run_utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index d0c1ce4..d38db97 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -62,7 +62,19 @@ def assert_structure(travel_speed: int = 10000): assert run.exists("kedro/catalog/datasets/planets") print("######################################################################") print("OS CWD:", os.getcwd()) - print(run["kedro/catalog/datasets/planets"].fetch()) + d1 = run["kedro/catalog/datasets/planets"].fetch() + d2 = { + "filepath": f"{os.getcwd()}/data/planets/planets.csv", + "name": "planets", + "protocol": "file", + "save_args": {"index": False}, + "type": "CSVDataSet", + "version": "None", + } + print(d1) + print(d2) + print(d1 == d2) + # assert run["kedro/catalog/datasets/planets"].fetch() == { # "filepath": f"{os.getcwd()}/data/planets/planets.csv", # "name": "planets", From db3072fbc12ed148824e5c1adb98e08f33e30474 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:50:36 +0100 Subject: [PATCH 35/40] uncomment dicts --- tests/kedro_neptune/utils/run_utils.py | 34 +++++++------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index d38db97..dcb3fd1 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -60,39 +60,23 @@ def assert_structure(travel_speed: int = 10000): assert run.exists("kedro/catalog/parameters") assert run.exists("kedro/catalog/datasets/planets") - print("######################################################################") - print("OS CWD:", os.getcwd()) - d1 = run["kedro/catalog/datasets/planets"].fetch() - d2 = { + + assert run["kedro/catalog/datasets/planets"].fetch() == { "filepath": f"{os.getcwd()}/data/planets/planets.csv", "name": "planets", - "protocol": "file", "save_args": {"index": False}, "type": "CSVDataSet", "version": "None", } - print(d1) - print(d2) - print(d1 == d2) - - # assert run["kedro/catalog/datasets/planets"].fetch() == { - # "filepath": f"{os.getcwd()}/data/planets/planets.csv", - # "name": "planets", - # "protocol": "file", - # "save_args": {"index": False}, - # "type": "CSVDataSet", - # "version": "None", - # } assert run.exists("kedro/catalog/datasets/planets@neptune") - # assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { - # "extension": "csv", - # "filepath": f"{os.getcwd()}/data/planets/planets.csv", - # "name": "planets@neptune", - # "protocol": "file", - # "type": "NeptuneFileDataSet", - # "version": "None", - # } + assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { + "extension": "csv", + "filepath": f"{os.getcwd()}/data/planets/planets.csv", + "name": "planets@neptune", + "type": "NeptuneFileDataSet", + "version": "None", + } assert run.exists("kedro/catalog/files/planets@neptune") run["kedro/catalog/files/planets@neptune"].download("/tmp/file") with open("/tmp/file", "rb") as handler: From 52dfc44d5d7b1d310f23c8851c4330276a3e5b52 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:55:29 +0100 Subject: [PATCH 36/40] some more debugging --- tests/kedro_neptune/utils/run_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index dcb3fd1..8ff1bd9 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -70,13 +70,24 @@ def assert_structure(travel_speed: int = 10000): } assert run.exists("kedro/catalog/datasets/planets@neptune") - assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { + d1 = run["kedro/catalog/datasets/planets@neptune"].fetch() + d2 = { "extension": "csv", "filepath": f"{os.getcwd()}/data/planets/planets.csv", "name": "planets@neptune", "type": "NeptuneFileDataSet", "version": "None", } + print(d1) + print(d2) + print(d1 == d2) + # assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { + # "extension": "csv", + # "filepath": f"{os.getcwd()}/data/planets/planets.csv", + # "name": "planets@neptune", + # "type": "NeptuneFileDataSet", + # "version": "None", + # } assert run.exists("kedro/catalog/files/planets@neptune") run["kedro/catalog/files/planets@neptune"].download("/tmp/file") with open("/tmp/file", "rb") as handler: From 23d845c26e6f9bda9e450c5aa8fa83f228deb6b5 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Thu, 9 Mar 2023 23:58:05 +0100 Subject: [PATCH 37/40] fix dict field --- tests/kedro_neptune/utils/run_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 8ff1bd9..017efc1 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -75,6 +75,7 @@ def assert_structure(travel_speed: int = 10000): "extension": "csv", "filepath": f"{os.getcwd()}/data/planets/planets.csv", "name": "planets@neptune", + "protocol": "file", "type": "NeptuneFileDataSet", "version": "None", } From 10bddca1cf45fda813dda2196380a4dfa8b3ddd2 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Fri, 10 Mar 2023 00:01:08 +0100 Subject: [PATCH 38/40] uncomment dicts again --- tests/kedro_neptune/utils/run_utils.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 017efc1..026e258 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -70,8 +70,7 @@ def assert_structure(travel_speed: int = 10000): } assert run.exists("kedro/catalog/datasets/planets@neptune") - d1 = run["kedro/catalog/datasets/planets@neptune"].fetch() - d2 = { + assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { "extension": "csv", "filepath": f"{os.getcwd()}/data/planets/planets.csv", "name": "planets@neptune", @@ -79,16 +78,6 @@ def assert_structure(travel_speed: int = 10000): "type": "NeptuneFileDataSet", "version": "None", } - print(d1) - print(d2) - print(d1 == d2) - # assert run["kedro/catalog/datasets/planets@neptune"].fetch() == { - # "extension": "csv", - # "filepath": f"{os.getcwd()}/data/planets/planets.csv", - # "name": "planets@neptune", - # "type": "NeptuneFileDataSet", - # "version": "None", - # } assert run.exists("kedro/catalog/files/planets@neptune") run["kedro/catalog/files/planets@neptune"].download("/tmp/file") with open("/tmp/file", "rb") as handler: From 6c4bfbee57f423ecee9bce8ccd8822ef4d908222 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Fri, 10 Mar 2023 00:04:51 +0100 Subject: [PATCH 39/40] cleanup --- .github/actions/e2e/action.yml | 10 +++++----- pyproject.toml | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index dff86b6..ea7ccb9 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -27,10 +27,10 @@ runs: - name: Test standard working-directory: ${{ inputs.working_directory }}/examples/planets - run: export PYTHONPATH=$PWD/src; pytest -s ../../tests/kedro_neptune/test_standard.py + run: export PYTHONPATH=$PWD/src; pytest ../../tests -m "not parallel" shell: bash -# - name: Test parallel run -# working-directory: ${{ inputs.working_directory }}/examples/planets -# run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel -# shell: bash + - name: Test parallel run + working-directory: ${{ inputs.working_directory }}/examples/planets + run: export PYTHONPATH=$PWD/src; pytest ../../tests -m parallel + shell: bash diff --git a/pyproject.toml b/pyproject.toml index b3fd7ae..897edee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ kedro = ">=0.18.0" pre-commit = { version = "*", optional = true } pytest = { version = ">=5.0", optional = true } pytest-cov = { version = "2.10.1", optional = true } -backoff = { version = "*", optional = true } neptune = { version = ">=1.0.0", optional = true } [tool.poetry.extras] @@ -30,7 +29,6 @@ dev = [ "pre-commit", "pytest", "pytest-cov", - "backoff", "neptune", ] From 3a1a4abdde172a7985c850791b738ba2940b7e59 Mon Sep 17 00:00:00 2001 From: Aleksander Wojnarowicz Date: Fri, 10 Mar 2023 08:39:46 +0100 Subject: [PATCH 40/40] apply review suggestions --- .github/workflows/ci.yml | 2 +- tests/kedro_neptune/utils/run_utils.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6ee4ef..3fb5bdb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: - name: Test env: NEPTUNE_API_TOKEN: ${{ secrets.E2E_NEPTUNE_API_TOKEN }} - NEPTUNE_PROJECT: e2e-tests/e2e + NEPTUNE_PROJECT: e2e-tests/integrations uses: ./.github/actions/e2e publish: diff --git a/tests/kedro_neptune/utils/run_utils.py b/tests/kedro_neptune/utils/run_utils.py index 026e258..71e5eb0 100644 --- a/tests/kedro_neptune/utils/run_utils.py +++ b/tests/kedro_neptune/utils/run_utils.py @@ -38,13 +38,7 @@ # @backoff.on_exception(backoff.expo, AssertionError, max_value=1, max_time=60) def assert_structure(travel_speed: int = 10000): time.sleep(30) - with init_run( - capture_stderr=False, - capture_stdout=False, - capture_hardware_metrics=False, - capture_traceback=False, - source_files=[], - ) as run: + with restore_run() as run: run.sync(wait=True) # Base run information assert run.exists("kedro")