From 4f1fa1fa55ef37f5cedbeeb36ab9c818105bc6d5 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 27 Mar 2024 15:36:18 -0500 Subject: [PATCH 1/4] resolve posix path warning Signed-off-by: ravi-kumar-pilla --- .../src/demo_project/pipelines/data_ingestion/nodes.py | 4 ++-- package/kedro_viz/integrations/kedro/hooks.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/demo-project/src/demo_project/pipelines/data_ingestion/nodes.py b/demo-project/src/demo_project/pipelines/data_ingestion/nodes.py index 020304299b..e26b5d6285 100755 --- a/demo-project/src/demo_project/pipelines/data_ingestion/nodes.py +++ b/demo-project/src/demo_project/pipelines/data_ingestion/nodes.py @@ -86,9 +86,9 @@ def aggregate_company_data(typed_companies: pd.DataFrame) -> pd.DataFrame: working_companies = typed_companies.groupby(["id"]).agg( { - "company_rating": np.mean, + "company_rating": "mean", "company_location": lambda x: list(set(x))[0], # Take first item - "total_fleet_count": max, + "total_fleet_count": "max", "iata_approved": any, } ) diff --git a/package/kedro_viz/integrations/kedro/hooks.py b/package/kedro_viz/integrations/kedro/hooks.py index da1f6fecba..4f49955a7d 100644 --- a/package/kedro_viz/integrations/kedro/hooks.py +++ b/package/kedro_viz/integrations/kedro/hooks.py @@ -5,6 +5,7 @@ import json import logging from collections import defaultdict +from pathlib import Path from typing import Any, Union from kedro.framework.hooks import hook_impl @@ -134,7 +135,7 @@ def get_file_size(self, dataset: Any) -> Union[int, None]: return None try: - file_path = get_filepath_str(dataset._filepath, dataset._protocol) + file_path = get_filepath_str(Path(dataset._filepath), dataset._protocol) return dataset._fs.size(file_path) except Exception as exc: From 3aafecfdbc67b09e18586b8db39bf79aebf5af1d Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Tue, 2 Apr 2024 15:02:34 -0500 Subject: [PATCH 2/4] debug draft Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/integrations/kedro/hooks.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/package/kedro_viz/integrations/kedro/hooks.py b/package/kedro_viz/integrations/kedro/hooks.py index 4f49955a7d..e8cb78f099 100644 --- a/package/kedro_viz/integrations/kedro/hooks.py +++ b/package/kedro_viz/integrations/kedro/hooks.py @@ -6,12 +6,13 @@ import logging from collections import defaultdict from pathlib import Path -from typing import Any, Union +from typing import Any, Dict, Union from kedro.framework.hooks import hook_impl from kedro.io import DataCatalog from kedro.io.core import get_filepath_str from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding +from kedro.runner.parallel_runner import ParallelRunner logger = logging.getLogger(__name__) @@ -38,6 +39,26 @@ def after_catalog_created(self, catalog: DataCatalog): # Support for Kedro 0.18.x self.datasets = catalog._data_sets # type: ignore[attr-defined] + @hook_impl + def before_pipeline_run( + self, run_params: Dict[str, Any], pipeline, catalog, **kwargs + ) -> None: + """Hook implementation to start an MLflow run + with the session_id of the Kedro pipeline run. + """ + import pdb + + pdb.set_trace() + + pipeline_runner_obj = run_params["runner"] + print(run_params) + if isinstance(pipeline_runner_obj, ParallelRunner): + print(pipeline_runner_obj._manager) + runnerManager = pipeline_runner_obj._manager + self._stats = runnerManager.dict() + else: + print("Sq Runner") + @hook_impl def after_dataset_loaded(self, dataset_name: str, data: Any): """Hook to be invoked after a dataset is loaded from the catalog. From 216dfbe84d678cf1b03ec73a40abb8f1171fa07d Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Tue, 2 Apr 2024 22:13:28 -0500 Subject: [PATCH 3/4] revert debugging Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/integrations/kedro/hooks.py | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/package/kedro_viz/integrations/kedro/hooks.py b/package/kedro_viz/integrations/kedro/hooks.py index e8cb78f099..4f49955a7d 100644 --- a/package/kedro_viz/integrations/kedro/hooks.py +++ b/package/kedro_viz/integrations/kedro/hooks.py @@ -6,13 +6,12 @@ import logging from collections import defaultdict from pathlib import Path -from typing import Any, Dict, Union +from typing import Any, Union from kedro.framework.hooks import hook_impl from kedro.io import DataCatalog from kedro.io.core import get_filepath_str from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding -from kedro.runner.parallel_runner import ParallelRunner logger = logging.getLogger(__name__) @@ -39,26 +38,6 @@ def after_catalog_created(self, catalog: DataCatalog): # Support for Kedro 0.18.x self.datasets = catalog._data_sets # type: ignore[attr-defined] - @hook_impl - def before_pipeline_run( - self, run_params: Dict[str, Any], pipeline, catalog, **kwargs - ) -> None: - """Hook implementation to start an MLflow run - with the session_id of the Kedro pipeline run. - """ - import pdb - - pdb.set_trace() - - pipeline_runner_obj = run_params["runner"] - print(run_params) - if isinstance(pipeline_runner_obj, ParallelRunner): - print(pipeline_runner_obj._manager) - runnerManager = pipeline_runner_obj._manager - self._stats = runnerManager.dict() - else: - print("Sq Runner") - @hook_impl def after_dataset_loaded(self, dataset_name: str, data: Any): """Hook to be invoked after a dataset is loaded from the catalog. From 6a315999ebe5c2750364929bc595c0e9535d28c4 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 3 Apr 2024 16:34:19 -0500 Subject: [PATCH 4/4] update release note Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index ae62fb6b0f..792cceda40 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,7 @@ Please follow the established format: - Increase Kedro-Viz timeout. (#1803) - Remove demo data source and update feature hints. (#1804) - Add markdown support for backticks in the pop-up reminder. (#1826) +- Fix posix path conversion on Windows in DatasetStatsHook. (#1843) # Release 8.0.1