From 365ca6dc6cdc51e8159a5d1d2a8dca575324b3c8 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 22 Nov 2023 11:00:06 +0100 Subject: [PATCH] Update test examples --- src/fondant/component/executor.py | 2 -- .../component_specs}/arguments/component.yaml | 0 .../arguments/component_default_args.yaml | 0 .../component_specs}/arguments/input_manifest.json | 2 +- .../examples/component_specs}/component.yaml | 0 .../examples/component_specs}/input_manifest.json | 0 .../examples/data}/components/1.yaml | 0 .../examples/data}/manifest.json | 2 +- .../examples/data}/raw/split.py | 0 .../examples/data}/raw/testset.parquet | Bin .../data}/subsets_input/index/part.0.parquet | Bin .../data}/subsets_input/index/part.1.parquet | Bin .../data}/subsets_input/index/part.2.parquet | Bin .../data}/subsets_input/properties/part.0.parquet | Bin .../data}/subsets_input/properties/part.1.parquet | Bin .../data}/subsets_input/properties/part.2.parquet | Bin .../data}/subsets_input/types/part.0.parquet | Bin .../data}/subsets_input/types/part.1.parquet | Bin .../data}/subsets_input/types/part.2.parquet | Bin .../mock_base_path/example_pipeline/cache/42.txt | 1 + .../example_pipeline_2023/component_1/manifest.json | 0 .../example_pipeline_2023/component_2/manifest.json | 0 .../example_pipeline_2024/component_1/manifest.json | 0 .../example_pipeline_2024/component_2/manifest.json | 0 tests/component/test_component.py | 4 ++-- tests/component/test_data_io.py | 4 ++-- .../mock_base_path/example_pipeline/cache/42.txt | 1 - 27 files changed, 7 insertions(+), 9 deletions(-) rename tests/{examples/example_specs/components => component/examples/component_specs}/arguments/component.yaml (100%) rename tests/{examples/example_specs/components => component/examples/component_specs}/arguments/component_default_args.yaml (100%) rename tests/{examples/example_specs/components => component/examples/component_specs}/arguments/input_manifest.json (81%) rename tests/{examples/example_specs/components => component/examples/component_specs}/component.yaml (100%) rename tests/{examples/example_specs/components => component/examples/component_specs}/input_manifest.json (100%) rename tests/{examples/example_data => component/examples/data}/components/1.yaml (100%) rename tests/{examples/example_data => component/examples/data}/manifest.json (91%) rename tests/{examples/example_data => component/examples/data}/raw/split.py (100%) rename tests/{examples/example_data => component/examples/data}/raw/testset.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/index/part.0.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/index/part.1.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/index/part.2.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/properties/part.0.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/properties/part.1.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/properties/part.2.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/types/part.0.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/types/part.1.parquet (100%) rename tests/{examples/example_data => component/examples/data}/subsets_input/types/part.2.parquet (100%) create mode 100644 tests/component/examples/mock_base_path/example_pipeline/cache/42.txt rename tests/{examples/example_specs => component/examples}/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json (100%) rename tests/{examples/example_specs => component/examples}/mock_base_path/example_pipeline/example_pipeline_2023/component_2/manifest.json (100%) rename tests/{examples/example_specs => component/examples}/mock_base_path/example_pipeline/example_pipeline_2024/component_1/manifest.json (100%) rename tests/{examples/example_specs => component/examples}/mock_base_path/example_pipeline/example_pipeline_2024/component_2/manifest.json (100%) delete mode 100644 tests/examples/example_specs/mock_base_path/example_pipeline/cache/42.txt diff --git a/src/fondant/component/executor.py b/src/fondant/component/executor.py index bed4df80d..d77200da8 100644 --- a/src/fondant/component/executor.py +++ b/src/fondant/component/executor.py @@ -549,8 +549,6 @@ def _execute_component( # Clear divisions if component spec indicates that the index is changed if self._infer_index_change(): - # TODO: might causing issues for merging components - # to guarantee fast merging of large dataframes we need to keep the division information dataframe.clear_divisions() return dataframe diff --git a/tests/examples/example_specs/components/arguments/component.yaml b/tests/component/examples/component_specs/arguments/component.yaml similarity index 100% rename from tests/examples/example_specs/components/arguments/component.yaml rename to tests/component/examples/component_specs/arguments/component.yaml diff --git a/tests/examples/example_specs/components/arguments/component_default_args.yaml b/tests/component/examples/component_specs/arguments/component_default_args.yaml similarity index 100% rename from tests/examples/example_specs/components/arguments/component_default_args.yaml rename to tests/component/examples/component_specs/arguments/component_default_args.yaml diff --git a/tests/examples/example_specs/components/arguments/input_manifest.json b/tests/component/examples/component_specs/arguments/input_manifest.json similarity index 81% rename from tests/examples/example_specs/components/arguments/input_manifest.json rename to tests/component/examples/component_specs/arguments/input_manifest.json index 9ee2494f9..526faf72f 100644 --- a/tests/examples/example_specs/components/arguments/input_manifest.json +++ b/tests/component/examples/component_specs/arguments/input_manifest.json @@ -1,7 +1,7 @@ { "metadata": { "pipeline_name": "example_pipeline", - "base_path": "tests/example_data/subsets_input/mock_base_path", + "base_path": "tests/component/examples/mock_base_path", "run_id": "example_pipeline_123", "component_id": "component_1", "cache_key": "00" diff --git a/tests/examples/example_specs/components/component.yaml b/tests/component/examples/component_specs/component.yaml similarity index 100% rename from tests/examples/example_specs/components/component.yaml rename to tests/component/examples/component_specs/component.yaml diff --git a/tests/examples/example_specs/components/input_manifest.json b/tests/component/examples/component_specs/input_manifest.json similarity index 100% rename from tests/examples/example_specs/components/input_manifest.json rename to tests/component/examples/component_specs/input_manifest.json diff --git a/tests/examples/example_data/components/1.yaml b/tests/component/examples/data/components/1.yaml similarity index 100% rename from tests/examples/example_data/components/1.yaml rename to tests/component/examples/data/components/1.yaml diff --git a/tests/examples/example_data/manifest.json b/tests/component/examples/data/manifest.json similarity index 91% rename from tests/examples/example_data/manifest.json rename to tests/component/examples/data/manifest.json index 14366fbbd..cc579fef1 100644 --- a/tests/examples/example_data/manifest.json +++ b/tests/component/examples/data/manifest.json @@ -1,7 +1,7 @@ { "metadata": { "pipeline_name": "test_pipeline", - "base_path": "tests/examples/example_data", + "base_path": "tests/component/examples/data", "run_id": "test_pipeline_12345", "component_id": "67890" }, diff --git a/tests/examples/example_data/raw/split.py b/tests/component/examples/data/raw/split.py similarity index 100% rename from tests/examples/example_data/raw/split.py rename to tests/component/examples/data/raw/split.py diff --git a/tests/examples/example_data/raw/testset.parquet b/tests/component/examples/data/raw/testset.parquet similarity index 100% rename from tests/examples/example_data/raw/testset.parquet rename to tests/component/examples/data/raw/testset.parquet diff --git a/tests/examples/example_data/subsets_input/index/part.0.parquet b/tests/component/examples/data/subsets_input/index/part.0.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/index/part.0.parquet rename to tests/component/examples/data/subsets_input/index/part.0.parquet diff --git a/tests/examples/example_data/subsets_input/index/part.1.parquet b/tests/component/examples/data/subsets_input/index/part.1.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/index/part.1.parquet rename to tests/component/examples/data/subsets_input/index/part.1.parquet diff --git a/tests/examples/example_data/subsets_input/index/part.2.parquet b/tests/component/examples/data/subsets_input/index/part.2.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/index/part.2.parquet rename to tests/component/examples/data/subsets_input/index/part.2.parquet diff --git a/tests/examples/example_data/subsets_input/properties/part.0.parquet b/tests/component/examples/data/subsets_input/properties/part.0.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/properties/part.0.parquet rename to tests/component/examples/data/subsets_input/properties/part.0.parquet diff --git a/tests/examples/example_data/subsets_input/properties/part.1.parquet b/tests/component/examples/data/subsets_input/properties/part.1.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/properties/part.1.parquet rename to tests/component/examples/data/subsets_input/properties/part.1.parquet diff --git a/tests/examples/example_data/subsets_input/properties/part.2.parquet b/tests/component/examples/data/subsets_input/properties/part.2.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/properties/part.2.parquet rename to tests/component/examples/data/subsets_input/properties/part.2.parquet diff --git a/tests/examples/example_data/subsets_input/types/part.0.parquet b/tests/component/examples/data/subsets_input/types/part.0.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/types/part.0.parquet rename to tests/component/examples/data/subsets_input/types/part.0.parquet diff --git a/tests/examples/example_data/subsets_input/types/part.1.parquet b/tests/component/examples/data/subsets_input/types/part.1.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/types/part.1.parquet rename to tests/component/examples/data/subsets_input/types/part.1.parquet diff --git a/tests/examples/example_data/subsets_input/types/part.2.parquet b/tests/component/examples/data/subsets_input/types/part.2.parquet similarity index 100% rename from tests/examples/example_data/subsets_input/types/part.2.parquet rename to tests/component/examples/data/subsets_input/types/part.2.parquet diff --git a/tests/component/examples/mock_base_path/example_pipeline/cache/42.txt b/tests/component/examples/mock_base_path/example_pipeline/cache/42.txt new file mode 100644 index 000000000..4a9ff8afc --- /dev/null +++ b/tests/component/examples/mock_base_path/example_pipeline/cache/42.txt @@ -0,0 +1 @@ +tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json \ No newline at end of file diff --git a/tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json b/tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json similarity index 100% rename from tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json rename to tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json diff --git a/tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2023/component_2/manifest.json b/tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_2/manifest.json similarity index 100% rename from tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2023/component_2/manifest.json rename to tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_2/manifest.json diff --git a/tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2024/component_1/manifest.json b/tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2024/component_1/manifest.json similarity index 100% rename from tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2024/component_1/manifest.json rename to tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2024/component_1/manifest.json diff --git a/tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2024/component_2/manifest.json b/tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2024/component_2/manifest.json similarity index 100% rename from tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2024/component_2/manifest.json rename to tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2024/component_2/manifest.json diff --git a/tests/component/test_component.py b/tests/component/test_component.py index 9f910dbcd..830ce2963 100644 --- a/tests/component/test_component.py +++ b/tests/component/test_component.py @@ -23,8 +23,8 @@ from fondant.core.component_spec import ComponentSpec from fondant.core.manifest import Manifest, Metadata -components_path = Path(__file__).parent.parent / "examples/example_specs/components" -base_path = Path(__file__).parent.parent / "examples/example_specs/mock_base_path" +components_path = Path(__file__).parent / "examples/component_specs" +base_path = Path(__file__).parent / "examples/mock_base_path" N_PARTITIONS = 2 diff --git a/tests/component/test_data_io.py b/tests/component/test_data_io.py index ed3e9169f..30a4b7c10 100644 --- a/tests/component/test_data_io.py +++ b/tests/component/test_data_io.py @@ -8,9 +8,9 @@ from fondant.core.component_spec import ComponentSpec from fondant.core.manifest import Manifest -manifest_path = Path(__file__).parent.parent / "examples/example_data/manifest.json" +manifest_path = Path(__file__).parent / "examples/data/manifest.json" component_spec_path = ( - Path(__file__).parent.parent / "examples/example_data/components/1.yaml" + Path(__file__).parent / "examples/data/components/1.yaml" ) NUMBER_OF_TEST_ROWS = 151 diff --git a/tests/examples/example_specs/mock_base_path/example_pipeline/cache/42.txt b/tests/examples/example_specs/mock_base_path/example_pipeline/cache/42.txt deleted file mode 100644 index 614074264..000000000 --- a/tests/examples/example_specs/mock_base_path/example_pipeline/cache/42.txt +++ /dev/null @@ -1 +0,0 @@ -tests/examples/example_specs/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json \ No newline at end of file