Update test examples

ml6team · Nov 22, 2023 · 365ca6d · 365ca6d
1 parent 0701662
commit 365ca6d
Show file tree

Hide file tree

Showing 27 changed files with 7 additions and 9 deletions.
diff --git a/src/fondant/component/executor.py b/src/fondant/component/executor.py
@@ -549,8 +549,6 @@ def _execute_component(
 
         # Clear divisions if component spec indicates that the index is changed
         if self._infer_index_change():
-            # TODO: might causing issues for merging components
-            # to guarantee fast merging of large dataframes we need to keep the division information
             dataframe.clear_divisions()
 
         return dataframe

diff --git a/...specs/components/arguments/component.yaml → .../component_specs/arguments/component.yaml b/...specs/components/arguments/component.yaml → .../component_specs/arguments/component.yaml
diff --git a/...nts/arguments/component_default_args.yaml → ...ecs/arguments/component_default_args.yaml b/...nts/arguments/component_default_args.yaml → ...ecs/arguments/component_default_args.yaml
diff --git a/.../components/arguments/input_manifest.json → ...onent_specs/arguments/input_manifest.json b/.../components/arguments/input_manifest.json → ...onent_specs/arguments/input_manifest.json
@@ -1,7 +1,7 @@
 {
   "metadata": {
     "pipeline_name": "example_pipeline",
-    "base_path": "tests/example_data/subsets_input/mock_base_path",
+    "base_path": "tests/component/examples/mock_base_path",
     "run_id": "example_pipeline_123",
     "component_id": "component_1",
     "cache_key": "00"

diff --git a/...s/example_specs/components/component.yaml → ...t/examples/component_specs/component.yaml b/...s/example_specs/components/component.yaml → ...t/examples/component_specs/component.yaml
diff --git a/...mple_specs/components/input_manifest.json → ...mples/component_specs/input_manifest.json b/...mple_specs/components/input_manifest.json → ...mples/component_specs/input_manifest.json
diff --git a/...s/examples/example_data/components/1.yaml → ...component/examples/data/components/1.yaml b/...s/examples/example_data/components/1.yaml → ...component/examples/data/components/1.yaml
diff --git a/tests/examples/example_data/manifest.json → tests/component/examples/data/manifest.json b/tests/examples/example_data/manifest.json → tests/component/examples/data/manifest.json
@@ -1,7 +1,7 @@
 {
   "metadata": {
     "pipeline_name": "test_pipeline",
-    "base_path": "tests/examples/example_data",
+    "base_path": "tests/component/examples/data",
     "run_id": "test_pipeline_12345",
     "component_id": "67890"
   },

diff --git a/tests/examples/example_data/raw/split.py → tests/component/examples/data/raw/split.py b/tests/examples/example_data/raw/split.py → tests/component/examples/data/raw/split.py
diff --git a/...examples/example_data/raw/testset.parquet → ...mponent/examples/data/raw/testset.parquet b/...examples/example_data/raw/testset.parquet → ...mponent/examples/data/raw/testset.parquet
diff --git a/...e_data/subsets_input/index/part.0.parquet → ...s/data/subsets_input/index/part.0.parquet b/...e_data/subsets_input/index/part.0.parquet → ...s/data/subsets_input/index/part.0.parquet
diff --git a/...e_data/subsets_input/index/part.1.parquet → ...s/data/subsets_input/index/part.1.parquet b/...e_data/subsets_input/index/part.1.parquet → ...s/data/subsets_input/index/part.1.parquet
diff --git a/...e_data/subsets_input/index/part.2.parquet → ...s/data/subsets_input/index/part.2.parquet b/...e_data/subsets_input/index/part.2.parquet → ...s/data/subsets_input/index/part.2.parquet
diff --git a/...a/subsets_input/properties/part.0.parquet → ...a/subsets_input/properties/part.0.parquet b/...a/subsets_input/properties/part.0.parquet → ...a/subsets_input/properties/part.0.parquet
diff --git a/...a/subsets_input/properties/part.1.parquet → ...a/subsets_input/properties/part.1.parquet b/...a/subsets_input/properties/part.1.parquet → ...a/subsets_input/properties/part.1.parquet
diff --git a/...a/subsets_input/properties/part.2.parquet → ...a/subsets_input/properties/part.2.parquet b/...a/subsets_input/properties/part.2.parquet → ...a/subsets_input/properties/part.2.parquet
diff --git a/...e_data/subsets_input/types/part.0.parquet → ...s/data/subsets_input/types/part.0.parquet b/...e_data/subsets_input/types/part.0.parquet → ...s/data/subsets_input/types/part.0.parquet
diff --git a/...e_data/subsets_input/types/part.1.parquet → ...s/data/subsets_input/types/part.1.parquet b/...e_data/subsets_input/types/part.1.parquet → ...s/data/subsets_input/types/part.1.parquet
diff --git a/...e_data/subsets_input/types/part.2.parquet → ...s/data/subsets_input/types/part.2.parquet b/...e_data/subsets_input/types/part.2.parquet → ...s/data/subsets_input/types/part.2.parquet
diff --git a/tests/component/examples/mock_base_path/example_pipeline/cache/42.txt b/tests/component/examples/mock_base_path/example_pipeline/cache/42.txt
@@ -0,0 +1 @@
+tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json
diff --git a/...e_pipeline_2023/component_1/manifest.json → ...e_pipeline_2023/component_1/manifest.json b/...e_pipeline_2023/component_1/manifest.json → ...e_pipeline_2023/component_1/manifest.json
diff --git a/...e_pipeline_2023/component_2/manifest.json → ...e_pipeline_2023/component_2/manifest.json b/...e_pipeline_2023/component_2/manifest.json → ...e_pipeline_2023/component_2/manifest.json
diff --git a/...e_pipeline_2024/component_1/manifest.json → ...e_pipeline_2024/component_1/manifest.json b/...e_pipeline_2024/component_1/manifest.json → ...e_pipeline_2024/component_1/manifest.json
diff --git a/...e_pipeline_2024/component_2/manifest.json → ...e_pipeline_2024/component_2/manifest.json b/...e_pipeline_2024/component_2/manifest.json → ...e_pipeline_2024/component_2/manifest.json
diff --git a/tests/component/test_component.py b/tests/component/test_component.py
@@ -23,8 +23,8 @@
 from fondant.core.component_spec import ComponentSpec
 from fondant.core.manifest import Manifest, Metadata
 
-components_path = Path(__file__).parent.parent / "examples/example_specs/components"
-base_path = Path(__file__).parent.parent / "examples/example_specs/mock_base_path"
+components_path = Path(__file__).parent / "examples/component_specs"
+base_path = Path(__file__).parent / "examples/mock_base_path"
 
 N_PARTITIONS = 2
 

diff --git a/tests/component/test_data_io.py b/tests/component/test_data_io.py
@@ -8,9 +8,9 @@
 from fondant.core.component_spec import ComponentSpec
 from fondant.core.manifest import Manifest
 
-manifest_path = Path(__file__).parent.parent / "examples/example_data/manifest.json"
+manifest_path = Path(__file__).parent / "examples/data/manifest.json"
 component_spec_path = (
-    Path(__file__).parent.parent / "examples/example_data/components/1.yaml"
+    Path(__file__).parent / "examples/data/components/1.yaml"
 )
 
 NUMBER_OF_TEST_ROWS = 151

diff --git a/tests/examples/example_specs/mock_base_path/example_pipeline/cache/42.txt b/tests/examples/example_specs/mock_base_path/example_pipeline/cache/42.txt
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		tests/component/examples/mock_base_path/example_pipeline/example_pipeline_2023/component_1/manifest.json