Tests

pyproject.toml

@@ -10,7 +10,6 @@ package-mode = false
 python = ">=3.11,<3.12"
 hydra-core = "^1.3.2"
 dvc = "^3.51.2"
-pytest = "^8.2.2"
 gdown = "^5.2.0"
 psycopg2-binary = "^2.9.9"
 pandas = "^2.2.2"
@@ -30,15 +29,27 @@ flask = "3.0.3"
 gradio = "3.50.2"
 imblearn = "^0.0"
 
+# Dev group
 [tool.poetry.group.dev]
 [tool.poetry.group.dev.dependencies]
 ruff = "^0.5.0"
 pre-commit = "^3.7.1"
 
 [tool.ruff]
-ignore = [
-    "E402",
-]
+ignore = ["E402"]
+
+# Test group
+[tool.poetry.group.test]
+[tool.poetry.group.test.dependencies]
+pytest = "^8.2.2"
+coverage = "^7.6.0"
+pytest-xdist = "^3.6.1"
+
+[tool.pytest.ini_options]
+addopts = "--disable-warnings"
+
+[tool.coverage.report]
+exclude_also = ["def __repr__"]
 
 [build-system]
 requires = ["poetry-core"]

requirements.txt

@@ -929,9 +929,6 @@ importlib-metadata==7.2.1 ; python_version >= "3.11" and python_version < "3.12"
 importlib-resources==6.4.0 ; python_version >= "3.11" and python_version < "3.12" \
     --hash=sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c \
     --hash=sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145
-iniconfig==2.0.0 ; python_version >= "3.11" and python_version < "3.12" \
-    --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
-    --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
 ipykernel==6.29.5 ; python_version >= "3.11" and python_version < "3.12" \
     --hash=sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5 \
     --hash=sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215
@@ -1604,9 +1601,6 @@ pillow==10.4.0 ; python_version >= "3.11" and python_version < "3.12" \
 platformdirs==3.11.0 ; python_version >= "3.11" and python_version < "3.12" \
     --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \
     --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e
-pluggy==1.5.0 ; python_version >= "3.11" and python_version < "3.12" \
-    --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \
-    --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669
 prometheus-client==0.20.0 ; python_version >= "3.11" and python_version < "3.12" \
     --hash=sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89 \
     --hash=sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7
@@ -1861,9 +1855,6 @@ pysocks==1.7.1 ; python_version >= "3.11" and python_version < "3.12" \
     --hash=sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299 \
     --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \
     --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0
-pytest==8.2.2 ; python_version >= "3.11" and python_version < "3.12" \
-    --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \
-    --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977
 python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "3.12" \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427

src/data.py

@@ -85,30 +85,22 @@ def sample_data(cfg: DictConfig):
 
 def validate_initial_data(cfg: DictConfig, df: pd.DataFrame):
     """
-    Validate the data using Great Expectations.
+    Validate initial data using Great Expectations.
     """
-    try:
-        context = gx.get_context(
-            project_root_dir=cfg.data.context_dir_path, mode="file"
-        )
-        ds: PandasDatasource = context.sources.add_or_update_pandas(name="sample_data")
-        ds.add_dataframe_asset(
-            name="sample_file",
-            dataframe=df,
-        )
-        checkpoint = context.get_checkpoint("sample_checkpoint")
+    context = gx.get_context(project_root_dir=cfg.data.context_dir_path, mode="file")
+    ds: PandasDatasource = context.sources.add_or_update_pandas(name="sample_data")
+    ds.add_dataframe_asset(
+        name="sample_file",
+        dataframe=df,
+    )
+    checkpoint = context.get_checkpoint("sample_checkpoint")
 
-        checkpoint_result = checkpoint.run()
+    checkpoint_result = checkpoint.run()
 
-        if checkpoint_result.success:
-            print("Validation successful.")
-            return True
-        else:
-            print("Validation failed.")
-            print(checkpoint_result)
-    except Exception as e:
-        print("Error in validating the data: ", e)
-    return False
+    if checkpoint_result.success:
+        print("Validation successful.")
+    else:
+        raise ValueError("Validation failed: ", checkpoint_result)
 
 
 def read_datastore() -> tuple[pd.DataFrame, str]:
@@ -189,7 +181,7 @@ def validate_features(
     X: pd.DataFrame, y: pd.DataFrame
 ) -> tuple[pd.DataFrame, pd.DataFrame]:
     """
-    Validate the data using Great Expectations.
+    Validate the data post-transformation using Great Expectations.
     """
     cfg = init_hydra("main")
     context = gx.get_context(project_root_dir=cfg.data.context_dir_path, mode="file")