From fa163a8675d9ee86c53cffd21e4240fad06a0f96 Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Wed, 4 Sep 2024 14:11:13 +0530 Subject: [PATCH 01/11] bump version --- src/zenml/integrations/deepchecks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zenml/integrations/deepchecks/__init__.py b/src/zenml/integrations/deepchecks/__init__.py index 7bde3008272..d0ac5eb5e50 100644 --- a/src/zenml/integrations/deepchecks/__init__.py +++ b/src/zenml/integrations/deepchecks/__init__.py @@ -35,7 +35,7 @@ class DeepchecksIntegration(Integration): NAME = DEEPCHECKS REQUIREMENTS = [ - "deepchecks[vision]>=0.18.0", + "deepchecks[vision]~=0.18.0", "torchvision>=0.14.0", "opencv-python==4.5.5.64", # pin to same version "opencv-python-headless==4.5.5.64", # pin to same version From 389814f9dc0e768982abc8094e8c5a949b83cfa2 Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Wed, 4 Sep 2024 14:11:27 +0530 Subject: [PATCH 02/11] update data checks --- .../deepchecks/validation_checks.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/zenml/integrations/deepchecks/validation_checks.py b/src/zenml/integrations/deepchecks/validation_checks.py index 711fd380cc8..b3ce878478e 100644 --- a/src/zenml/integrations/deepchecks/validation_checks.py +++ b/src/zenml/integrations/deepchecks/validation_checks.py @@ -153,8 +153,8 @@ class DeepchecksDataIntegrityCheck(DeepchecksValidationCheck): This list reflects the set of data integrity checks provided by Deepchecks: - * [for tabular data](https://docs.deepchecks.com/en/stable/checks_gallery/tabular.html#data-integrity) - * [for computer vision](https://docs.deepchecks.com/en/stable/checks_gallery/vision.html#data-integrity) + * [for tabular data](https://docs.deepchecks.com/stable/tabular/auto_checks/data_integrity/index.html) + * [for computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/data_integrity/index.html) All these checks inherit from `deepchecks.tabular.SingleDatasetCheck` or `deepchecks.vision.SingleDatasetCheck` and require a single dataset as input. @@ -176,6 +176,9 @@ class DeepchecksDataIntegrityCheck(DeepchecksValidationCheck): TABULAR_FEATURE_LABEL_CORRELATION = source_utils.resolve( tabular_checks.FeatureLabelCorrelation ).import_path + TABULAR_IDENTIFIER_LABEL_CORRELATION = source_utils.resolve( + tabular_checks.IdentifierLabelCorrelation + ).import_path TABULAR_IS_SINGLE_VALUE = source_utils.resolve( tabular_checks.IsSingleValue ).import_path @@ -197,6 +200,12 @@ class DeepchecksDataIntegrityCheck(DeepchecksValidationCheck): TABULAR_STRING_MISMATCH = source_utils.resolve( tabular_checks.StringMismatch ).import_path + TABULAR_CLASS_IMBALANCE = source_utils.resolve( + tabular_checks.ClassImbalance + ).import_path + TABULAR_PERCENT_OF_NULLS = source_utils.resolve( + tabular_checks.PercentOfNulls + ).import_path VISION_IMAGE_PROPERTY_OUTLIERS = source_utils.resolve( vision_checks.ImagePropertyOutliers @@ -204,6 +213,9 @@ class DeepchecksDataIntegrityCheck(DeepchecksValidationCheck): VISION_LABEL_PROPERTY_OUTLIERS = source_utils.resolve( vision_checks.LabelPropertyOutliers ).import_path + VISION_PROPERTY_LABEL_CORRELATION = source_utils.resolve( + vision_checks.PropertyLabelCorrelation + ).import_path class DeepchecksDataDriftCheck(DeepchecksValidationCheck): @@ -255,10 +267,22 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck): TABULAR_WHOLE_DATASET_DRIFT = source_utils.resolve( tabular_checks.WholeDatasetDrift ).import_path + TABULAR_NEW_CATEGORY_TRAIN_TEST = source_utils.resolve( + tabular_checks.NewCategoryTrainTest + ).import_path + TABULAR_MULTIVARIATE_DRIFT = source_utils.resolve( + tabular_checks.MultivariateDrift + ).import_path + VISION_PROPERTY_LABEL_CORRELATION_CHANGE = source_utils.resolve( + vision_checks.PropertyLabelCorrelationChange + ).import_path VISION_HEATMAP_COMPARISON = source_utils.resolve( vision_checks.HeatmapComparison ).import_path + VISION_LABEL_DRIFT = source_utils.resolve( + vision_checks.LabelDrift + ).import_path VISION_IMAGE_DATASET_DRIFT = source_utils.resolve( vision_checks.ImageDatasetDrift ).import_path @@ -268,9 +292,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck): VISION_NEW_LABELS = source_utils.resolve( vision_checks.NewLabels ).import_path - VISION_TRAIN_TEST_LABEL_DRIFT = source_utils.resolve( - vision_checks.TrainTestLabelDrift - ).import_path class DeepchecksModelValidationCheck(DeepchecksValidationCheck): From 281e7f7318ae869357f22f393c51da864c784192 Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Wed, 4 Sep 2024 14:31:07 +0530 Subject: [PATCH 03/11] update model validation and drift checks --- .../deepchecks/validation_checks.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/zenml/integrations/deepchecks/validation_checks.py b/src/zenml/integrations/deepchecks/validation_checks.py index b3ce878478e..b381ca03dfb 100644 --- a/src/zenml/integrations/deepchecks/validation_checks.py +++ b/src/zenml/integrations/deepchecks/validation_checks.py @@ -258,9 +258,15 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck): TABULAR_TRAIN_TEST_FEATURE_DRIFT = source_utils.resolve( tabular_checks.TrainTestFeatureDrift ).import_path + TABULAR_FEATURE_DRIFT = source_utils.resolve( + tabular_checks.FeatureDrift + ).import_path TABULAR_TRAIN_TEST_LABEL_DRIFT = source_utils.resolve( tabular_checks.TrainTestLabelDrift ).import_path + TABULAR_LABEL_DRIFT = source_utils.resolve( + tabular_checks.LabelDrift + ).import_path TABULAR_TRAIN_TEST_SAMPLES_MIX = source_utils.resolve( tabular_checks.TrainTestSamplesMix ).import_path @@ -317,6 +323,12 @@ class DeepchecksModelValidationCheck(DeepchecksValidationCheck): TABULAR_MODEL_INFERENCE_TIME = source_utils.resolve( tabular_checks.ModelInferenceTime ).import_path + TABULAR_MODEL_INFO = source_utils.resolve( + tabular_checks.ModelInfo + ).import_path + TABULAR_PERFORMANCE_BIAS = source_utils.resolve( + tabular_checks.model_evaluation.PerformanceBias + ).import_path TABULAR_REGRESSION_ERROR_DISTRIBUTION = source_utils.resolve( tabular_checks.RegressionErrorDistribution ).import_path @@ -329,6 +341,18 @@ class DeepchecksModelValidationCheck(DeepchecksValidationCheck): TABULAR_SEGMENT_PERFORMANCE = source_utils.resolve( tabular_checks.SegmentPerformance ).import_path + TABULAR_WEAK_SEGMENT_PERFORMANCE = source_utils.resolve( + tabular_checks.WeakSegmentsPerformance + ).import_path + TABULAR_SINGLE_DATASET_PERFORMANCE = source_utils.resolve( + tabular_checks.SingleDatasetPerformance + ).import_path + TABULAR_TRAIN_TEST_PERFORMANCE = source_utils.resolve( + tabular_checks.TrainTestPerformance + ).import_path + TABULAR_MULTI_MODEL_PERFORMANCE_REPORT = source_utils.resolve( + tabular_checks.MultiModelPerformanceReport + ).import_path VISION_CONFUSION_MATRIX_REPORT = source_utils.resolve( vision_checks.ConfusionMatrixReport @@ -339,6 +363,12 @@ class DeepchecksModelValidationCheck(DeepchecksValidationCheck): VISION_MEAN_AVERAGE_RECALL_REPORT = source_utils.resolve( vision_checks.MeanAverageRecallReport ).import_path + VISION_SINGLE_DATASET_PERFORMANCE = source_utils.resolve( + vision_checks.SingleDatasetPerformance + ).import_path + VISION_WEAK_SEGMENT_PERFORMANCE = source_utils.resolve( + vision_checks.WeakSegmentsPerformance + ).import_path class DeepchecksModelDriftCheck(DeepchecksValidationCheck): @@ -364,6 +394,9 @@ class DeepchecksModelDriftCheck(DeepchecksValidationCheck): TABULAR_TRAIN_TEST_PREDICTION_DRIFT = source_utils.resolve( tabular_checks.TrainTestPredictionDrift ).import_path + TABULAR_PREDICTION_DRIFT = source_utils.resolve( + tabular_checks.PredictionDrift + ).import_path TABULAR_UNUSED_FEATURES = source_utils.resolve( tabular_checks.UnusedFeatures ).import_path @@ -377,3 +410,6 @@ class DeepchecksModelDriftCheck(DeepchecksValidationCheck): VISION_TRAIN_TEST_PREDICTION_DRIFT = source_utils.resolve( vision_checks.TrainTestPredictionDrift ).import_path + VISION_PREDICTION_DRIFT = source_utils.resolve( + vision_checks.PredictionDrift + ).import_path From 130cbbff867592a2301029a0b74f2707ed5f790e Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Wed, 4 Sep 2024 17:27:03 +0530 Subject: [PATCH 04/11] add support for multi modle comparison --- .../deepchecks_data_validator.py | 68 +++++++++++++++---- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py index 7fde074d4fb..09d8cae43ce 100644 --- a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +++ b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py @@ -17,6 +17,7 @@ Any, ClassVar, Dict, + List, Optional, Sequence, Tuple, @@ -30,6 +31,8 @@ from deepchecks.tabular import Dataset as TabularData from deepchecks.tabular import Suite as TabularSuite +from deepchecks.tabular import ModelComparisonSuite + # not part of deepchecks.tabular.checks from deepchecks.tabular.suites import full_suite as full_tabular_suite from deepchecks.vision import Suite as VisionSuite @@ -102,7 +105,7 @@ def _create_and_run_check_suite( comparison_dataset: Optional[ Union[pd.DataFrame, DataLoader[Any]] ] = None, - model: Optional[Union[ClassifierMixin, Module]] = None, + models: Optional[List[Union[ClassifierMixin, Module]]] = None, check_list: Optional[Sequence[str]] = None, dataset_kwargs: Dict[str, Any] = {}, check_kwargs: Dict[str, Dict[str, Any]] = {}, @@ -123,7 +126,7 @@ def _create_and_run_check_suite( validation. comparison_dataset: Optional secondary (comparison) dataset argument used during comparison checks. - model: Optional model argument used during validation. + models: Optional model argument used during validation. check_list: Optional list of ZenML Deepchecks check identifiers specifying the list of Deepchecks checks to be performed. dataset_kwargs: Additional keyword arguments to be passed to the @@ -149,6 +152,7 @@ def _create_and_run_check_suite( # arguments and the check list. is_tabular = False is_vision = False + is_multi_model = False for dataset in [reference_dataset, comparison_dataset]: if dataset is None: continue @@ -163,7 +167,18 @@ def _create_and_run_check_suite( f"data and {str(DataLoader)} for computer vision data." ) - if model: + if models: + # if there's more than one models, we should set the + # is_multi_model to True + if len(models) > 1: + is_multi_model = True + # if the models are of different types, raise an error + # only the same type of models can be used for comparison + if len(set(type(model) for model in models)) > 1: + raise TypeError( + f"Models used for comparison checks must be of the same type." + ) + model = models[0] if isinstance(model, ClassifierMixin): is_tabular = True elif isinstance(model, Module): @@ -190,8 +205,18 @@ def _create_and_run_check_suite( if not check_list: # default to executing all the checks listed in the supplied # checks enum type if a custom check list is not supplied + # don't include the TABULAR_PERFORMANCE_BIAS check enum value + # as it requires a protected feature name to be set + checks_to_exclude = [ + DeepchecksModelValidationCheck.TABULAR_PERFORMANCE_BIAS + ] + check_enum_values = [ + check.value + for check in check_enum + if check not in checks_to_exclude + ] tabular_checks, vision_checks = cls._split_checks( - check_enum.values() + check_enum_values ) if is_tabular: check_list = tabular_checks @@ -254,6 +279,10 @@ def _create_and_run_check_suite( suite_class = VisionSuite full_suite = full_vision_suite() + # if is_multi_model is True, we need to use the ModelComparisonSuite + if is_multi_model: + suite_class = ModelComparisonSuite + train_dataset = dataset_class(reference_dataset, **dataset_kwargs) test_dataset = None if comparison_dataset is not None: @@ -294,13 +323,28 @@ def _create_and_run_check_suite( continue condition_method(**condition_kwargs) - suite.add(check) - return suite.run( - train_dataset=train_dataset, - test_dataset=test_dataset, - model=model, - **run_kwargs, - ) + # if the check is supported by the suite, add it + if isinstance(check, suite.supported_checks()): + suite.add(check) + else: + logger.warning( + f"Check {check_name} is not supported by the {suite_class} " + "suite. Ignoring the check." + ) + + if isinstance(suite, ModelComparisonSuite): + return suite.run( + models=models, + train_datasets=train_dataset, + test_datasets=test_dataset, + ) + else: + return suite.run( + train_dataset=train_dataset, + test_dataset=test_dataset, + model=models[0] if models else None, + **run_kwargs, + ) def data_validation( self, @@ -444,7 +488,7 @@ def model_validation( check_enum=check_enum, reference_dataset=dataset, comparison_dataset=comparison_dataset, - model=model, + models=[model], check_list=check_list, dataset_kwargs=dataset_kwargs, check_kwargs=check_kwargs, From a088ca8f9d61736efe873f4a9ff3dd33c598202e Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Fri, 20 Sep 2024 16:49:46 +0530 Subject: [PATCH 05/11] update deepchecks docs --- .../data-validators/deepchecks.md | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/book/component-guide/data-validators/deepchecks.md b/docs/book/component-guide/data-validators/deepchecks.md index ed23082f69d..82b6e1b23a9 100644 --- a/docs/book/component-guide/data-validators/deepchecks.md +++ b/docs/book/component-guide/data-validators/deepchecks.md @@ -12,13 +12,14 @@ The Deepchecks [Data Validator](./data-validators.md) flavor provided with the Z [Deepchecks](https://deepchecks.com/) is an open-source library that you can use to run a variety of data and model validation tests, from data integrity tests that work with a single dataset to model evaluation tests to data drift analyses and model performance comparison tests. All this can be done with minimal configuration input from the user, or customized with specialized conditions that the validation tests should perform. -Deepchecks works with both tabular data and computer vision data (currently in beta). For tabular, the supported dataset format is `pandas.DataFrame` and the supported model format is `sklearn.base.ClassifierMixin`. For computer vision, the supported dataset format is `torch.utils.data.dataloader.DataLoader` and supported model format is `torch.nn.Module`. +Deepchecks works with both tabular data and computer vision data. For tabular, the supported dataset format is `pandas.DataFrame` and the supported model format is `sklearn.base.ClassifierMixin`. For computer vision, the supported dataset format is `torch.utils.data.dataloader.DataLoader` and supported model format is `torch.nn.Module`. You should use the Deepchecks Data Validator when you need the following data and/or model validation features that are possible with Deepchecks: -* Data Integrity Checks [for tabular](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html#data-integrity) or [computer vision](https://docs.deepchecks.com/en/stable/checks\_gallery/vision.html#data-integrity) data: detect data integrity problems within a single dataset (e.g. missing values, conflicting labels, mixed data types etc.). -* Data Drift Checks [for tabular](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html#train-test-validation) or [computer vision](https://docs.deepchecks.com/en/stable/checks\_gallery/vision.html#train-test-validation) data: detect data skew and data drift problems by comparing a target dataset against a reference dataset (e.g. feature drift, label drift, new labels etc.). -* Model Performance Checks [for tabular](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html#model-evaluation) or [computer vision](https://docs.deepchecks.com/en/stable/checks\_gallery/vision.html#model-evaluation) data: evaluate a model and detect problems with its performance (e.g. confusion matrix, boosting overfit, model error analysis) +* Data Integrity Checks [for tabular](https://docs.deepchecks.com/stable/tabular/auto_checks/data_integrity/index.html) or [computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/data_integrity/index.html) data: detect data integrity problems within a single dataset (e.g. missing values, conflicting labels, mixed data types etc.). +* Data Drift Checks [for tabular](https://docs.deepchecks.com/stable/tabular/auto_checks/train_test_validation/index.html) or [computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/train_test_validation/index.html) data: detect data skew and data drift problems by comparing a target dataset against a reference dataset (e.g. feature drift, label drift, new labels etc.). +* Model Performance Checks [for tabular](https://docs.deepchecks.com/stable/tabular/auto_checks/model_evaluation/index.html) or [computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/model_evaluation/index.html) data: evaluate a model and detect problems with its performance (e.g. confusion matrix, boosting overfit, model error analysis) +* Multi-Model Performance Reports [for tabular](https://docs.deepchecks.com/stable/tabular/auto_checks/model_evaluation/plot_multi_model_performance_report.html#sphx-glr-tabular-auto-checks-model-evaluation-plot-multi-model-performance-report-py): produce a summary of performance scores for multiple models on test datasets. You should consider one of the other [Data Validator flavors](./data-validators.md#data-validator-flavors) if you need a different set of data validation features. @@ -44,10 +45,10 @@ zenml stack register custom_stack -dv deepchecks_data_validator ... --set The ZenML integration restructures the way Deepchecks validation checks are organized in four categories, based on the type and number of input parameters that they expect as input. This makes it easier to reason about them when you decide which tests to use in your pipeline steps: -* **data integrity checks** expect a single dataset as input. These correspond one-to-one to the set of Deepchecks data integrity checks [for tabular](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html#data-integrity) and [computer vision](https://docs.deepchecks.com/en/stable/checks\_gallery/vision.html#data-integrity) data -* **data drift checks** require two datasets as input: target and reference. These correspond one-to-one to the set of Deepchecks train-test checks [for tabular data](https://docs.deepchecks.com/stable/checks\_gallery/tabular.html#train-test-validation) and [for computer vision](https://docs.deepchecks.com/stable/checks\_gallery/vision.html#train-test-validation). -* **model validation checks** require a single dataset and a mandatory model as input. This list includes a subset of the model evaluation checks provided by Deepchecks [for tabular data](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html#model-evaluation) and [for computer vision](https://docs.deepchecks.com/stable/checks\_gallery/vision.html#model-evaluation) that expect a single dataset as input. -* **model drift checks** require two datasets and a mandatory model as input. This list includes a subset of the model evaluation checks provided by Deepchecks [for tabular data](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html#model-evaluation) and [for computer vision](https://docs.deepchecks.com/stable/checks\_gallery/vision.html#model-evaluation) that expect two datasets as input: target and reference. +* **data integrity checks** expect a single dataset as input. These correspond one-to-one to the set of Deepchecks data integrity checks [for tabular](https://docs.deepchecks.com/stable/tabular/auto_checks/data_integrity/index.html) and [computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/data_integrity/index.html) data +* **data drift checks** require two datasets as input: target and reference. These correspond one-to-one to the set of Deepchecks train-test checks [for tabular data](https://docs.deepchecks.com/stable/tabular/auto_checks/train_test_validation/index.html) and [for computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/train_test_validation/index.html). +* **model validation checks** require a single dataset and a mandatory model as input. This list includes a subset of the model evaluation checks provided by Deepchecks [for tabular data](https://docs.deepchecks.com/stable/tabular/auto_checks/model_evaluation/index.html) and [for computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/model_evaluation/index.html) that expect a single dataset as input. +* **model drift checks** require two datasets and a mandatory model as input. This list includes a subset of the model evaluation checks provided by Deepchecks [for tabular data](https://docs.deepchecks.com/stable/tabular/auto_checks/model_evaluation/index.html) and [for computer vision](https://docs.deepchecks.com/stable/vision/auto_checks/model_evaluation/index.html) that expect two datasets as input: target and reference. This structure is directly reflected in how Deepchecks can be used with ZenML: there are four different Deepchecks standard steps and four different [ZenML enums for Deepchecks checks](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.validation\_checks) . [The Deepchecks Data Validator API](deepchecks.md#the-deepchecks-data-validator) is also modeled to reflect this same structure. @@ -108,17 +109,17 @@ From here on, you can continue to use the deepchecks integration as is explained ZenML wraps the Deepchecks functionality for tabular data in the form of four standard steps: -* [`DeepchecksDataIntegrityCheckStep`](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks\_data\_drift.DeepchecksDataDriftCheckStep): use it in your pipelines to run data integrity tests on a single dataset -* [`DeepchecksDataDriftCheckStep`](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks\_data\_integrity.DeepchecksDataIntegrityCheckStep): use it in your pipelines to run data drift tests on two datasets as input: target and reference. -* [`DeepchecksModelValidationCheckStep`](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks\_model\_validation.DeepchecksModelValidationCheckStep): class DeepchecksModelDriftCheckStep(BaseStep): use it in your pipelines to run model performance tests using a single dataset and a mandatory model artifact as input -* [`DeepchecksModelDriftCheckStep`](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks\_model\_drift.DeepchecksModelDriftCheckStep): use it in your pipelines to run model comparison/drift tests using a mandatory model artifact and two datasets as input: target and reference. +* [`deepchecks_data_integrity_check_step`](https://sdkdocs.zenml.io/latest/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks_data_integrity): use it in your pipelines to run data integrity tests on a single dataset +* [`deepchecks_data_drift_check_step`](https://sdkdocs.zenml.io/latest/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks_data_drift): use it in your pipelines to run data drift tests on two datasets as input: target and reference. +* [`deepchecks_model_validation_check_step`](https://sdkdocs.zenml.io/latest/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks_model_validation): use it in your pipelines to run model performance tests using a single dataset and a mandatory model artifact as input +* [`deepchecks_model_drift_check_step`](https://sdkdocs.zenml.io/latest/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks_model_drift): use it in your pipelines to run model comparison/drift tests using a mandatory model artifact and two datasets as input: target and reference. The integration doesn't yet include standard steps for computer vision, but you can still write your own custom steps that call [the Deepchecks Data Validator API](deepchecks.md#the-deepchecks-data-validator) or even [call the Deepchecks library directly](deepchecks.md#call-deepchecks-directly). All four standard steps behave similarly regarding the configuration parameters and returned artifacts, with the following differences: * the type and number of input artifacts are different, as mentioned above -* each step expects a different enum data type to be used when explicitly listing the checks to be performed via the `check_list` configuration attribute. See the [`zenml.integrations.deepchecks.validation_checks`](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.validation\_checks) module for more details about these enums (e.g. the data integrity step expects a list of `DeepchecksDataIntegrityCheck` values). +* each step expects a different enum data type to be used when explicitly listing the checks to be performed via the `check_list` configuration attribute. See the [`zenml.integrations.deepchecks.validation_checks`](https://sdkdocs.zenml.io/0.66.0/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.validation_checks) module for more details about these enums (e.g. the data integrity step expects a list of `DeepchecksDataIntegrityCheck` values). This section will only cover how you can use the data integrity step, with a similar usage to be easily inferred for the other three steps. @@ -150,7 +151,7 @@ def data_validation_pipeline(): data_validation_pipeline() ``` -As can be seen from the [step definition](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks\_data\_integrity.deepchecks\_data\_integrity\_check\_step) , the step takes in a dataset and it returns a Deepchecks `SuiteResult` object that contains the test results: +As can be seen from the [step definition](https://sdkdocs.zenml.io/0.66.0/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks_data_integrity) , the step takes in a dataset and it returns a Deepchecks `SuiteResult` object that contains the test results: ```python @step @@ -183,7 +184,7 @@ def validation_pipeline(): ) ``` -You should consult [the official Deepchecks documentation](https://docs.deepchecks.com/en/stable/checks\_gallery/tabular.html) for more information on what each test is useful for. +You should consult [the official Deepchecks documentation](https://docs.deepchecks.com/stable/tabular/auto_checks/data_integrity/index.html) for more information on what each test is useful for. For more customization, the data integrity step also allows for additional keyword arguments to be supplied to be passed transparently to the Deepchecks library: @@ -219,7 +220,7 @@ For more customization, the data integrity step also allows for additional keywo ``` * `run_kwargs`: Additional keyword arguments to be passed to the Deepchecks Suite `run` method. -The `check_kwargs` attribute can also be used to customize [the conditions](https://docs.deepchecks.com/en/stable/user-guide/general/deepchecks\_hierarchy.html#condition) configured for each Deepchecks test. ZenML attaches a special meaning to all check arguments that start with `condition_` and have a dictionary as value. This is required because there is no declarative way to specify conditions for Deepchecks checks. For example, the following step configuration: +The `check_kwargs` attribute can also be used to customize [the conditions](https://docs.deepchecks.com/stable/general/usage/customizations/auto_examples/plot_configure_check_conditions.html#configure-check-conditions) configured for each Deepchecks test. ZenML attaches a special meaning to all check arguments that start with `condition_` and have a dictionary as value. This is required because there is no declarative way to specify conditions for Deepchecks checks. For example, the following step configuration: ```python deepchecks_data_integrity_check_step( @@ -288,8 +289,6 @@ check.add_condition_number_of_outliers_less_or_equal( suite.run(train_dataset=train_dataset) ``` -You can view [the complete list of configuration parameters](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.steps.deepchecks\_data\_integrity.DeepchecksDataIntegrityCheckStepParameters) in the SDK docs. - #### The Deepchecks Data Validator The Deepchecks Data Validator implements the same interface as do all Data Validators, so this method forces you to maintain some level of compatibility with the overall Data Validator abstraction, which guarantees an easier migration in case you decide to switch to another Data Validator. @@ -336,7 +335,7 @@ def data_integrity_check( The arguments that the Deepchecks Data Validator methods can take in are the same as those used for [the Deepchecks standard steps](deepchecks.md#the-deepchecks-standard-steps). -Have a look at [the complete list of methods and parameters available in the `DeepchecksDataValidator` API](https://sdkdocs.zenml.io/latest/integration\_code\_docs/integrations-deepchecks/#zenml.integrations.deepchecks.data\_validators.deepchecks\_data\_validator.DeepchecksDataValidator) in the SDK docs. +Have a look at [the complete list of methods and parameters available in the `DeepchecksDataValidator` API](https://sdkdocs.zenml.io/latest/integration_code_docs/integrations-deepchecks/#zenml.integrations.deepchecks.data_validators.deepchecks_data_validator.DeepchecksDataValidator) in the SDK docs. #### Call Deepchecks directly From ee1092551e8f4ba48bd38d1cf675dfd50bdffcb7 Mon Sep 17 00:00:00 2001 From: Hamza Tahir Date: Mon, 23 Sep 2024 17:57:10 +0200 Subject: [PATCH 06/11] Update src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py --- .../deepchecks/data_validators/deepchecks_data_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py index 09d8cae43ce..d2211878eb3 100644 --- a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +++ b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py @@ -176,7 +176,7 @@ def _create_and_run_check_suite( # only the same type of models can be used for comparison if len(set(type(model) for model in models)) > 1: raise TypeError( - f"Models used for comparison checks must be of the same type." + "Models used for comparison checks must be of the same type." ) model = models[0] if isinstance(model, ClassifierMixin): From 81c85a5011b8035da0fe774fdb000bd2a3b1467c Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 23 Sep 2024 16:42:34 +0000 Subject: [PATCH 07/11] Auto-update of Starter template --- examples/mlops_starter/.copier-answers.yml | 2 +- examples/mlops_starter/README.md | 8 ++++---- examples/mlops_starter/configs/feature_engineering.yaml | 1 + examples/mlops_starter/configs/inference.yaml | 1 + examples/mlops_starter/configs/training_rf.yaml | 1 + examples/mlops_starter/configs/training_sgd.yaml | 1 + examples/mlops_starter/quickstart.ipynb | 2 +- examples/mlops_starter/requirements.txt | 1 + 8 files changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/mlops_starter/.copier-answers.yml b/examples/mlops_starter/.copier-answers.yml index 48aecdfa31b..49b01a7c71e 100644 --- a/examples/mlops_starter/.copier-answers.yml +++ b/examples/mlops_starter/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 2024.08.28 +_commit: 2024.09.23 _src_path: gh:zenml-io/template-starter email: info@zenml.io full_name: ZenML GmbH diff --git a/examples/mlops_starter/README.md b/examples/mlops_starter/README.md index 10dc01e622d..b25258f78bd 100644 --- a/examples/mlops_starter/README.md +++ b/examples/mlops_starter/README.md @@ -24,7 +24,7 @@ Along the way we will also show you how to: You can use Google Colab to see ZenML in action, no signup / installation required! -Open In Colab +Open In Colab ## :computer: Run Locally @@ -36,7 +36,7 @@ pip install "zenml[server]" # clone the ZenML repository git clone https://github.com/zenml-io/zenml.git -cd zenml/examples/quickstart +cd zenml/examples/mlops_starter ``` Now we're ready to start. You have two options for running the quickstart locally: @@ -45,13 +45,13 @@ Now we're ready to start. You have two options for running the quickstart locall ```bash pip install notebook jupyter notebook -# open notebooks/quickstart.ipynb +# open quickstart.ipynb ``` #### Option 2 - Execute the whole ML pipeline from a Python script: ```bash # Install required zenml integrations -zenml integration install sklearn -y +zenml integration install sklearn pandas -y # Initialize ZenML zenml init diff --git a/examples/mlops_starter/configs/feature_engineering.yaml b/examples/mlops_starter/configs/feature_engineering.yaml index d5ab212951e..055bb6facac 100644 --- a/examples/mlops_starter/configs/feature_engineering.yaml +++ b/examples/mlops_starter/configs/feature_engineering.yaml @@ -3,6 +3,7 @@ settings: docker: required_integrations: - sklearn + - pandas requirements: - pyarrow diff --git a/examples/mlops_starter/configs/inference.yaml b/examples/mlops_starter/configs/inference.yaml index 1dcefe44cf6..8f73d76256a 100644 --- a/examples/mlops_starter/configs/inference.yaml +++ b/examples/mlops_starter/configs/inference.yaml @@ -3,6 +3,7 @@ settings: docker: required_integrations: - sklearn + - pandas requirements: - pyarrow diff --git a/examples/mlops_starter/configs/training_rf.yaml b/examples/mlops_starter/configs/training_rf.yaml index 8d75610985f..70fa6413646 100644 --- a/examples/mlops_starter/configs/training_rf.yaml +++ b/examples/mlops_starter/configs/training_rf.yaml @@ -3,6 +3,7 @@ settings: docker: required_integrations: - sklearn + - pandas requirements: - pyarrow diff --git a/examples/mlops_starter/configs/training_sgd.yaml b/examples/mlops_starter/configs/training_sgd.yaml index 857cdf7f82a..386b53b8c30 100644 --- a/examples/mlops_starter/configs/training_sgd.yaml +++ b/examples/mlops_starter/configs/training_sgd.yaml @@ -3,6 +3,7 @@ settings: docker: required_integrations: - sklearn + - pandas requirements: - pyarrow diff --git a/examples/mlops_starter/quickstart.ipynb b/examples/mlops_starter/quickstart.ipynb index 3604e7dd6a7..df8c010b5ea 100644 --- a/examples/mlops_starter/quickstart.ipynb +++ b/examples/mlops_starter/quickstart.ipynb @@ -31,7 +31,7 @@ "required!\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](\n", - "https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/quickstart/quickstart.ipynb)" + "https://colab.research.google.com/github/zenml-io/zenml/blob/main/examples/mlops_starter/quickstart.ipynb)" ] }, { diff --git a/examples/mlops_starter/requirements.txt b/examples/mlops_starter/requirements.txt index d38ead5e262..1e0a8ac5dac 100644 --- a/examples/mlops_starter/requirements.txt +++ b/examples/mlops_starter/requirements.txt @@ -2,3 +2,4 @@ zenml[server]>=0.50.0 notebook scikit-learn pyarrow +pandas From 8454e5945a81e137e0381ba921b3f568f632753b Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 23 Sep 2024 16:52:09 +0000 Subject: [PATCH 08/11] Auto-update of NLP template --- examples/e2e_nlp/.copier-answers.yml | 2 +- examples/e2e_nlp/config.yaml | 1 - examples/e2e_nlp/requirements.txt | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/e2e_nlp/.copier-answers.yml b/examples/e2e_nlp/.copier-answers.yml index ca8152a6309..3ca2ba198fe 100644 --- a/examples/e2e_nlp/.copier-answers.yml +++ b/examples/e2e_nlp/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 2024.08.29 +_commit: 2024.09.23 _src_path: gh:zenml-io/template-nlp accelerator: cpu cloud_of_choice: aws diff --git a/examples/e2e_nlp/config.yaml b/examples/e2e_nlp/config.yaml index 764ff93c8fc..e5c0b0cdfe6 100644 --- a/examples/e2e_nlp/config.yaml +++ b/examples/e2e_nlp/config.yaml @@ -28,7 +28,6 @@ settings: - mlflow - discord requirements: - - accelerate - zenml[server] extra: diff --git a/examples/e2e_nlp/requirements.txt b/examples/e2e_nlp/requirements.txt index f7f98175b16..e79245c3df9 100644 --- a/examples/e2e_nlp/requirements.txt +++ b/examples/e2e_nlp/requirements.txt @@ -1,4 +1,4 @@ torchvision -accelerate gradio zenml[server]>=0.56.3 +datasets>=2.12.0,<3.0.0 From 69bceb31123a5f3337ea37b9d1c844ade4dda203 Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Tue, 24 Sep 2024 13:09:35 +0530 Subject: [PATCH 09/11] fix import block sorting --- .../deepchecks_data_validator.py | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py index 09d8cae43ce..fc17daadc58 100644 --- a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +++ b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py @@ -13,30 +13,15 @@ # permissions and limitations under the License. """Implementation of the Deepchecks data validator.""" -from typing import ( - Any, - ClassVar, - Dict, - List, - Optional, - Sequence, - Tuple, - Type, - Union, -) +from typing import Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type, Union import pandas as pd from deepchecks.core.checks import BaseCheck from deepchecks.core.suite import SuiteResult from deepchecks.tabular import Dataset as TabularData -from deepchecks.tabular import Suite as TabularSuite - -from deepchecks.tabular import ModelComparisonSuite - -# not part of deepchecks.tabular.checks +from deepchecks.tabular import ModelComparisonSuite, Suite as TabularSuite from deepchecks.tabular.suites import full_suite as full_tabular_suite -from deepchecks.vision import Suite as VisionSuite -from deepchecks.vision import VisionData +from deepchecks.vision import Suite as VisionSuite, VisionData from deepchecks.vision.suites import full_suite as full_vision_suite from sklearn.base import ClassifierMixin from torch.nn import Module From ffdb530c3a481c5a05c7b58eff2aff703eaaa729 Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Tue, 24 Sep 2024 13:39:41 +0530 Subject: [PATCH 10/11] add model_evaluation to mocked_libs --- docs/mocked_libs.json | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mocked_libs.json b/docs/mocked_libs.json index a7746f14bed..796dbeea0a1 100644 --- a/docs/mocked_libs.json +++ b/docs/mocked_libs.json @@ -44,6 +44,7 @@ "deepchecks.tabular", "deepchecks.tabular.checks", "deepchecks.tabular.checks.data_integrity", + "deepchecks.tabular.checks.model_evaluation", "deepchecks.tabular.suites", "deepchecks.vision", "deepchecks.vision.checks", From 8ddcf2663ff4e4bee03c36077b80f13ad57b5bd4 Mon Sep 17 00:00:00 2001 From: Baris Can Durak Date: Tue, 24 Sep 2024 15:38:08 +0200 Subject: [PATCH 11/11] formatting --- .../deepchecks_data_validator.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py index 1dd4cf7fcd4..94aa67cf65f 100644 --- a/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +++ b/src/zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py @@ -13,15 +13,27 @@ # permissions and limitations under the License. """Implementation of the Deepchecks data validator.""" -from typing import Any, ClassVar, Dict, List, Optional, Sequence, Tuple, Type, Union +from typing import ( + Any, + ClassVar, + Dict, + List, + Optional, + Sequence, + Tuple, + Type, + Union, +) import pandas as pd from deepchecks.core.checks import BaseCheck from deepchecks.core.suite import SuiteResult from deepchecks.tabular import Dataset as TabularData -from deepchecks.tabular import ModelComparisonSuite, Suite as TabularSuite +from deepchecks.tabular import ModelComparisonSuite +from deepchecks.tabular import Suite as TabularSuite from deepchecks.tabular.suites import full_suite as full_tabular_suite -from deepchecks.vision import Suite as VisionSuite, VisionData +from deepchecks.vision import Suite as VisionSuite +from deepchecks.vision import VisionData from deepchecks.vision.suites import full_suite as full_vision_suite from sklearn.base import ClassifierMixin from torch.nn import Module @@ -153,7 +165,7 @@ def _create_and_run_check_suite( ) if models: - # if there's more than one models, we should set the + # if there's more than one models, we should set the # is_multi_model to True if len(models) > 1: is_multi_model = True