From 218e6c99018dfae9b0734f310b25aaa55acaeefa Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 08:01:05 -0700 Subject: [PATCH 01/40] Add tests and update api --- .../ExpectColumnDistinctValuesToBeInSet.json | 18 ++--- ...xpectColumnDistinctValuesToContainSet.json | 18 ++--- .../ExpectColumnDistinctValuesToEqualSet.json | 18 ++--- .../ExpectColumnKLDivergenceToBeLessThan.json | 18 ++--- .../schemas/ExpectColumnMaxToBeBetween.json | 18 ++--- .../schemas/ExpectColumnMeanToBeBetween.json | 18 ++--- .../ExpectColumnMedianToBeBetween.json | 18 ++--- .../schemas/ExpectColumnMinToBeBetween.json | 18 ++--- .../ExpectColumnMostCommonValueToBeInSet.json | 18 ++--- ...pectColumnPairValuesAToBeGreaterThanB.json | 18 ++--- .../ExpectColumnPairValuesToBeEqual.json | 18 ++--- .../ExpectColumnPairValuesToBeInSet.json | 18 ++--- ...mnProportionOfUniqueValuesToBeBetween.json | 18 ++--- ...ExpectColumnQuantileValuesToBeBetween.json | 18 ++--- .../schemas/ExpectColumnStdevToBeBetween.json | 18 ++--- .../schemas/ExpectColumnSumToBeBetween.json | 18 ++--- ...pectColumnUniqueValueCountToBeBetween.json | 18 ++--- .../ExpectColumnValueLengthsToBeBetween.json | 18 ++--- .../ExpectColumnValueLengthsToEqual.json | 18 ++--- .../ExpectColumnValueZScoresToBeLessThan.json | 18 ++--- .../ExpectColumnValuesToBeBetween.json | 18 ++--- .../schemas/ExpectColumnValuesToBeInSet.json | 18 ++--- .../ExpectColumnValuesToBeInTypeList.json | 18 ++--- .../schemas/ExpectColumnValuesToBeNull.json | 18 ++--- .../schemas/ExpectColumnValuesToBeOfType.json | 18 ++--- .../schemas/ExpectColumnValuesToBeUnique.json | 18 ++--- .../ExpectColumnValuesToMatchLikePattern.json | 18 ++--- ...ectColumnValuesToMatchLikePatternList.json | 18 ++--- .../ExpectColumnValuesToMatchRegex.json | 18 ++--- .../ExpectColumnValuesToMatchRegexList.json | 18 ++--- .../ExpectColumnValuesToNotBeInSet.json | 18 ++--- .../ExpectColumnValuesToNotBeNull.json | 18 ++--- ...pectColumnValuesToNotMatchLikePattern.json | 18 ++--- ...ColumnValuesToNotMatchLikePatternList.json | 18 ++--- .../ExpectColumnValuesToNotMatchRegex.json | 18 ++--- ...ExpectColumnValuesToNotMatchRegexList.json | 18 ++--- .../ExpectCompoundColumnsToBeUnique.json | 18 ++--- .../schemas/ExpectMulticolumnSumToEqual.json | 18 ++--- ...ectColumnValuesToBeUniqueWithinRecord.json | 18 ++--- .../expectations/expectation.py | 9 +-- .../test_expectation_conditions.py | 75 +++++++++++++++++++ 41 files changed, 313 insertions(+), 473 deletions(-) create mode 100644 tests/integration/data_sources_and_expectations/test_expectation_conditions.py diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json index edd4d4697f50..3bde3dd7533a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_set": { "title": "Value Set", @@ -280,17 +285,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json index a96a5306ac91..2097b90dc92a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_set": { "title": "Value Set", @@ -280,17 +285,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json index 24a2264c8d5a..cf2886baabe2 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_set": { "title": "Value Set", @@ -280,17 +285,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json index c314e5e1bd14..21b0e4423934 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "partition_object": { "title": "Partition Object", @@ -282,17 +287,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json index 4be7a8bbf66d..28ab33e6031b 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -261,17 +266,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json index 7fd498935b05..df6a8cc0e6f5 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -261,17 +266,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json index 740cfe7a4a9e..fe730b4d4d08 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -261,17 +266,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json index 01d68005b67c..cda77859b85c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -261,17 +266,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json index 6608c3e14414..8960f0485595 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_set": { "title": "Value Set", @@ -285,17 +290,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json index 2579fef51888..453e8dfe954c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json @@ -92,7 +92,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "or_equal": { "title": "Or Equal", @@ -240,17 +245,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json index aa7e980fe151..a650d2f419b9 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json @@ -92,7 +92,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "ignore_row_if": { "title": "Ignore Row If", @@ -236,17 +241,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json index 41197fe628b7..80ea0b8137bb 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json @@ -92,7 +92,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_pairs_set": { "title": "Value Pairs Set", @@ -249,17 +254,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json index 1d44cd059aae..4a67051c8e82 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -278,17 +283,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json index 7eccce672d97..4d8c10dde714 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "quantile_ranges": { "title": "Quantile Ranges", @@ -231,17 +236,6 @@ ], "additionalProperties": false }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" - }, "QuantileRange": { "title": "QuantileRange", "type": "object", diff --git a/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json index 36a2ac3346c3..d9a5c081894b 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -261,17 +266,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json index a55c0f3d732a..83a11c74534e 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -260,17 +265,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json index f5d0a23ac260..852139485e2e 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json @@ -77,7 +77,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -278,17 +283,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json index 3c3fe0298148..5fe0ef3d473a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -262,17 +267,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json index 7d49abc4515a..07b5ea6a5e73 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value": { "title": "Value", @@ -231,17 +236,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json index 7ecfb20bdcbb..8e62107caa78 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "threshold": { "title": "Threshold", @@ -244,17 +249,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json index 368652e22f54..e4bb826b46d3 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "min_value": { "title": "Min Value", @@ -269,17 +274,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json index 11fca2518450..9f7664cc261b 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_set": { "title": "Value Set", @@ -289,17 +294,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json index a1b69a6bebf5..856b02b161ae 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "type_list": { "title": "Type List", @@ -233,17 +238,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json index b3ff965417eb..f7fcd56b5fad 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "metadata": { "type": "object", @@ -218,17 +223,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json index 0d9f52413744..ed36b39da4d1 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "type_": { "title": "Type ", @@ -224,17 +229,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json index 778ca834867f..aadf6d7455d0 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "metadata": { "type": "object", @@ -217,17 +222,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json index 11adfe72a4d2..1d4d54c99df5 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "like_pattern": { "title": "Like Pattern", @@ -227,17 +232,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json index 0c1955b4645e..01aa524c95fa 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "like_pattern_list": { "title": "Like Pattern List", @@ -240,17 +245,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json index f9f30ac80ca7..85569d993f46 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "regex": { "title": "Regex", @@ -227,17 +232,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json index 73c05b4cef16..f6fef176ff0c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "regex_list": { "title": "Regex List", @@ -240,17 +245,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json index c380f9087c5f..87a3c27dc702 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "value_set": { "title": "Value Set", @@ -289,17 +294,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json index 52e4418e5b62..a9dda798868c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "metadata": { "type": "object", @@ -218,17 +223,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json index 3a092c1e1ed0..28d64d433ecb 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "like_pattern": { "title": "Like Pattern", @@ -227,17 +232,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json index b8e938b1162c..11581ee55455 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "like_pattern_list": { "title": "Like Pattern List", @@ -230,17 +235,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json index fa4509fde816..3027cb4a81bc 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "regex": { "title": "Regex", @@ -227,17 +232,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json index e64b79a0fe26..abfded6ce70a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json @@ -86,7 +86,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "regex_list": { "title": "Regex List", @@ -230,17 +235,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json b/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json index 7a81307fb399..1f83cb4c355d 100644 --- a/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json +++ b/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json @@ -88,7 +88,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "ignore_row_if": { "title": "Ignore Row If", @@ -230,17 +235,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json b/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json index 9126f935700f..f7c5629a9c32 100644 --- a/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json @@ -88,7 +88,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "ignore_row_if": { "title": "Ignore Row If", @@ -237,17 +242,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json b/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json index df1fcd601126..8b74fe2ddf48 100644 --- a/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json +++ b/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json @@ -88,7 +88,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "pandas", + "great_expectations" + ], + "type": "string" }, "ignore_row_if": { "title": "Ignore Row If", @@ -226,17 +231,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index b84882b0d4a4..9b37a9ba40b4 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -65,7 +65,6 @@ WINDOWS_DESCRIPTION, ) from great_expectations.expectations.model_field_types import ( - ConditionParser, MostlyField, ) from great_expectations.expectations.registry import ( @@ -1791,7 +1790,7 @@ class ColumnAggregateExpectation(BatchExpectation, ABC): column: StrictStr = Field(min_length=1, description=COLUMN_DESCRIPTION) row_condition: Union[str, None] = None - condition_parser: Union[ConditionParser, None] = None + condition_parser: Union[Literal["pandas", "great_expectations"], None] = None domain_keys: ClassVar[Tuple[str, ...]] = ( "batch_id", @@ -1844,7 +1843,7 @@ class ColumnMapExpectation(BatchExpectation, ABC): column: StrictStr = Field(min_length=1, description=COLUMN_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None - condition_parser: Union[ConditionParser, None] = None + condition_parser: Union[Literal["pandas", "great_expectations"], None] = None catch_exceptions: bool = True @@ -2112,7 +2111,7 @@ class ColumnPairMapExpectation(BatchExpectation, ABC): column_B: StrictStr = Field(min_length=1, description=COLUMN_B_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None - condition_parser: Union[ConditionParser, None] = None + condition_parser: Union[Literal["pandas", "great_expectations"], None] = None catch_exceptions: bool = True @@ -2368,7 +2367,7 @@ class MulticolumnMapExpectation(BatchExpectation, ABC): column_list: List[StrictStr] = pydantic.Field(description=COLUMN_LIST_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None - condition_parser: Union[ConditionParser, None] = None + condition_parser: Union[Literal["pandas", "great_expectations"], None] = None ignore_row_if: Literal["all_values_are_missing", "any_value_is_missing", "never"] = ( "all_values_are_missing" ) diff --git a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py new file mode 100644 index 000000000000..5f6df81d6265 --- /dev/null +++ b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py @@ -0,0 +1,75 @@ +from datetime import datetime, timezone + +import pandas as pd + +import great_expectations.expectations as gxe +from great_expectations.compatibility.sqlalchemy import sqltypes +from tests.integration.conftest import parameterize_batch_for_data_sources +from tests.integration.test_utils.data_source_config import ( + MSSQLDatasourceTestConfig, + MySQLDatasourceTestConfig, + PandasDataFrameDatasourceTestConfig, + PandasFilesystemCsvDatasourceTestConfig, + PostgreSQLDatasourceTestConfig, + SnowflakeDatasourceTestConfig, + SqliteDatasourceTestConfig, +) + + +@parameterize_batch_for_data_sources( + data_source_configs=[ + PandasDataFrameDatasourceTestConfig(), + PandasFilesystemCsvDatasourceTestConfig(), + ], + data=pd.DataFrame( + { + "date": [ + datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), + ], + "quantity": [1, 2, 3], + } + ), +) +def test_expect_column_min_to_be_between__pandas_row_condition(batch_for_datasource) -> None: + expectation = gxe.ExpectColumnMinToBeBetween( + column="date", + min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), + max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), + row_condition="quantity<2", + condition_parser="pandas", + ) + result = batch_for_datasource.validate(expectation) + assert result.success + + +@parameterize_batch_for_data_sources( + data_source_configs=[ + MSSQLDatasourceTestConfig(column_types={"date": sqltypes.DATE}), + MySQLDatasourceTestConfig(column_types={"date": sqltypes.DATE}), + PostgreSQLDatasourceTestConfig(column_types={"date": sqltypes.DATE}), + SnowflakeDatasourceTestConfig(column_types={"date": sqltypes.DATE}), + SqliteDatasourceTestConfig(column_types={"date": sqltypes.DATE}), + ], + data=pd.DataFrame( + { + "date": [ + datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), + ], + "quantity": [1, 2, 3], + } + ), +) +def test_expect_column_min_to_be_between__sql_row_condition(batch_for_datasource) -> None: + expectation = gxe.ExpectColumnMinToBeBetween( + column="date", + min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), + max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), + row_condition='col("quantity")<2', + condition_parser="great_expectations", + ) + result = batch_for_datasource.validate(expectation) + assert result.success From c1d5c736d6cb955d3d368cafd176def72b5b7f98 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 08:07:53 -0700 Subject: [PATCH 02/40] Update tests --- .../expect_column_value_lengths_to_be_between.json | 2 +- .../expect_column_value_lengths_to_equal.json | 2 +- .../expect_column_values_to_be_between.json | 2 +- .../expect_column_values_to_be_between_tz_naive.json | 2 +- .../expect_column_values_to_be_in_set.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json index 361e9c711425..bf412954986b 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json +++ b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json @@ -206,7 +206,7 @@ "condition_parser": "bad_parser" }, "out" : { - "traceback_substring": "not a valid enumeration member" + "traceback_substring": "unexpected value" } }] }] diff --git a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json index 727076ea1287..7e7cc9dc4a92 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json +++ b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json @@ -129,7 +129,7 @@ "condition_parser": "bad_parser" }, "out" : { - "traceback_substring": "not a valid enumeration member" + "traceback_substring": "unexpected value" } }] }] diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json index f9c6375da979..733896cb313f 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json +++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json @@ -776,7 +776,7 @@ "condition_parser": "bad_parser" }, "out": { - "traceback_substring": "not a valid enumeration member" + "traceback_substring": "unexpected value" } } ] diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json index 2a69593642ff..7e87825e691a 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json +++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json @@ -572,7 +572,7 @@ "condition_parser": "bad_parser" }, "out": { - "traceback_substring": "not a valid enumeration member" + "traceback_substring": "unexpected value" } } ] diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json index c0b543334a25..e5afe95e82ce 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json +++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json @@ -177,7 +177,7 @@ "condition_parser": "bad_parser" }, "out": { - "traceback_substring": "not a valid enumeration member" + "traceback_substring": "unexpected value" } } ]}, From 62f0ac29c26ac0c2441b75b20f8253dba523ad53 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 08:25:20 -0700 Subject: [PATCH 03/40] Make docs changes --- .../docs/components/examples_under_test.py | 6 +++--- ...n_row_conditions.py => expectation_conditions.py} | 12 ++++++------ .../customize_expectations/customize_expectations.md | 2 +- ...n_row_conditions.md => expectation_conditions.md} | 8 ++++---- docs/docusaurus/sidebars.js | 2 +- docs/docusaurus/static/_redirects | 4 ++++ 6 files changed, 19 insertions(+), 15 deletions(-) rename docs/docusaurus/docs/core/customize_expectations/_examples/{expectation_row_conditions.py => expectation_conditions.py} (84%) rename docs/docusaurus/docs/core/customize_expectations/{expectation_row_conditions.md => expectation_conditions.md} (96%) diff --git a/docs/docusaurus/docs/components/examples_under_test.py b/docs/docusaurus/docs/components/examples_under_test.py index 13562cf47c07..996703c4a02f 100644 --- a/docs/docusaurus/docs/components/examples_under_test.py +++ b/docs/docusaurus/docs/components/examples_under_test.py @@ -473,9 +473,9 @@ ), IntegrationTestFixture( # To test, run: - # pytest --docs-tests -k "docs_example_expectation_row_conditions" tests/integration/test_script_runner.py - name="docs_example_expectation_row_conditions", - user_flow_script="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py", + # pytest --docs-tests -k "docs_example_expectation_conditions" tests/integration/test_script_runner.py + name="docs_example_expectation_conditions", + user_flow_script="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py", data_dir="docs/docusaurus/docs/components/_testing/test_data_sets/titantic_test_file", # data_context_dir="", backend_dependencies=[], diff --git a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py similarity index 84% rename from docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py rename to docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py index b5d32ab6025b..88f279ed9789 100644 --- a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py +++ b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py @@ -2,7 +2,7 @@ This is an example script for how to use Expectation row conditions. To test, run: -pytest --docs-tests -k "doc_example_expectation_row_conditions" tests/integration/test_script_runner.py +pytest --docs-tests -k "docs_example_expectation_conditions" tests/integration/test_script_runner.py """ @@ -31,7 +31,7 @@ def set_up_context_for_example(context): # EXAMPLE SCRIPT STARTS HERE: -# +# import great_expectations as gx context = gx.get_context() @@ -50,7 +50,7 @@ def set_up_context_for_example(context): ) # An unconditional Expectation is defined without the `row_condition` or `condition_parser` parameters: -# +# expectation = gx.expectations.ExpectColumnValuesToBeInSet( column="Survived", value_set=[0, 1] ) @@ -60,11 +60,11 @@ def set_up_context_for_example(context): print(batch.validate(expectation)) # A Conditional Expectation for a pandas Data Source would be defined like this: -# +# conditional_expectation = gx.expectations.ExpectColumnValuesToBeInSet( column="Survived", value_set=[1], - # + # condition_parser="pandas", row_condition='PClass=="1st"', # @@ -78,7 +78,7 @@ def set_up_context_for_example(context): conditional_expectation = gx.expectations.ExpectColumnValuesToBeInSet( column="Survived", value_set=[1], - # + # condition_parser="spark", row_condition='PClass=="1st"', # diff --git a/docs/docusaurus/docs/core/customize_expectations/customize_expectations.md b/docs/docusaurus/docs/core/customize_expectations/customize_expectations.md index c1f9ce94bab4..02b820b695ea 100644 --- a/docs/docusaurus/docs/core/customize_expectations/customize_expectations.md +++ b/docs/docusaurus/docs/core/customize_expectations/customize_expectations.md @@ -18,7 +18,7 @@ import OverviewCard from '@site/src/components/OverviewCard'; topIcon label="Restrict an Expectation with row conditions" description="Use `row_conditions` to restrict the data an Expectation evaluates" - to="/core/customize_expectations/expectation_row_conditions" + to="/core/customize_expectations/expectation_conditions" icon="/img/expectation_icon.svg" /> - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py - pandas example row_condition" + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - pandas example row_condition" ``` - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py - spark example row_condition" + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - spark example row_condition" ``` @@ -153,12 +153,12 @@ The data used in the examples for this procedure is passenger data for the Titan For example, the following code creates a unconditional Expectation that the value of the `"Survived"` column is either 0 or 1: - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py - example unconditional Expectation" + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - example unconditional Expectation" ``` And this code creates a Conditional version of the same Expectation that specifies the value of the `"Survived"` column is `1` if the individual was a first class passenger: - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py - example conditional Expectation" + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - example conditional Expectation" ``` diff --git a/docs/docusaurus/sidebars.js b/docs/docusaurus/sidebars.js index 8a5c9ba28cdf..66f99622558e 100644 --- a/docs/docusaurus/sidebars.js +++ b/docs/docusaurus/sidebars.js @@ -117,7 +117,7 @@ module.exports = { label: 'Customize Expectations', link: { type: 'doc', id: 'core/customize_expectations/customize_expectations' }, items: [ - { type: 'doc', id: 'core/customize_expectations/expectation_row_conditions' }, + { type: 'doc', id: 'core/customize_expectations/expectation_conditions' }, { type: 'doc', id: 'core/customize_expectations/define_a_custom_expectation_class' }, { type: 'doc', id: 'core/customize_expectations/use_sql_to_define_a_custom_expectation' }, ] diff --git a/docs/docusaurus/static/_redirects b/docs/docusaurus/static/_redirects index 13a54b646cab..1ed6b92da3c3 100644 --- a/docs/docusaurus/static/_redirects +++ b/docs/docusaurus/static/_redirects @@ -490,3 +490,7 @@ docs/guides/setup/configuring_data_docs/how_to_host_and_share_data_docs_on_azure /docs/reference/learn/expectations/* /docs/0.18/reference/learn/expectations/:splat /docs/reference/learn/conceptual_guides/* /docs/0.18/reference/learn/conceptual_guides/:splat /docs/reference/api/* /docs/0.18/reference/api/:splat + + +## Redirects for renamed core docs +/docs/core/customize_expectations/expectation_row_conditions /docs/core/customize_expectations/expectation_conditions From 7ce97181c9f6f9ee3ead12ddaccc776afc9ba0bb Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 08:36:56 -0700 Subject: [PATCH 04/40] Update md and py file content --- .../_examples/expectation_conditions.py | 38 ++-- .../expectation_conditions.md | 173 ++++++++++++------ 2 files changed, 141 insertions(+), 70 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py index 88f279ed9789..b051f05dd5bf 100644 --- a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py +++ b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py @@ -1,5 +1,5 @@ """ -This is an example script for how to use Expectation row conditions. +This is an example script for how to use Expectation conditions. To test, run: pytest --docs-tests -k "docs_example_expectation_conditions" tests/integration/test_script_runner.py @@ -49,8 +49,8 @@ def set_up_context_for_example(context): .get_batch() ) -# An unconditional Expectation is defined without the `row_condition` or `condition_parser` parameters: -# +# An Expectation without conditions is defined without the `row_condition` or `condition_parser` parameters: +# expectation = gx.expectations.ExpectColumnValuesToBeInSet( column="Survived", value_set=[0, 1] ) @@ -59,9 +59,9 @@ def set_up_context_for_example(context): # Test the Expectation: print(batch.validate(expectation)) -# A Conditional Expectation for a pandas Data Source would be defined like this: -# -conditional_expectation = gx.expectations.ExpectColumnValuesToBeInSet( +# An Expectation condition for a pandas Data Source would be defined like this: +# +expectation_with_condition = gx.expectations.ExpectColumnValuesToBeInSet( column="Survived", value_set=[1], # @@ -71,16 +71,30 @@ def set_up_context_for_example(context): ) # -# Test the Conditional Expectation: -print(batch.validate(conditional_expectation)) +# Test the Expectation condition: +print(batch.validate(expectation_with_condition)) -# A Conditional Expectation for a Spark or SQL Data Source would be defined like this: -conditional_expectation = gx.expectations.ExpectColumnValuesToBeInSet( +# A Conditional Expectation for a Spark Data Source would be defined like this: +# +expectation_with_condition = gx.expectations.ExpectColumnValuesToBeInSet( column="Survived", value_set=[1], # - condition_parser="spark", - row_condition='PClass=="1st"', + condition_parser="great_expectations", + row_condition='col("PClass")=="1st"', + # +) +# + +# A Conditional Expectation for a SQL Data Source would be defined like this: +# +expectation_with_condition = gx.expectations.ExpectColumnValuesToBeInSet( + column="Survived", + value_set=[1], + # + condition_parser="great_expectations", + row_condition='col("PClass")=="1st"', # ) # +# diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 0f4a3680c3ba..d345c7f0885a 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -1,5 +1,5 @@ --- -title: Apply Conditional Expectations to specific rows within a Batch +title: Apply Expectation conditions to specific rows within a Batch --- import TabItem from '@theme/TabItem'; import Tabs from '@theme/Tabs'; @@ -9,16 +9,15 @@ import PrereqGxInstalled from '../_core_components/prerequisites/_gx_installatio import PrereqPreconfiguredDataContext from '../_core_components/prerequisites/_preconfigured_data_context.md'; import PrereqPreconfiguredDataSourceAndAsset from '../_core_components/prerequisites/_data_source_and_asset_connected_to_data.md'; -By default Expectations apply to the entire dataset retrieved in a Batch. However, sometimes an Expectation is not relevant for every row and validating every row could cause false positives or false negatives in the Validation Results. +By default, Expectations apply to the entire dataset retrieved in a Batch. However, there are instances when an Expectation may not be relevant for every row. Validating every row could lead to false positives or false negatives in the Validation Results. -For example, you may define an Expectation that a column specifying the country of origin of a product should not be null. If that Expectation is only relevant when the product is a foreign import then applying the Expectation to every row in the Batch could result in a large number of false negatives when the country of origin column is null for products produced by local industry. +For example, you might define an Expectation that a column indicating the country of origin for a product should not be null. If this Expectation is only applicable when the product is an import, applying it to every row in the Batch could result in many false negatives when the country of origin column is null for products produced locally. -To solve this issue GX allows you to define Conditional Expectations that only apply to a subset of the data retrieved in a Batch. +To address this issue, GX allows you to define Expectation conditions that apply only to a subset of the data retrieved in a Batch. -## Create a Conditional Expectation - -Great Expectations lets you express Conditional Expectations with a `row_condition` argument that can be passed to all Expectations that evaluate rows within a Dataset. The `row_condition` argument should be a boolean expression string. The Conditional Expectation will validate rows that result in the `row_condition` string being `True`. When the `row_condition` string evaluates as `False`, the row in question will not be validated by the Expectation. +## Create an Expectation condition +Great Expectations allows you to specify conditions for validating rows using the `row_condition` argument, which can be applied to all Expectations that assess rows within a Dataset. The `row_condition` argument should be a string that represents a boolean expression. Rows will be validated when the `row_condition` expression evaluates to `True`. Conversely, if the `row_condition` evaluates to `False`, the corresponding row will not be validated by the Expectation. ### Prerequisites - . @@ -39,78 +38,98 @@ Great Expectations lets you express Conditional Expectations with a `row_conditi -In this procedure, your Data Context is assumed to be stored in the variable `context` and your Expectation Suite is assumed to be stored in the variable `suite`. `suite` can be a newly created and empty Expectation Suite, or an existing Expectation Suite retrieved from the Data Context. +In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. -The data used in the examples for this procedure is passenger data for the Titanic, including what class of ticket the passenger held and whether or not they survived the journey. +The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. 1. Determine the `condition_parser` for your `row_condition`. - The `condition_parser` defines the syntax of `row_condition` strings. When implementing conditional Expectations with pandas, this argument must be set to `"pandas"`. When implementing conditional Expectations with Spark or SQLAlchemy, this argument must be set to `"great_expectations"`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. If you are adding Expectation conditions using Spark or SQLAlchemy, set this argument to `"great_expectations"`. - Conditional Expectations will fail if the Batch they are validating comes from a different type of Data Source than is indicated by the `condition_parser` argument. + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. 2. Determine the `row_condition` expression. - The `row_condition` argument should be a boolean expression string which will be evaluated for each row in the Batch the Expectation validates. When the `row_condition` evaluates as `True` the row will be included in the Expectation's validations. When the `row_condition` evaluates as `False`, the Expectation will be skipped for that row. + The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. + + The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. - The syntax of the `row_condition` argument is based on the `condition_parser` that was previously specified. +3. Create the Expectation. - + An Expectation with conditions is created similarly to a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + + - In pandas the `row_condition` value is passed to `pandas.DataFrame.query()` before Expectation Validation and the returned rows from the evaluated Batch will be validated by the Conditional Expectation. - - + In pandas, the `row_condition` value is passed to `pandas.DataFrame.query()` prior to Expectation Validation, and the resulting rows from the evaluated Batch will undergo validation by the Expectation. - + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - pandas example row_condition" + ``` + + Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: - In Spark and SQLAlchemy, the `row_condition` value uses SQL syntax and is parsed as a data filter or a query before Expectation Validation. + ```python title="Python" + row_condition = "PClass=='1st'" # Don't do this. Single quotes aren't valid! + + row_condition=""" + PClass=="1st" + """ # Don't do this. Newlines and \n aren't valid! + + row_condition = 'PClass=="1st"' # Do this instead. + ``` - - + -3. Create a Conditional Expectation. + In Spark, the `row_condition` uses custom syntax, which is parsed as a data filter or query prior to Expectation Validation. - A Conditional Expectation is created exactly like a regular Expectation, except that the `row_condition` and `condition_parser` parameters are provided in addition to the Expectation's other arguments. - - + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - spark example row_condition" + ``` - + Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - pandas example row_condition" + ```python title="Python" + row_condition = "col('PClass')=='1st'" # Don't do this. Single quotes aren't valid! + + row_condition=""" + col("PClass")=="1st" + """ # Don't do this. Newlines and \n aren't valid! + + row_condition = 'col("PClass")=="1st"' # Do this instead. ``` - + - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - spark example row_condition" + In SQL, the `row_condition` uses custom syntax, which is parsed as a data filter or query prior to Expectation Validation. + + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - sql example row_condition" + ``` + + Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: + + ```python title="Python" + row_condition = "col('PClass')=='1st'" # Don't do this. Single quotes aren't valid! + + row_condition=""" + col("PClass")=="1st" + """ # Don't do this. Newlines and \n aren't valid! + + row_condition = 'col("PClass")=="1st"' # Do this instead. ``` - Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: - - ```python title="Python" - row_condition = "PClass=='1st'" # Don't do this. Single quotes aren't valid! - - row_condition=""" - PClass=="1st" - """ # Don't do this. Newlines and \n aren't valid! - - row_condition = 'PClass=="1st"' # Do this instead. - ``` - - + - With pandas you can indicate variables from the environment by prefacing them with `@`. You can also indicate columns with a space in their name by wrapping the name with backticks: `` ` ``. + In pandas, you can reference variables from the environment by prefixing them with `@`. Additionally, when a column name contains spaces, you can specify it by enclosing the name in backticks: `` ` ``. Some examples of valid `row_condition` values for pandas include: @@ -124,11 +143,32 @@ The data used in the examples for this procedure is passenger data for the Titan - + - For Spark and SQL, you will also want to specify your columns using the `col()` function. + For Spark, you should specify your columns using the `col()` function. - Some examples of valid `row_condition` values for Spark and SQL include: + Some examples of valid `row_condition` values for Spark include: + + ```python title="Python" + row_condition='col("foo") == "Two Two"' # foo is 'Two Two' + + row_condition='col("foo").notNull()' # foo is not null + + row_condition='col("foo") > 5' # foo is greater than 5 + + row_condition='col("foo") <= 3.14' # foo is less than 3.14 + + row_condition='col("foo") <= date("2023-03-13")' # foo is earlier than 2023-03-13 + + ``` + + + + + + For SQL, you should specify your columns using the `col()` function. + + Some examples of valid `row_condition` values for SQL include: ```python title="Python" row_condition='col("foo") == "Two Two"' # foo is 'Two Two' @@ -147,20 +187,38 @@ The data used in the examples for this procedure is passenger data for the Titan -4. Optional. Create additional Conditional Expectations. +4. Optional. Create additional Expectation conditions + + Expectations that have different conditions are treated as unique, even if they belong to the same type and apply to the same column within an Expectation Suite. This approach allows you to create one unconditional Expectation and an unlimited number of Conditional Expectations, each with a distinct condition. + + For instance, the following code establishes an Expectation that the value in the `"Survived"` column is either 0 or 1: + + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - example Expectation without conditions" + ``` + + And this code adds a condition to the Expectation that specifies the value of the `"Survived"` column is `1` if the individual was a first class passenger: + + + + - Expectations with different conditions are treated as unique even if they are of the same type and apply to the same column within an Expectation Suite. This allows you to create one unconditional Expectation and an arbitrary number of Conditional Expectations (each with a different condition). + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - pandas example Expectation with conditions" + ``` + - For example, the following code creates a unconditional Expectation that the value of the `"Survived"` column is either 0 or 1: + - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - example unconditional Expectation" + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - spark example Expectation with conditions" ``` + - And this code creates a Conditional version of the same Expectation that specifies the value of the `"Survived"` column is `1` if the individual was a first class passenger: + - ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - example conditional Expectation" + ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - sql example Expectation with conditions" ``` + + @@ -196,22 +254,21 @@ conditional_expectation = suite.add_expectation( -## Data Docs and Conditional Expectations +## Data Docs and Expectation conditions -Conditional Expectations are displayed differently from standard Expectations in the Data Docs. Each Conditional Expectation is qualified with *if 'row_condition_string', then values must be...* as shown in the following image: +Expectations with conditions are presented differently from standard Expectations in the Data Docs. Each Expectation with conditions is prefaced with *if 'row_condition_string', then values must be...* as illustrated in the following image: ![Image](/docs/oss/images/conditional_data_docs_screenshot.png) -If *'row_condition_string'* is a complex expression, it is split into several components to improve readability. +If the *'row_condition_string'* is a complex expression, it will be divided into several components to enhance readability. ## Scope and limitations -While conditions can be attached to most Expectations, the following Expectations cannot be conditioned and do not take the `row_condition` argument: +While conditions can be applied to most Expectations, the following Expectations cannot be conditioned and do not accept the `row_condition` argument: * `expect_column_to_exist` * `expect_table_columns_to_match_ordered_list` * `expect_table_columns_to_match_set` * `expect_table_column_count_to_be_between` * `expect_table_column_count_to_equal` - - +* `unexpected_rows_expectation` From d15babc84a47e3b6855b30f21a1fbac693c7172b Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 08:52:28 -0700 Subject: [PATCH 05/40] Remove sample code and add tabs for logic near the top --- .../expectation_conditions.md | 65 ++++++------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index d345c7f0885a..30c8d8adeb31 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -27,16 +27,7 @@ Great Expectations allows you to specify conditions for validating rows using th ### Procedure - - - + In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. @@ -44,7 +35,25 @@ The examples in this procedure use passenger data from the Titanic, which includ 1. Determine the `condition_parser` for your `row_condition`. - The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. If you are adding Expectation conditions using Spark or SQLAlchemy, set this argument to `"great_expectations"`. + + + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. + + + + + + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with Spark, set this argument to `"great_expectations"`. + + + + + + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with SQL, set this argument to `"great_expectations"`. + + + + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. @@ -220,40 +229,6 @@ The examples in this procedure use passenger data from the Titanic, which includ - - - - -```python title="Python" -import great_expectations as gx -import great_expectations.expectations as gxe -from great_expectations.core.expectation_suite import ExpectationSuite - -context = gx.get_context() - -suite = context.suites.add(ExpectationSuite(name="my_expectation_suite")) - -expectation = suite.add_expectation( - gxe.ExpectColumnValuesToBeInSet( - column="Survived", - value_set=[0, 1] - ) -) - -conditional_expectation = suite.add_expectation( - gxe.ExpectColumnValuesToBeInSet( - column='Survived', - value_set=[1], - condition_parser='pandas', - row_condition='PClass=="1st"' - ) -) -``` - - - - - ## Data Docs and Expectation conditions Expectations with conditions are presented differently from standard Expectations in the Data Docs. Each Expectation with conditions is prefaced with *if 'row_condition_string', then values must be...* as illustrated in the following image: From 495e82d03fb4d7d77a1a55ef96c14f6654d6d55e Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 09:15:29 -0700 Subject: [PATCH 06/40] Minor text change --- .../docs/core/customize_expectations/expectation_conditions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 30c8d8adeb31..b2208eaa80e6 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -65,7 +65,7 @@ The examples in this procedure use passenger data from the Titanic, which includ 3. Create the Expectation. - An Expectation with conditions is created similarly to a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. From 6f67e83325de6ab157dde460ebe6905504873e40 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 09:30:44 -0700 Subject: [PATCH 07/40] Add more test cases --- .../test_expectation_conditions.py | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py index 5f6df81d6265..519703730e9a 100644 --- a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py +++ b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py @@ -1,6 +1,7 @@ from datetime import datetime, timezone import pandas as pd +import pytest import great_expectations.expectations as gxe from great_expectations.compatibility.sqlalchemy import sqltypes @@ -15,29 +16,47 @@ SqliteDatasourceTestConfig, ) +data = pd.DataFrame( + { + "date": [ + datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), + ], + "quantity": [1, 2, 3], + "name": ["albert", "issac", "galileo"], + } +) + @parameterize_batch_for_data_sources( data_source_configs=[ PandasDataFrameDatasourceTestConfig(), PandasFilesystemCsvDatasourceTestConfig(), ], - data=pd.DataFrame( - { - "date": [ - datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), - ], - "quantity": [1, 2, 3], - } - ), + data=data, +) +@pytest.mark.parametrize( + "row_condition", + [ + pytest.param( + 'name=="albert"', + id="text", + ), + pytest.param( + "quantity<3", + id="number", + ), + ], ) -def test_expect_column_min_to_be_between__pandas_row_condition(batch_for_datasource) -> None: +def test_expect_column_min_to_be_between__pandas_row_condition( + batch_for_datasource, row_condition +) -> None: expectation = gxe.ExpectColumnMinToBeBetween( column="date", min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition="quantity<2", + row_condition=row_condition, condition_parser="pandas", ) result = batch_for_datasource.validate(expectation) @@ -52,23 +71,29 @@ def test_expect_column_min_to_be_between__pandas_row_condition(batch_for_datasou SnowflakeDatasourceTestConfig(column_types={"date": sqltypes.DATE}), SqliteDatasourceTestConfig(column_types={"date": sqltypes.DATE}), ], - data=pd.DataFrame( - { - "date": [ - datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), - ], - "quantity": [1, 2, 3], - } - ), + data=data, +) +@pytest.mark.parametrize( + "row_condition", + [ + pytest.param( + 'col("name")=="albert"', + id="text", + ), + pytest.param( + 'col("quantity")<3', + id="number", + ), + ], ) -def test_expect_column_min_to_be_between__sql_row_condition(batch_for_datasource) -> None: +def test_expect_column_min_to_be_between__sql_row_condition( + batch_for_datasource, row_condition +) -> None: expectation = gxe.ExpectColumnMinToBeBetween( column="date", min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition='col("quantity")<2', + row_condition=row_condition, condition_parser="great_expectations", ) result = batch_for_datasource.validate(expectation) From 4358e82723e68abd59eaa5500623caf745dc53c0 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 09:30:44 -0700 Subject: [PATCH 08/40] Add more test cases --- .../test_expectation_conditions.py | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py index 5f6df81d6265..519703730e9a 100644 --- a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py +++ b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py @@ -1,6 +1,7 @@ from datetime import datetime, timezone import pandas as pd +import pytest import great_expectations.expectations as gxe from great_expectations.compatibility.sqlalchemy import sqltypes @@ -15,29 +16,47 @@ SqliteDatasourceTestConfig, ) +data = pd.DataFrame( + { + "date": [ + datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), + ], + "quantity": [1, 2, 3], + "name": ["albert", "issac", "galileo"], + } +) + @parameterize_batch_for_data_sources( data_source_configs=[ PandasDataFrameDatasourceTestConfig(), PandasFilesystemCsvDatasourceTestConfig(), ], - data=pd.DataFrame( - { - "date": [ - datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), - ], - "quantity": [1, 2, 3], - } - ), + data=data, +) +@pytest.mark.parametrize( + "row_condition", + [ + pytest.param( + 'name=="albert"', + id="text", + ), + pytest.param( + "quantity<3", + id="number", + ), + ], ) -def test_expect_column_min_to_be_between__pandas_row_condition(batch_for_datasource) -> None: +def test_expect_column_min_to_be_between__pandas_row_condition( + batch_for_datasource, row_condition +) -> None: expectation = gxe.ExpectColumnMinToBeBetween( column="date", min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition="quantity<2", + row_condition=row_condition, condition_parser="pandas", ) result = batch_for_datasource.validate(expectation) @@ -52,23 +71,29 @@ def test_expect_column_min_to_be_between__pandas_row_condition(batch_for_datasou SnowflakeDatasourceTestConfig(column_types={"date": sqltypes.DATE}), SqliteDatasourceTestConfig(column_types={"date": sqltypes.DATE}), ], - data=pd.DataFrame( - { - "date": [ - datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), - ], - "quantity": [1, 2, 3], - } - ), + data=data, +) +@pytest.mark.parametrize( + "row_condition", + [ + pytest.param( + 'col("name")=="albert"', + id="text", + ), + pytest.param( + 'col("quantity")<3', + id="number", + ), + ], ) -def test_expect_column_min_to_be_between__sql_row_condition(batch_for_datasource) -> None: +def test_expect_column_min_to_be_between__sql_row_condition( + batch_for_datasource, row_condition +) -> None: expectation = gxe.ExpectColumnMinToBeBetween( column="date", min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition='col("quantity")<2', + row_condition=row_condition, condition_parser="great_expectations", ) result = batch_for_datasource.validate(expectation) From a819dc030d449f6fe58203a003fe22bf1a0941b5 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 09:33:52 -0700 Subject: [PATCH 09/40] Revert "Add more test cases" This reverts commit 6f67e83325de6ab157dde460ebe6905504873e40. --- .../test_expectation_conditions.py | 73 ++++++------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py index 519703730e9a..5f6df81d6265 100644 --- a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py +++ b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py @@ -1,7 +1,6 @@ from datetime import datetime, timezone import pandas as pd -import pytest import great_expectations.expectations as gxe from great_expectations.compatibility.sqlalchemy import sqltypes @@ -16,47 +15,29 @@ SqliteDatasourceTestConfig, ) -data = pd.DataFrame( - { - "date": [ - datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), - ], - "quantity": [1, 2, 3], - "name": ["albert", "issac", "galileo"], - } -) - @parameterize_batch_for_data_sources( data_source_configs=[ PandasDataFrameDatasourceTestConfig(), PandasFilesystemCsvDatasourceTestConfig(), ], - data=data, -) -@pytest.mark.parametrize( - "row_condition", - [ - pytest.param( - 'name=="albert"', - id="text", - ), - pytest.param( - "quantity<3", - id="number", - ), - ], + data=pd.DataFrame( + { + "date": [ + datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), + ], + "quantity": [1, 2, 3], + } + ), ) -def test_expect_column_min_to_be_between__pandas_row_condition( - batch_for_datasource, row_condition -) -> None: +def test_expect_column_min_to_be_between__pandas_row_condition(batch_for_datasource) -> None: expectation = gxe.ExpectColumnMinToBeBetween( column="date", min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition=row_condition, + row_condition="quantity<2", condition_parser="pandas", ) result = batch_for_datasource.validate(expectation) @@ -71,29 +52,23 @@ def test_expect_column_min_to_be_between__pandas_row_condition( SnowflakeDatasourceTestConfig(column_types={"date": sqltypes.DATE}), SqliteDatasourceTestConfig(column_types={"date": sqltypes.DATE}), ], - data=data, -) -@pytest.mark.parametrize( - "row_condition", - [ - pytest.param( - 'col("name")=="albert"', - id="text", - ), - pytest.param( - 'col("quantity")<3', - id="number", - ), - ], + data=pd.DataFrame( + { + "date": [ + datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), + datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), + ], + "quantity": [1, 2, 3], + } + ), ) -def test_expect_column_min_to_be_between__sql_row_condition( - batch_for_datasource, row_condition -) -> None: +def test_expect_column_min_to_be_between__sql_row_condition(batch_for_datasource) -> None: expectation = gxe.ExpectColumnMinToBeBetween( column="date", min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition=row_condition, + row_condition='col("quantity")<2', condition_parser="great_expectations", ) result = batch_for_datasource.validate(expectation) From 37b15eddfbde9531ddf3a054ac05a4cccacc436e Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 10:05:09 -0700 Subject: [PATCH 10/40] Minor text change again --- .../core/customize_expectations/expectation_conditions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index b2208eaa80e6..46a5432e187b 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -76,7 +76,7 @@ The examples in this procedure use passenger data from the Titanic, which includ ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - pandas example row_condition" ``` - Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: + Do not use single quotes, newlines, or `\n` in the specified `row_condition` as shown in the following examples: ```python title="Python" row_condition = "PClass=='1st'" # Don't do this. Single quotes aren't valid! @@ -97,7 +97,7 @@ The examples in this procedure use passenger data from the Titanic, which includ ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - spark example row_condition" ``` - Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: + Do not use single quotes, newlines, or `\n` in the specified `row_condition` as shown in the following examples: ```python title="Python" row_condition = "col('PClass')=='1st'" # Don't do this. Single quotes aren't valid! @@ -118,7 +118,7 @@ The examples in this procedure use passenger data from the Titanic, which includ ```python title="Python" name="docs/docusaurus/docs/core/customize_expectations/_examples/expectation_conditions.py - sql example row_condition" ``` - Do not use single quotes, newlines, or `\n` inside the specified `row_condition` as shown in the following examples: + Do not use single quotes, newlines, or `\n` in the specified `row_condition` as shown in the following examples: ```python title="Python" row_condition = "col('PClass')=='1st'" # Don't do this. Single quotes aren't valid! From 3106d5f2cefaca6c1ed88bde023f63205c750ff6 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 10:17:37 -0700 Subject: [PATCH 11/40] Minor update to snippet --- .../_examples/expectation_row_conditions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py index b5d32ab6025b..769c12ff46ca 100644 --- a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py +++ b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py @@ -79,8 +79,8 @@ def set_up_context_for_example(context): column="Survived", value_set=[1], # - condition_parser="spark", - row_condition='PClass=="1st"', + condition_parser="great_expectations", + row_condition='col("PClass")=="1st"', # ) # From 28ec5919bd862741818d39b38bad32cbe3e1cec3 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 11:54:12 -0700 Subject: [PATCH 12/40] Make ruff happy --- .../_examples/expectation_row_conditions.py | 4 +-- .../pandas_execution_engine.py | 4 +-- .../sparkdf_execution_engine.py | 6 ++-- .../sqlalchemy_execution_engine.py | 4 +-- .../expectations/expectation.py | 30 +++++++++++++++++++ .../expectations/model_field_types.py | 18 +++++++++-- 6 files changed, 55 insertions(+), 11 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py index 769c12ff46ca..b5d32ab6025b 100644 --- a/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py +++ b/docs/docusaurus/docs/core/customize_expectations/_examples/expectation_row_conditions.py @@ -79,8 +79,8 @@ def set_up_context_for_example(context): column="Survived", value_set=[1], # - condition_parser="great_expectations", - row_condition='col("PClass")=="1st"', + condition_parser="spark", + row_condition='PClass=="1st"', # ) # diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py index df8a9e5a17dc..9471fab46303 100644 --- a/great_expectations/execution_engine/pandas_execution_engine.py +++ b/great_expectations/execution_engine/pandas_execution_engine.py @@ -54,7 +54,7 @@ from great_expectations.execution_engine.partition_and_sample.pandas_data_sampler import ( PandasDataSampler, ) -from great_expectations.expectations.model_field_types import ConditionParser +from great_expectations.expectations.model_field_types import ConditionParserEnum if TYPE_CHECKING: from typing_extensions import TypeAlias @@ -526,7 +526,7 @@ def get_domain_records( # noqa: C901, PLR0912 if row_condition: condition_parser = domain_kwargs.get("condition_parser", None) - if condition_parser == ConditionParser.PANDAS: + if condition_parser == ConditionParserEnum.PANDAS: data = data.query(row_condition, parser=condition_parser) else: raise ValueError( # noqa: TRY003 diff --git a/great_expectations/execution_engine/sparkdf_execution_engine.py b/great_expectations/execution_engine/sparkdf_execution_engine.py index 4feb31c7c909..3669e0da1050 100644 --- a/great_expectations/execution_engine/sparkdf_execution_engine.py +++ b/great_expectations/execution_engine/sparkdf_execution_engine.py @@ -60,7 +60,7 @@ SparkDataSampler, ) from great_expectations.execution_engine.sparkdf_batch_data import SparkDFBatchData -from great_expectations.expectations.model_field_types import ConditionParser +from great_expectations.expectations.model_field_types import ConditionParserEnum from great_expectations.expectations.row_conditions import ( RowCondition, RowConditionParserType, @@ -665,9 +665,9 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 row_condition = domain_kwargs.get("row_condition", None) if row_condition: condition_parser = domain_kwargs.get("condition_parser", None) - if condition_parser == ConditionParser.SPARK: + if condition_parser == ConditionParserEnum.SPARK: data = data.filter(row_condition) - elif condition_parser in [ConditionParser.GX, ConditionParser.GX_DEPRECATED]: + elif condition_parser in [ConditionParserEnum.GX, ConditionParserEnum.GX_DEPRECATED]: parsed_condition = parse_condition_to_spark(row_condition) data = data.filter(parsed_condition) else: diff --git a/great_expectations/execution_engine/sqlalchemy_execution_engine.py b/great_expectations/execution_engine/sqlalchemy_execution_engine.py index e34e4e47de3f..d469070a44fa 100644 --- a/great_expectations/execution_engine/sqlalchemy_execution_engine.py +++ b/great_expectations/execution_engine/sqlalchemy_execution_engine.py @@ -54,7 +54,7 @@ from great_expectations.execution_engine.partition_and_sample.sqlalchemy_data_sampler import ( SqlAlchemyDataSampler, ) -from great_expectations.expectations.model_field_types import ConditionParser +from great_expectations.expectations.model_field_types import ConditionParserEnum from great_expectations.util import convert_to_json_serializable # noqa: TID251 from great_expectations.validator.computed_metric import MetricValue # noqa: TCH001 @@ -654,7 +654,7 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 # Filtering by row condition. if "row_condition" in domain_kwargs and domain_kwargs["row_condition"] is not None: condition_parser = domain_kwargs["condition_parser"] - if condition_parser in [ConditionParser.GX, ConditionParser.GX_DEPRECATED]: + if condition_parser in [ConditionParserEnum.GX, ConditionParserEnum.GX_DEPRECATED]: parsed_condition = parse_condition_to_sqlalchemy(domain_kwargs["row_condition"]) selectable = sa.select(sa.text("*")).select_from(selectable).where(parsed_condition) # type: ignore[arg-type] else: diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index 9b37a9ba40b4..25edde91c93b 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -65,6 +65,8 @@ WINDOWS_DESCRIPTION, ) from great_expectations.expectations.model_field_types import ( + ConditionParserEnum, + ConditionParserException, MostlyField, ) from great_expectations.expectations.registry import ( @@ -1816,6 +1818,13 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnAggregateExpectation] } ) + @pydantic.validator("condition_parser") + def condition_parser_allows_legacy_strings(cls, v): + if v not in ConditionParserEnum: + # error raised tells the user they should use what's in the public API + raise ConditionParserException(v) + return v + class ColumnMapExpectation(BatchExpectation, ABC): """Base class for ColumnMapExpectations. @@ -1873,6 +1882,13 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnMapExpectation]) -> N } ) + @pydantic.validator("condition_parser") + def condition_parser_allows_legacy_strings(cls, v): + if v not in ConditionParserEnum: + # error raised tells the user they should use what's in the public API + raise ConditionParserException(v) + return v + @classmethod @override def is_abstract(cls) -> bool: @@ -2142,6 +2158,13 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnPairMapExpectation]) } ) + @pydantic.validator("condition_parser") + def condition_parser_allows_legacy_strings(cls, v): + if v not in ConditionParserEnum: + # error raised tells the user they should use what's in the public API + raise ConditionParserException(v) + return v + @classmethod @override def is_abstract(cls) -> bool: @@ -2400,6 +2423,13 @@ def schema_extra(schema: Dict[str, Any], model: Type[MulticolumnMapExpectation]) } ) + @pydantic.validator("condition_parser") + def condition_parser_allows_legacy_strings(cls, v): + if v not in ConditionParserEnum: + # error raised tells the user they should use what's in the public API + raise ConditionParserException(v) + return v + @classmethod @override def is_abstract(cls) -> bool: diff --git a/great_expectations/expectations/model_field_types.py b/great_expectations/expectations/model_field_types.py index 4564c2bda2be..5a3b6151c29a 100644 --- a/great_expectations/expectations/model_field_types.py +++ b/great_expectations/expectations/model_field_types.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Sequence, Union +from typing import Literal, Sequence, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import Annotated @@ -69,10 +69,24 @@ ] -class ConditionParser(str, Enum): +# Type errors will surface in static analysis if not using these strings, +# but all strings in ConditionParserEnum will work at runtime +ConditionParser = Literal["great_expectations", "pandas"] + + +class ConditionParserException(ValueError): + def __init__(self, value: str) -> None: + super().__init__( + f"unexpected value `{value}`, condition_parser must be one of {ConditionParser}" + ) + + +class ConditionParserEnum(str, Enum): """Type of parser to be used to interpret a Row Condition.""" GX = "great_expectations" + # no longer part of public API, but remains to be non-breaking GX_DEPRECATED = "great_expectations__experimental__" PANDAS = "pandas" + # no longer part of public API, but remains to be non-breaking SPARK = "spark" From 105bdd36b0be61d41ff424b72b975233f895c229 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 11:55:57 -0700 Subject: [PATCH 13/40] Use type alias --- great_expectations/expectations/expectation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index 25edde91c93b..0e761760d32f 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -65,6 +65,7 @@ WINDOWS_DESCRIPTION, ) from great_expectations.expectations.model_field_types import ( + ConditionParser, ConditionParserEnum, ConditionParserException, MostlyField, @@ -1792,7 +1793,7 @@ class ColumnAggregateExpectation(BatchExpectation, ABC): column: StrictStr = Field(min_length=1, description=COLUMN_DESCRIPTION) row_condition: Union[str, None] = None - condition_parser: Union[Literal["pandas", "great_expectations"], None] = None + condition_parser: Union[ConditionParser, None] = None domain_keys: ClassVar[Tuple[str, ...]] = ( "batch_id", @@ -1852,7 +1853,7 @@ class ColumnMapExpectation(BatchExpectation, ABC): column: StrictStr = Field(min_length=1, description=COLUMN_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None - condition_parser: Union[Literal["pandas", "great_expectations"], None] = None + condition_parser: Union[ConditionParser, None] = None catch_exceptions: bool = True @@ -2127,7 +2128,7 @@ class ColumnPairMapExpectation(BatchExpectation, ABC): column_B: StrictStr = Field(min_length=1, description=COLUMN_B_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None - condition_parser: Union[Literal["pandas", "great_expectations"], None] = None + condition_parser: Union[ConditionParser, None] = None catch_exceptions: bool = True @@ -2390,7 +2391,7 @@ class MulticolumnMapExpectation(BatchExpectation, ABC): column_list: List[StrictStr] = pydantic.Field(description=COLUMN_LIST_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None - condition_parser: Union[Literal["pandas", "great_expectations"], None] = None + condition_parser: Union[ConditionParser, None] = None ignore_row_if: Literal["all_values_are_missing", "any_value_is_missing", "never"] = ( "all_values_are_missing" ) From 2805f475594c5700d1ca22fa29891185b30d5f9a Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 12:55:56 -0700 Subject: [PATCH 14/40] Update enum handling for python 3.12 --- .../expectations/expectation.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index 0e761760d32f..91fb21ede97c 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -1821,9 +1821,11 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnAggregateExpectation] @pydantic.validator("condition_parser") def condition_parser_allows_legacy_strings(cls, v): - if v not in ConditionParserEnum: + try: + ConditionParserEnum(v) + except ValueError as e: # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) + raise ConditionParserException(v) from e return v @@ -1885,9 +1887,11 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnMapExpectation]) -> N @pydantic.validator("condition_parser") def condition_parser_allows_legacy_strings(cls, v): - if v not in ConditionParserEnum: + try: + ConditionParserEnum(v) + except ValueError as e: # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) + raise ConditionParserException(v) from e return v @classmethod @@ -2161,9 +2165,11 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnPairMapExpectation]) @pydantic.validator("condition_parser") def condition_parser_allows_legacy_strings(cls, v): - if v not in ConditionParserEnum: + try: + ConditionParserEnum(v) + except ValueError as e: # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) + raise ConditionParserException(v) from e return v @classmethod @@ -2426,9 +2432,11 @@ def schema_extra(schema: Dict[str, Any], model: Type[MulticolumnMapExpectation]) @pydantic.validator("condition_parser") def condition_parser_allows_legacy_strings(cls, v): - if v not in ConditionParserEnum: + try: + ConditionParserEnum(v) + except ValueError as e: # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) + raise ConditionParserException(v) from e return v @classmethod From 5ad1ebc011d8ad0da8187f5da57a23ab7851dc49 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 13:25:58 -0700 Subject: [PATCH 15/40] Simplfy --- .../expectations/expectation.py | 33 +++++-------------- .../expectations/model_field_types.py | 7 ---- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index 91fb21ede97c..0c3506e214ae 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -67,7 +67,6 @@ from great_expectations.expectations.model_field_types import ( ConditionParser, ConditionParserEnum, - ConditionParserException, MostlyField, ) from great_expectations.expectations.registry import ( @@ -1820,12 +1819,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnAggregateExpectation] ) @pydantic.validator("condition_parser") - def condition_parser_allows_legacy_strings(cls, v): - try: - ConditionParserEnum(v) - except ValueError as e: - # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) from e + def condition_parser_allows_deprecated_strings(cls, v): + ConditionParserEnum(v) return v @@ -1886,12 +1881,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnMapExpectation]) -> N ) @pydantic.validator("condition_parser") - def condition_parser_allows_legacy_strings(cls, v): - try: - ConditionParserEnum(v) - except ValueError as e: - # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) from e + def condition_parser_allows_deprecated_strings(cls, v): + ConditionParserEnum(v) return v @classmethod @@ -2164,12 +2155,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnPairMapExpectation]) ) @pydantic.validator("condition_parser") - def condition_parser_allows_legacy_strings(cls, v): - try: - ConditionParserEnum(v) - except ValueError as e: - # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) from e + def condition_parser_allows_deprecated_strings(cls, v): + ConditionParserEnum(v) return v @classmethod @@ -2431,12 +2418,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[MulticolumnMapExpectation]) ) @pydantic.validator("condition_parser") - def condition_parser_allows_legacy_strings(cls, v): - try: - ConditionParserEnum(v) - except ValueError as e: - # error raised tells the user they should use what's in the public API - raise ConditionParserException(v) from e + def condition_parser_allows_deprecated_strings(cls, v): + ConditionParserEnum(v) return v @classmethod diff --git a/great_expectations/expectations/model_field_types.py b/great_expectations/expectations/model_field_types.py index 5a3b6151c29a..b10043c045b5 100644 --- a/great_expectations/expectations/model_field_types.py +++ b/great_expectations/expectations/model_field_types.py @@ -74,13 +74,6 @@ ConditionParser = Literal["great_expectations", "pandas"] -class ConditionParserException(ValueError): - def __init__(self, value: str) -> None: - super().__init__( - f"unexpected value `{value}`, condition_parser must be one of {ConditionParser}" - ) - - class ConditionParserEnum(str, Enum): """Type of parser to be used to interpret a Row Condition.""" From d8ead36de55ab0f1992e7cdfafeb3e1fbf4e2dae Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 13:57:58 -0700 Subject: [PATCH 16/40] Update schemas again --- .../ExpectColumnDistinctValuesToBeInSet.json | 4 ++-- ...ExpectColumnDistinctValuesToContainSet.json | 4 ++-- .../ExpectColumnDistinctValuesToEqualSet.json | 4 ++-- .../ExpectColumnKLDivergenceToBeLessThan.json | 4 ++-- .../schemas/ExpectColumnMaxToBeBetween.json | 4 ++-- .../schemas/ExpectColumnMeanToBeBetween.json | 4 ++-- .../schemas/ExpectColumnMedianToBeBetween.json | 4 ++-- .../schemas/ExpectColumnMinToBeBetween.json | 4 ++-- .../ExpectColumnMostCommonValueToBeInSet.json | 4 ++-- ...xpectColumnPairValuesAToBeGreaterThanB.json | 4 ++-- .../ExpectColumnPairValuesToBeEqual.json | 4 ++-- .../ExpectColumnPairValuesToBeInSet.json | 4 ++-- ...umnProportionOfUniqueValuesToBeBetween.json | 4 ++-- .../ExpectColumnQuantileValuesToBeBetween.json | 4 ++-- .../schemas/ExpectColumnStdevToBeBetween.json | 4 ++-- .../schemas/ExpectColumnSumToBeBetween.json | 4 ++-- ...xpectColumnUniqueValueCountToBeBetween.json | 4 ++-- .../ExpectColumnValueLengthsToBeBetween.json | 4 ++-- .../ExpectColumnValueLengthsToEqual.json | 4 ++-- .../ExpectColumnValueZScoresToBeLessThan.json | 4 ++-- .../schemas/ExpectColumnValuesToBeBetween.json | 4 ++-- .../schemas/ExpectColumnValuesToBeInSet.json | 4 ++-- .../ExpectColumnValuesToBeInTypeList.json | 4 ++-- .../schemas/ExpectColumnValuesToBeNull.json | 4 ++-- .../schemas/ExpectColumnValuesToBeOfType.json | 4 ++-- .../schemas/ExpectColumnValuesToBeUnique.json | 4 ++-- .../ExpectColumnValuesToMatchLikePattern.json | 4 ++-- ...pectColumnValuesToMatchLikePatternList.json | 4 ++-- .../ExpectColumnValuesToMatchRegex.json | 4 ++-- .../ExpectColumnValuesToMatchRegexList.json | 4 ++-- .../ExpectColumnValuesToNotBeInSet.json | 4 ++-- .../schemas/ExpectColumnValuesToNotBeNull.json | 4 ++-- ...xpectColumnValuesToNotMatchLikePattern.json | 4 ++-- ...tColumnValuesToNotMatchLikePatternList.json | 4 ++-- .../ExpectColumnValuesToNotMatchRegex.json | 4 ++-- .../ExpectColumnValuesToNotMatchRegexList.json | 4 ++-- .../ExpectCompoundColumnsToBeUnique.json | 4 ++-- .../schemas/ExpectMulticolumnSumToEqual.json | 4 ++-- ...lectColumnValuesToBeUniqueWithinRecord.json | 4 ++-- .../ExpectTableRowCountToBeBetween.json | 18 ++++++------------ .../schemas/ExpectTableRowCountToEqual.json | 18 ++++++------------ .../ExpectTableRowCountToEqualOtherTable.json | 18 ++++++------------ 42 files changed, 96 insertions(+), 114 deletions(-) diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json index 3bde3dd7533a..dac588fa91ef 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json index 2097b90dc92a..efeb4c600b85 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json index cf2886baabe2..fae264474230 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json index 21b0e4423934..388e19f33d2e 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json index 28ab33e6031b..7e59dd21cf1a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json index df6a8cc0e6f5..d7bb6fdbf8e1 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json index fe730b4d4d08..068ebb2c7d05 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json index cda77859b85c..e0ade2c6b752 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json index 8960f0485595..558a2d3170ad 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json index 453e8dfe954c..2d8afbbc115f 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json @@ -94,8 +94,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json index a650d2f419b9..0b03d7752ced 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json @@ -94,8 +94,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json index 80ea0b8137bb..5afeee2c6f4e 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json @@ -94,8 +94,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json index 4a67051c8e82..7f932790dbc8 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json index 4d8c10dde714..44b3ebb1ec3d 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json index d9a5c081894b..6bfdcf410dd0 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json index 83a11c74534e..2e0f93de3022 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json index 852139485e2e..f82ff0bacc91 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json @@ -79,8 +79,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json index 5fe0ef3d473a..e7190c8e4d27 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json index 07b5ea6a5e73..5f234c5588a2 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json index 8e62107caa78..f462163bb986 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json index e4bb826b46d3..d32ffc39378a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json index 9f7664cc261b..fe13c5ae75ea 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json index 856b02b161ae..a556534e5a3c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json index f7fcd56b5fad..8dc979a1e19c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json index ed36b39da4d1..6f536a469bb5 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json index aadf6d7455d0..1e43fea8c8dc 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json index 1d4d54c99df5..d9aa9d8d5846 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json index 01aa524c95fa..c26936e79c82 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json index 85569d993f46..d01ebecd73bd 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json index f6fef176ff0c..9eb63583a233 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json index 87a3c27dc702..9238e30908f4 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json index a9dda798868c..5b1357c76a82 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json index 28d64d433ecb..375a4589beb5 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json index 11581ee55455..b6ba2e811a0f 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json index 3027cb4a81bc..d89dfbb01c31 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json index abfded6ce70a..634c8c3e2f86 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json @@ -88,8 +88,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json b/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json index 1f83cb4c355d..15d1e14df2d0 100644 --- a/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json +++ b/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json @@ -90,8 +90,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json b/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json index f7c5629a9c32..4520585ec6d3 100644 --- a/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json @@ -90,8 +90,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json b/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json index 8b74fe2ddf48..b139d43a51a9 100644 --- a/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json +++ b/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json @@ -90,8 +90,8 @@ "condition_parser": { "title": "Condition Parser", "enum": [ - "pandas", - "great_expectations" + "great_expectations", + "pandas" ], "type": "string" }, diff --git a/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json index 62ee93d1a62b..8d0d4fd127c5 100644 --- a/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json @@ -103,7 +103,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "great_expectations", + "pandas" + ], + "type": "string" }, "metadata": { "type": "object", @@ -232,17 +237,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json index a648c04ccb15..5b3017ec2d4c 100644 --- a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json @@ -83,7 +83,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "great_expectations", + "pandas" + ], + "type": "string" }, "metadata": { "type": "object", @@ -215,17 +220,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json index 82756bf5b593..5d7c4c87f3e5 100644 --- a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json +++ b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json @@ -76,7 +76,12 @@ "type": "string" }, "condition_parser": { - "$ref": "#/definitions/ConditionParser" + "title": "Condition Parser", + "enum": [ + "great_expectations", + "pandas" + ], + "type": "string" }, "metadata": { "type": "object", @@ -205,17 +210,6 @@ "offset" ], "additionalProperties": false - }, - "ConditionParser": { - "title": "ConditionParser", - "description": "Type of parser to be used to interpret a Row Condition.", - "enum": [ - "great_expectations", - "great_expectations__experimental__", - "pandas", - "spark" - ], - "type": "string" } } } From 5cea1c639435ae3fd3c086ab56ce4e234c99ee62 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 14:00:14 -0700 Subject: [PATCH 17/40] condition_parser validator should allow None --- great_expectations/expectations/expectation.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index 0c3506e214ae..b134adabf498 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -1820,7 +1820,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnAggregateExpectation] @pydantic.validator("condition_parser") def condition_parser_allows_deprecated_strings(cls, v): - ConditionParserEnum(v) + if v is not None: + ConditionParserEnum(v) return v @@ -1882,7 +1883,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnMapExpectation]) -> N @pydantic.validator("condition_parser") def condition_parser_allows_deprecated_strings(cls, v): - ConditionParserEnum(v) + if v is not None: + ConditionParserEnum(v) return v @classmethod @@ -2156,7 +2158,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnPairMapExpectation]) @pydantic.validator("condition_parser") def condition_parser_allows_deprecated_strings(cls, v): - ConditionParserEnum(v) + if v is not None: + ConditionParserEnum(v) return v @classmethod @@ -2419,7 +2422,8 @@ def schema_extra(schema: Dict[str, Any], model: Type[MulticolumnMapExpectation]) @pydantic.validator("condition_parser") def condition_parser_allows_deprecated_strings(cls, v): - ConditionParserEnum(v) + if v is not None: + ConditionParserEnum(v) return v @classmethod From e8a5d9a0fc523fe6803868d4043bc179f83b7687 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 15:13:56 -0700 Subject: [PATCH 18/40] Move functional changes into another PR --- .../pandas_execution_engine.py | 4 +-- .../sparkdf_execution_engine.py | 6 ++--- .../sqlalchemy_execution_engine.py | 4 +-- .../ExpectColumnDistinctValuesToBeInSet.json | 18 ++++++++----- ...xpectColumnDistinctValuesToContainSet.json | 18 ++++++++----- .../ExpectColumnDistinctValuesToEqualSet.json | 18 ++++++++----- .../ExpectColumnKLDivergenceToBeLessThan.json | 18 ++++++++----- .../schemas/ExpectColumnMaxToBeBetween.json | 18 ++++++++----- .../schemas/ExpectColumnMeanToBeBetween.json | 18 ++++++++----- .../ExpectColumnMedianToBeBetween.json | 18 ++++++++----- .../schemas/ExpectColumnMinToBeBetween.json | 18 ++++++++----- .../ExpectColumnMostCommonValueToBeInSet.json | 18 ++++++++----- ...pectColumnPairValuesAToBeGreaterThanB.json | 18 ++++++++----- .../ExpectColumnPairValuesToBeEqual.json | 18 ++++++++----- .../ExpectColumnPairValuesToBeInSet.json | 18 ++++++++----- ...mnProportionOfUniqueValuesToBeBetween.json | 18 ++++++++----- ...ExpectColumnQuantileValuesToBeBetween.json | 18 ++++++++----- .../schemas/ExpectColumnStdevToBeBetween.json | 18 ++++++++----- .../schemas/ExpectColumnSumToBeBetween.json | 18 ++++++++----- ...pectColumnUniqueValueCountToBeBetween.json | 18 ++++++++----- .../ExpectColumnValueLengthsToBeBetween.json | 18 ++++++++----- .../ExpectColumnValueLengthsToEqual.json | 18 ++++++++----- .../ExpectColumnValueZScoresToBeLessThan.json | 18 ++++++++----- .../ExpectColumnValuesToBeBetween.json | 18 ++++++++----- .../schemas/ExpectColumnValuesToBeInSet.json | 18 ++++++++----- .../ExpectColumnValuesToBeInTypeList.json | 18 ++++++++----- .../schemas/ExpectColumnValuesToBeNull.json | 18 ++++++++----- .../schemas/ExpectColumnValuesToBeOfType.json | 18 ++++++++----- .../schemas/ExpectColumnValuesToBeUnique.json | 18 ++++++++----- .../ExpectColumnValuesToMatchLikePattern.json | 18 ++++++++----- ...ectColumnValuesToMatchLikePatternList.json | 18 ++++++++----- .../ExpectColumnValuesToMatchRegex.json | 18 ++++++++----- .../ExpectColumnValuesToMatchRegexList.json | 18 ++++++++----- .../ExpectColumnValuesToNotBeInSet.json | 18 ++++++++----- .../ExpectColumnValuesToNotBeNull.json | 18 ++++++++----- ...pectColumnValuesToNotMatchLikePattern.json | 18 ++++++++----- ...ColumnValuesToNotMatchLikePatternList.json | 18 ++++++++----- .../ExpectColumnValuesToNotMatchRegex.json | 18 ++++++++----- ...ExpectColumnValuesToNotMatchRegexList.json | 18 ++++++++----- .../ExpectCompoundColumnsToBeUnique.json | 18 ++++++++----- .../schemas/ExpectMulticolumnSumToEqual.json | 18 ++++++++----- ...ectColumnValuesToBeUniqueWithinRecord.json | 18 ++++++++----- .../ExpectTableRowCountToBeBetween.json | 18 ++++++++----- .../schemas/ExpectTableRowCountToEqual.json | 18 ++++++++----- .../ExpectTableRowCountToEqualOtherTable.json | 18 ++++++++----- .../expectations/expectation.py | 25 ------------------- .../expectations/model_field_types.py | 11 ++------ 47 files changed, 513 insertions(+), 293 deletions(-) diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py index 9471fab46303..df8a9e5a17dc 100644 --- a/great_expectations/execution_engine/pandas_execution_engine.py +++ b/great_expectations/execution_engine/pandas_execution_engine.py @@ -54,7 +54,7 @@ from great_expectations.execution_engine.partition_and_sample.pandas_data_sampler import ( PandasDataSampler, ) -from great_expectations.expectations.model_field_types import ConditionParserEnum +from great_expectations.expectations.model_field_types import ConditionParser if TYPE_CHECKING: from typing_extensions import TypeAlias @@ -526,7 +526,7 @@ def get_domain_records( # noqa: C901, PLR0912 if row_condition: condition_parser = domain_kwargs.get("condition_parser", None) - if condition_parser == ConditionParserEnum.PANDAS: + if condition_parser == ConditionParser.PANDAS: data = data.query(row_condition, parser=condition_parser) else: raise ValueError( # noqa: TRY003 diff --git a/great_expectations/execution_engine/sparkdf_execution_engine.py b/great_expectations/execution_engine/sparkdf_execution_engine.py index 3669e0da1050..4feb31c7c909 100644 --- a/great_expectations/execution_engine/sparkdf_execution_engine.py +++ b/great_expectations/execution_engine/sparkdf_execution_engine.py @@ -60,7 +60,7 @@ SparkDataSampler, ) from great_expectations.execution_engine.sparkdf_batch_data import SparkDFBatchData -from great_expectations.expectations.model_field_types import ConditionParserEnum +from great_expectations.expectations.model_field_types import ConditionParser from great_expectations.expectations.row_conditions import ( RowCondition, RowConditionParserType, @@ -665,9 +665,9 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 row_condition = domain_kwargs.get("row_condition", None) if row_condition: condition_parser = domain_kwargs.get("condition_parser", None) - if condition_parser == ConditionParserEnum.SPARK: + if condition_parser == ConditionParser.SPARK: data = data.filter(row_condition) - elif condition_parser in [ConditionParserEnum.GX, ConditionParserEnum.GX_DEPRECATED]: + elif condition_parser in [ConditionParser.GX, ConditionParser.GX_DEPRECATED]: parsed_condition = parse_condition_to_spark(row_condition) data = data.filter(parsed_condition) else: diff --git a/great_expectations/execution_engine/sqlalchemy_execution_engine.py b/great_expectations/execution_engine/sqlalchemy_execution_engine.py index d469070a44fa..e34e4e47de3f 100644 --- a/great_expectations/execution_engine/sqlalchemy_execution_engine.py +++ b/great_expectations/execution_engine/sqlalchemy_execution_engine.py @@ -54,7 +54,7 @@ from great_expectations.execution_engine.partition_and_sample.sqlalchemy_data_sampler import ( SqlAlchemyDataSampler, ) -from great_expectations.expectations.model_field_types import ConditionParserEnum +from great_expectations.expectations.model_field_types import ConditionParser from great_expectations.util import convert_to_json_serializable # noqa: TID251 from great_expectations.validator.computed_metric import MetricValue # noqa: TCH001 @@ -654,7 +654,7 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 # Filtering by row condition. if "row_condition" in domain_kwargs and domain_kwargs["row_condition"] is not None: condition_parser = domain_kwargs["condition_parser"] - if condition_parser in [ConditionParserEnum.GX, ConditionParserEnum.GX_DEPRECATED]: + if condition_parser in [ConditionParser.GX, ConditionParser.GX_DEPRECATED]: parsed_condition = parse_condition_to_sqlalchemy(domain_kwargs["row_condition"]) selectable = sa.select(sa.text("*")).select_from(selectable).where(parsed_condition) # type: ignore[arg-type] else: diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json index dac588fa91ef..edd4d4697f50 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToBeInSet.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_set": { "title": "Value Set", @@ -285,6 +280,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json index efeb4c600b85..a96a5306ac91 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToContainSet.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_set": { "title": "Value Set", @@ -285,6 +280,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json index fae264474230..24a2264c8d5a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnDistinctValuesToEqualSet.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_set": { "title": "Value Set", @@ -285,6 +280,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json index 388e19f33d2e..c314e5e1bd14 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnKLDivergenceToBeLessThan.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "partition_object": { "title": "Partition Object", @@ -287,6 +282,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json index 7e59dd21cf1a..4be7a8bbf66d 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMaxToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -266,6 +261,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json index d7bb6fdbf8e1..7fd498935b05 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMeanToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -266,6 +261,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json index 068ebb2c7d05..740cfe7a4a9e 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMedianToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -266,6 +261,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json index e0ade2c6b752..01d68005b67c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMinToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -266,6 +261,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json index 558a2d3170ad..6608c3e14414 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnMostCommonValueToBeInSet.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_set": { "title": "Value Set", @@ -290,6 +285,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json index 2d8afbbc115f..2579fef51888 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesAToBeGreaterThanB.json @@ -92,12 +92,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "or_equal": { "title": "Or Equal", @@ -245,6 +240,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json index 0b03d7752ced..aa7e980fe151 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeEqual.json @@ -92,12 +92,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "ignore_row_if": { "title": "Ignore Row If", @@ -241,6 +236,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json index 5afeee2c6f4e..41197fe628b7 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnPairValuesToBeInSet.json @@ -92,12 +92,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_pairs_set": { "title": "Value Pairs Set", @@ -254,6 +249,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json index 7f932790dbc8..1d44cd059aae 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnProportionOfUniqueValuesToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -283,6 +278,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json index 44b3ebb1ec3d..7eccce672d97 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnQuantileValuesToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "quantile_ranges": { "title": "Quantile Ranges", @@ -236,6 +231,17 @@ ], "additionalProperties": false }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" + }, "QuantileRange": { "title": "QuantileRange", "type": "object", diff --git a/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json index 6bfdcf410dd0..36a2ac3346c3 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnStdevToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -266,6 +261,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json index 2e0f93de3022..a55c0f3d732a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnSumToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -265,6 +260,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json index f82ff0bacc91..f5d0a23ac260 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnUniqueValueCountToBeBetween.json @@ -77,12 +77,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -283,6 +278,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json index e7190c8e4d27..3c3fe0298148 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToBeBetween.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -267,6 +262,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json index 5f234c5588a2..7d49abc4515a 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueLengthsToEqual.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value": { "title": "Value", @@ -236,6 +231,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json b/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json index f462163bb986..7ecfb20bdcbb 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValueZScoresToBeLessThan.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "threshold": { "title": "Threshold", @@ -249,6 +244,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json index d32ffc39378a..368652e22f54 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeBetween.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "min_value": { "title": "Min Value", @@ -274,6 +269,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json index fe13c5ae75ea..11fca2518450 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInSet.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_set": { "title": "Value Set", @@ -294,6 +289,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json index a556534e5a3c..a1b69a6bebf5 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeInTypeList.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "type_list": { "title": "Type List", @@ -238,6 +233,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json index 8dc979a1e19c..b3ff965417eb 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeNull.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "metadata": { "type": "object", @@ -223,6 +218,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json index 6f536a469bb5..0d9f52413744 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeOfType.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "type_": { "title": "Type ", @@ -229,6 +224,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json index 1e43fea8c8dc..778ca834867f 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToBeUnique.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "metadata": { "type": "object", @@ -222,6 +217,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json index d9aa9d8d5846..11adfe72a4d2 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePattern.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "like_pattern": { "title": "Like Pattern", @@ -232,6 +227,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json index c26936e79c82..0c1955b4645e 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchLikePatternList.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "like_pattern_list": { "title": "Like Pattern List", @@ -245,6 +240,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json index d01ebecd73bd..f9f30ac80ca7 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegex.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "regex": { "title": "Regex", @@ -232,6 +227,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json index 9eb63583a233..73c05b4cef16 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToMatchRegexList.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "regex_list": { "title": "Regex List", @@ -245,6 +240,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json index 9238e30908f4..c380f9087c5f 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeInSet.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "value_set": { "title": "Value Set", @@ -294,6 +289,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json index 5b1357c76a82..52e4418e5b62 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeNull.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "metadata": { "type": "object", @@ -223,6 +218,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json index 375a4589beb5..3a092c1e1ed0 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePattern.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "like_pattern": { "title": "Like Pattern", @@ -232,6 +227,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json index b6ba2e811a0f..b8e938b1162c 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchLikePatternList.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "like_pattern_list": { "title": "Like Pattern List", @@ -235,6 +230,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json index d89dfbb01c31..fa4509fde816 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegex.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "regex": { "title": "Regex", @@ -232,6 +227,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json index 634c8c3e2f86..e64b79a0fe26 100644 --- a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotMatchRegexList.json @@ -86,12 +86,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "regex_list": { "title": "Regex List", @@ -235,6 +230,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json b/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json index 15d1e14df2d0..7a81307fb399 100644 --- a/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json +++ b/great_expectations/expectations/core/schemas/ExpectCompoundColumnsToBeUnique.json @@ -88,12 +88,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "ignore_row_if": { "title": "Ignore Row If", @@ -235,6 +230,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json b/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json index 4520585ec6d3..9126f935700f 100644 --- a/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectMulticolumnSumToEqual.json @@ -88,12 +88,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "ignore_row_if": { "title": "Ignore Row If", @@ -242,6 +237,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json b/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json index b139d43a51a9..df1fcd601126 100644 --- a/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json +++ b/great_expectations/expectations/core/schemas/ExpectSelectColumnValuesToBeUniqueWithinRecord.json @@ -88,12 +88,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "ignore_row_if": { "title": "Ignore Row If", @@ -231,6 +226,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json b/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json index 8d0d4fd127c5..62ee93d1a62b 100644 --- a/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json +++ b/great_expectations/expectations/core/schemas/ExpectTableRowCountToBeBetween.json @@ -103,12 +103,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "metadata": { "type": "object", @@ -237,6 +232,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json index 5b3017ec2d4c..a648c04ccb15 100644 --- a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json +++ b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqual.json @@ -83,12 +83,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "metadata": { "type": "object", @@ -220,6 +215,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json index 5d7c4c87f3e5..82756bf5b593 100644 --- a/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json +++ b/great_expectations/expectations/core/schemas/ExpectTableRowCountToEqualOtherTable.json @@ -76,12 +76,7 @@ "type": "string" }, "condition_parser": { - "title": "Condition Parser", - "enum": [ - "great_expectations", - "pandas" - ], - "type": "string" + "$ref": "#/definitions/ConditionParser" }, "metadata": { "type": "object", @@ -210,6 +205,17 @@ "offset" ], "additionalProperties": false + }, + "ConditionParser": { + "title": "ConditionParser", + "description": "Type of parser to be used to interpret a Row Condition.", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" } } } diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index b134adabf498..b84882b0d4a4 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -66,7 +66,6 @@ ) from great_expectations.expectations.model_field_types import ( ConditionParser, - ConditionParserEnum, MostlyField, ) from great_expectations.expectations.registry import ( @@ -1818,12 +1817,6 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnAggregateExpectation] } ) - @pydantic.validator("condition_parser") - def condition_parser_allows_deprecated_strings(cls, v): - if v is not None: - ConditionParserEnum(v) - return v - class ColumnMapExpectation(BatchExpectation, ABC): """Base class for ColumnMapExpectations. @@ -1881,12 +1874,6 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnMapExpectation]) -> N } ) - @pydantic.validator("condition_parser") - def condition_parser_allows_deprecated_strings(cls, v): - if v is not None: - ConditionParserEnum(v) - return v - @classmethod @override def is_abstract(cls) -> bool: @@ -2156,12 +2143,6 @@ def schema_extra(schema: Dict[str, Any], model: Type[ColumnPairMapExpectation]) } ) - @pydantic.validator("condition_parser") - def condition_parser_allows_deprecated_strings(cls, v): - if v is not None: - ConditionParserEnum(v) - return v - @classmethod @override def is_abstract(cls) -> bool: @@ -2420,12 +2401,6 @@ def schema_extra(schema: Dict[str, Any], model: Type[MulticolumnMapExpectation]) } ) - @pydantic.validator("condition_parser") - def condition_parser_allows_deprecated_strings(cls, v): - if v is not None: - ConditionParserEnum(v) - return v - @classmethod @override def is_abstract(cls) -> bool: diff --git a/great_expectations/expectations/model_field_types.py b/great_expectations/expectations/model_field_types.py index b10043c045b5..4564c2bda2be 100644 --- a/great_expectations/expectations/model_field_types.py +++ b/great_expectations/expectations/model_field_types.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Literal, Sequence, Union +from typing import Sequence, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import Annotated @@ -69,17 +69,10 @@ ] -# Type errors will surface in static analysis if not using these strings, -# but all strings in ConditionParserEnum will work at runtime -ConditionParser = Literal["great_expectations", "pandas"] - - -class ConditionParserEnum(str, Enum): +class ConditionParser(str, Enum): """Type of parser to be used to interpret a Row Condition.""" GX = "great_expectations" - # no longer part of public API, but remains to be non-breaking GX_DEPRECATED = "great_expectations__experimental__" PANDAS = "pandas" - # no longer part of public API, but remains to be non-breaking SPARK = "spark" From d0561a58461ca70d246512b66e402b7e7e488558 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 15:16:31 -0700 Subject: [PATCH 19/40] Move functional changes into another PR --- .../test_expectation_conditions.py | 100 ------------------ ...ct_column_value_lengths_to_be_between.json | 2 +- .../expect_column_value_lengths_to_equal.json | 2 +- .../expect_column_values_to_be_between.json | 2 +- ..._column_values_to_be_between_tz_naive.json | 2 +- .../expect_column_values_to_be_in_set.json | 2 +- 6 files changed, 5 insertions(+), 105 deletions(-) delete mode 100644 tests/integration/data_sources_and_expectations/test_expectation_conditions.py diff --git a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py b/tests/integration/data_sources_and_expectations/test_expectation_conditions.py deleted file mode 100644 index 519703730e9a..000000000000 --- a/tests/integration/data_sources_and_expectations/test_expectation_conditions.py +++ /dev/null @@ -1,100 +0,0 @@ -from datetime import datetime, timezone - -import pandas as pd -import pytest - -import great_expectations.expectations as gxe -from great_expectations.compatibility.sqlalchemy import sqltypes -from tests.integration.conftest import parameterize_batch_for_data_sources -from tests.integration.test_utils.data_source_config import ( - MSSQLDatasourceTestConfig, - MySQLDatasourceTestConfig, - PandasDataFrameDatasourceTestConfig, - PandasFilesystemCsvDatasourceTestConfig, - PostgreSQLDatasourceTestConfig, - SnowflakeDatasourceTestConfig, - SqliteDatasourceTestConfig, -) - -data = pd.DataFrame( - { - "date": [ - datetime(year=2021, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2022, month=1, day=31, tzinfo=timezone.utc).date(), - datetime(year=2023, month=1, day=31, tzinfo=timezone.utc).date(), - ], - "quantity": [1, 2, 3], - "name": ["albert", "issac", "galileo"], - } -) - - -@parameterize_batch_for_data_sources( - data_source_configs=[ - PandasDataFrameDatasourceTestConfig(), - PandasFilesystemCsvDatasourceTestConfig(), - ], - data=data, -) -@pytest.mark.parametrize( - "row_condition", - [ - pytest.param( - 'name=="albert"', - id="text", - ), - pytest.param( - "quantity<3", - id="number", - ), - ], -) -def test_expect_column_min_to_be_between__pandas_row_condition( - batch_for_datasource, row_condition -) -> None: - expectation = gxe.ExpectColumnMinToBeBetween( - column="date", - min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), - max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition=row_condition, - condition_parser="pandas", - ) - result = batch_for_datasource.validate(expectation) - assert result.success - - -@parameterize_batch_for_data_sources( - data_source_configs=[ - MSSQLDatasourceTestConfig(column_types={"date": sqltypes.DATE}), - MySQLDatasourceTestConfig(column_types={"date": sqltypes.DATE}), - PostgreSQLDatasourceTestConfig(column_types={"date": sqltypes.DATE}), - SnowflakeDatasourceTestConfig(column_types={"date": sqltypes.DATE}), - SqliteDatasourceTestConfig(column_types={"date": sqltypes.DATE}), - ], - data=data, -) -@pytest.mark.parametrize( - "row_condition", - [ - pytest.param( - 'col("name")=="albert"', - id="text", - ), - pytest.param( - 'col("quantity")<3', - id="number", - ), - ], -) -def test_expect_column_min_to_be_between__sql_row_condition( - batch_for_datasource, row_condition -) -> None: - expectation = gxe.ExpectColumnMinToBeBetween( - column="date", - min_value=datetime(year=2021, month=1, day=1, tzinfo=timezone.utc).date(), - max_value=datetime(year=2022, month=1, day=1, tzinfo=timezone.utc).date(), - row_condition=row_condition, - condition_parser="great_expectations", - ) - result = batch_for_datasource.validate(expectation) - assert result.success diff --git a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json index bf412954986b..361e9c711425 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json +++ b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_be_between.json @@ -206,7 +206,7 @@ "condition_parser": "bad_parser" }, "out" : { - "traceback_substring": "unexpected value" + "traceback_substring": "not a valid enumeration member" } }] }] diff --git a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json index 7e7cc9dc4a92..727076ea1287 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json +++ b/tests/test_definitions/column_map_expectations/expect_column_value_lengths_to_equal.json @@ -129,7 +129,7 @@ "condition_parser": "bad_parser" }, "out" : { - "traceback_substring": "unexpected value" + "traceback_substring": "not a valid enumeration member" } }] }] diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json index 733896cb313f..f9c6375da979 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json +++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between.json @@ -776,7 +776,7 @@ "condition_parser": "bad_parser" }, "out": { - "traceback_substring": "unexpected value" + "traceback_substring": "not a valid enumeration member" } } ] diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json index 7e87825e691a..2a69593642ff 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json +++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_between_tz_naive.json @@ -572,7 +572,7 @@ "condition_parser": "bad_parser" }, "out": { - "traceback_substring": "unexpected value" + "traceback_substring": "not a valid enumeration member" } } ] diff --git a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json index e5afe95e82ce..c0b543334a25 100644 --- a/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json +++ b/tests/test_definitions/column_map_expectations/expect_column_values_to_be_in_set.json @@ -177,7 +177,7 @@ "condition_parser": "bad_parser" }, "out": { - "traceback_substring": "unexpected value" + "traceback_substring": "not a valid enumeration member" } } ]}, From 0639ba568885ed98c307ddca2dc48b60233d3e96 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 15:54:22 -0700 Subject: [PATCH 20/40] Indentation --- .../expectation_conditions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 46a5432e187b..4e4257172d07 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -29,27 +29,27 @@ Great Expectations allows you to specify conditions for validating rows using th -In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. - -The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. - -1. Determine the `condition_parser` for your `row_condition`. + In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. + + The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. + + 1. Determine the `condition_parser` for your `row_condition`. - The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. - The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with Spark, set this argument to `"great_expectations"`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with Spark, set this argument to `"great_expectations"`. - The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with SQL, set this argument to `"great_expectations"`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with SQL, set this argument to `"great_expectations"`. From bc05790ccfac4d840855ef795f548479917841ab Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:02:14 -0700 Subject: [PATCH 21/40] Indentation --- .../core/customize_expectations/expectation_conditions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 4e4257172d07..ad4eb94f6d99 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -27,14 +27,14 @@ Great Expectations allows you to specify conditions for validating rows using th ### Procedure - - In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. 1. Determine the `condition_parser` for your `row_condition`. + + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. From 02239621fc28509034137c1db07bfead75b0fd4a Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:17:04 -0700 Subject: [PATCH 22/40] Move tabs up below procedure --- .../expectation_conditions.md | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index ad4eb94f6d99..c1dd072dedaa 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -27,28 +27,40 @@ Great Expectations allows you to specify conditions for validating rows using th ### Procedure - In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. - - The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. - - 1. Determine the `condition_parser` for your `row_condition`. - + In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. + + The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. + + 1. Determine the `condition_parser` for your `row_condition`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. + In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. + + The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. + + 1. Determine the `condition_parser` for your `row_condition`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with Spark, set this argument to `"great_expectations"`. + In this procedure, it is assumed that your Data Context is stored in the variable `context`, and your Expectation Suite is stored in the variable `suite`. The `suite` can either be a newly created and empty Expectation Suite or an existing Expectation Suite retrieved from the Data Context. + + The examples in this procedure use passenger data from the Titanic, which includes details about the class of ticket held by the passenger and whether or not they survived the journey. + + 1. Determine the `condition_parser` for your `row_condition`. + The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with SQL, set this argument to `"great_expectations"`. From 5e33dfd6ac02090aaf76ac6eb40a4f308cc3e7ac Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:22:06 -0700 Subject: [PATCH 23/40] Try to make all tabs one --- .../core/customize_expectations/expectation_conditions.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index c1dd072dedaa..3d31c57215f2 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -65,8 +65,6 @@ Great Expectations allows you to specify conditions for validating rows using th - - Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. 2. Determine the `row_condition` expression. @@ -78,8 +76,6 @@ Great Expectations allows you to specify conditions for validating rows using th 3. Create the Expectation. An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. - - From 3b859652594eb9b1b70dedb29c26f2e97c4db4af Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:27:37 -0700 Subject: [PATCH 24/40] Revert "Try to make all tabs one" This reverts commit 5e33dfd6ac02090aaf76ac6eb40a4f308cc3e7ac. --- .../core/customize_expectations/expectation_conditions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 3d31c57215f2..c1dd072dedaa 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -65,6 +65,8 @@ Great Expectations allows you to specify conditions for validating rows using th + + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. 2. Determine the `row_condition` expression. @@ -76,6 +78,8 @@ Great Expectations allows you to specify conditions for validating rows using th 3. Create the Expectation. An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + + From 26f7956db0aaf67ca0022711a85e65b187afbacd Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:29:26 -0700 Subject: [PATCH 25/40] Try again --- .../expectation_conditions.md | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index c1dd072dedaa..5cd1aa1e6e63 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -39,6 +39,18 @@ Great Expectations allows you to specify conditions for validating rows using th The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. + + 2. Determine the `row_condition` expression. + + The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. + + The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. + + 3. Create the Expectation. + + An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + @@ -51,6 +63,18 @@ Great Expectations allows you to specify conditions for validating rows using th The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with Spark, set this argument to `"great_expectations"`. + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. + + 2. Determine the `row_condition` expression. + + The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. + + The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. + + 3. Create the Expectation. + + An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + @@ -63,21 +87,21 @@ Great Expectations allows you to specify conditions for validating rows using th The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with SQL, set this argument to `"great_expectations"`. - + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. - + 2. Determine the `row_condition` expression. - Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. + The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. -2. Determine the `row_condition` expression. + The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. - The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. + 3. Create the Expectation. - The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. + An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. -3. Create the Expectation. + - An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + From 7a4511b053e49af2e76fb1089e7cf0c13d6813f9 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:35:21 -0700 Subject: [PATCH 26/40] Revert "Try again" This reverts commit 26f7956db0aaf67ca0022711a85e65b187afbacd. --- .../expectation_conditions.md | 40 ++++--------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 5cd1aa1e6e63..c1dd072dedaa 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -39,18 +39,6 @@ Great Expectations allows you to specify conditions for validating rows using th The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with pandas, set this argument to `"pandas"`. - Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. - - 2. Determine the `row_condition` expression. - - The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. - - The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. - - 3. Create the Expectation. - - An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. - @@ -63,18 +51,6 @@ Great Expectations allows you to specify conditions for validating rows using th The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with Spark, set this argument to `"great_expectations"`. - Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. - - 2. Determine the `row_condition` expression. - - The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. - - The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. - - 3. Create the Expectation. - - An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. - @@ -87,21 +63,21 @@ Great Expectations allows you to specify conditions for validating rows using th The `condition_parser` defines the syntax of `row_condition` strings. When implementing Expectation conditions with SQL, set this argument to `"great_expectations"`. - Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. + - 2. Determine the `row_condition` expression. + - The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. - The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. +2. Determine the `row_condition` expression. - 3. Create the Expectation. + The `row_condition` argument should be a boolean expression string that is evaluated for each row in the Batch that the Expectation validates. If the `row_condition` evaluates to `True`, the row will be included in the Expectation's validations. If it evaluates to `False`, the Expectation will be skipped for that row. - An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. + The syntax of the `row_condition` argument is based on the `condition_parser` specified earlier. - +3. Create the Expectation. - + An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. From cb8a40b2c222a6cd939e003bf11542622705da7f Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 16:36:18 -0700 Subject: [PATCH 27/40] Hide unwanted tabs --- .../docs/core/customize_expectations/expectation_conditions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index c1dd072dedaa..a2b570f47331 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -79,7 +79,7 @@ Great Expectations allows you to specify conditions for validating rows using th An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. - + From b6814987733f8a0ccbc51b808fce8ef7973a1393 Mon Sep 17 00:00:00 2001 From: Farmer Date: Thu, 14 Nov 2024 19:58:42 -0700 Subject: [PATCH 28/40] Try to outdent Tabs> --- .../expectation_conditions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index a2b570f47331..29702c0fcf15 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -27,7 +27,7 @@ Great Expectations allows you to specify conditions for validating rows using th ### Procedure - + @@ -65,7 +65,7 @@ Great Expectations allows you to specify conditions for validating rows using th - + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. @@ -79,7 +79,7 @@ Great Expectations allows you to specify conditions for validating rows using th An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. - + @@ -144,9 +144,9 @@ Great Expectations allows you to specify conditions for validating rows using th - + - + @@ -206,7 +206,7 @@ Great Expectations allows you to specify conditions for validating rows using th - + 4. Optional. Create additional Expectation conditions @@ -219,7 +219,7 @@ Great Expectations allows you to specify conditions for validating rows using th And this code adds a condition to the Expectation that specifies the value of the `"Survived"` column is `1` if the individual was a first class passenger: - + @@ -239,7 +239,7 @@ Great Expectations allows you to specify conditions for validating rows using th ``` - + ## Data Docs and Expectation conditions From 6c0023300d8a16af1d935ef65cbf7faf1915ef09 Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 07:36:59 -0700 Subject: [PATCH 29/40] Revert "Try to outdent Tabs>" This reverts commit b6814987733f8a0ccbc51b808fce8ef7973a1393. --- .../expectation_conditions.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md index 29702c0fcf15..a2b570f47331 100644 --- a/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md +++ b/docs/docusaurus/docs/core/customize_expectations/expectation_conditions.md @@ -27,7 +27,7 @@ Great Expectations allows you to specify conditions for validating rows using th ### Procedure - + @@ -65,7 +65,7 @@ Great Expectations allows you to specify conditions for validating rows using th - + Note that the Expectation with conditions will fail if the Batch being validated is from a different type of Data Source than indicated by the `condition_parser`. @@ -79,7 +79,7 @@ Great Expectations allows you to specify conditions for validating rows using th An Expectation with conditions is created like a regular Expectation, with the addition of the `row_condition` and `condition_parser` parameters alongside the Expectation's other arguments. - + @@ -144,9 +144,9 @@ Great Expectations allows you to specify conditions for validating rows using th - + - + @@ -206,7 +206,7 @@ Great Expectations allows you to specify conditions for validating rows using th - + 4. Optional. Create additional Expectation conditions @@ -219,7 +219,7 @@ Great Expectations allows you to specify conditions for validating rows using th And this code adds a condition to the Expectation that specifies the value of the `"Survived"` column is `1` if the individual was a first class passenger: - + @@ -239,7 +239,7 @@ Great Expectations allows you to specify conditions for validating rows using th ``` - + ## Data Docs and Expectation conditions From 90e8790cfd1be4a0d09bceb716b7ca48c8bfbbe8 Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 08:56:14 -0700 Subject: [PATCH 30/40] Exclude new link due to limitations of link checker --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a989b3bb422e..bcb394daf2e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' --exclude='https://docs.greatexpectations.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From c812f13d36bbd1d1e061811507e183561d7ab090 Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 09:30:36 -0700 Subject: [PATCH 31/40] Escape periods --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcb394daf2e6..a555a296b48b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --exclude='https://docs.greatexpectations.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' --exclude='^https://docs\\.greatexpectations\\.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From b189d523ed8e08c1384c927a0e757200a86f6ad2 Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 09:48:09 -0700 Subject: [PATCH 32/40] Try adding index.html --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a555a296b48b..9f8a71f57619 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --exclude='^https://docs\\.greatexpectations\\.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' --exclude='^https://docs\\.greatexpectations\\.io/docs/core/customize_expectations/expectation_conditions/index\\.html' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From 6e0ca973c50bc8800a05678449ccd83247a73651 Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 10:59:18 -0700 Subject: [PATCH 33/40] Try wildcard --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f8a71f57619..e9407ef36f32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --exclude='^https://docs\\.greatexpectations\\.io/docs/core/customize_expectations/expectation_conditions/index\\.html' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' --exclude='*/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From ee3ae04466e58ab1e8491a5135cfcc0937af365d Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 11:11:10 -0700 Subject: [PATCH 34/40] Fix syntax --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9407ef36f32..9167f84dfa6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --exclude='*/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' --exclude='.*/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From 425db6f58bdc450c31273f01b81c925450a4243a Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 11:17:41 -0700 Subject: [PATCH 35/40] Don't use exclude keyword twice --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9167f84dfa6c..b2c4f002f0d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --exclude='.*/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' 'https://docs\\.greatexpectations\\.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From 6495bac8172f8256e5b937a9368ea7f1378c1ffd Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 11:44:25 -0700 Subject: [PATCH 36/40] Don't escape periods --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2c4f002f0d4..e45b240ebaa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' 'https://docs\\.greatexpectations\\.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' 'https://docs.greatexpectations.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From f5ad0185273fddcda390124805c293032bc2397e Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 12:41:37 -0700 Subject: [PATCH 37/40] Try wildcard --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e45b240ebaa0..3f978df22de1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' 'https://docs.greatexpectations.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' '.*/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From 72f86a94c64be96fd144017a8f3837250a01283e Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 13:11:14 -0700 Subject: [PATCH 38/40] Try again --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f978df22de1..e45b240ebaa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' '.*/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' 'https://docs.greatexpectations.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From c297e449fd210cbb401d98f2c4c7be74e2e64267 Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 13:30:23 -0700 Subject: [PATCH 39/40] Remove include --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e45b240ebaa0..5804d91d3f7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' 'https://docs.greatexpectations.io/docs/core/customize_expectations/expectation_conditions' --include='^https://(.+\\.)?greatexpectations\\.io/' 'docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' --include='docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest From e2a2c2703407c62f64a550693330c881a40f45fd Mon Sep 17 00:00:00 2001 From: Farmer Date: Fri, 15 Nov 2024 13:50:23 -0700 Subject: [PATCH 40/40] Trying everything I can --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5804d91d3f7a..0a1574ed371b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: # We decided to exclude all external HTTP requests but the ones that under the domain greatexpectations.io # The reason is to avoid having network errors such as pages that throw 429 after too many requests (like Github) # and to prevent other possible errors related to user agent or lychee capturing hrefs from metadata that don't resolve to a specific page (preconnects in JS) - args: "--exclude='http.*' --include='docs/docusaurus/build/**/*.html'" + args: "--exclude='http.*' 'docs/docusaurus/build/**/*.html'" docs-tests: runs-on: ubuntu-latest