From fb20020d51ea8e7263127e98193538258e909029 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 7 Jul 2023 15:33:37 -0400 Subject: [PATCH 01/36] basic test --- .../src/Common_Table_Operations/Main.enso | 28 ++++++++++--------- .../Common_Table_Operations/Text_Spec.enso | 24 ++++++++++++++++ 2 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index 1225b78b5a69..f12f19a40c24 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -19,6 +19,7 @@ import project.Common_Table_Operations.Missing_Values_Spec import project.Common_Table_Operations.Order_By_Spec import project.Common_Table_Operations.Select_Columns_Spec import project.Common_Table_Operations.Take_Drop_Spec +import project.Common_Table_Operations.Text_Spec import project.Common_Table_Operations.Transpose_Spec from project.Common_Table_Operations.Util import run_default_backend @@ -96,25 +97,26 @@ type Test_Selection Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False take_drop=True allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True distinct_returns_first_row_from_group_if_ordered=True date_time=True fixed_length_text_columns=False supports_decimal_type=False spec setup = - Core_Spec.spec setup - Select_Columns_Spec.spec setup + Add_Row_Number_Spec.spec setup + Aggregate_Spec.spec setup Column_Operations_Spec.spec setup - Date_Time_Spec.spec setup Conversion_Spec.spec setup - Aggregate_Spec.spec setup + Core_Spec.spec setup + Cross_Join_Spec.spec setup + Cross_Tab_Spec.spec setup + Date_Time_Spec.spec setup + Distinct_Spec.spec setup + Expression_Spec.spec detailed=False setup Filter_Spec.spec setup + Integration_Tests.spec setup + Join_Spec.spec setup Missing_Values_Spec.spec setup Order_By_Spec.spec setup + Select_Columns_Spec.spec setup Take_Drop_Spec.spec setup - Expression_Spec.spec detailed=False setup - Join_Spec.spec setup - Cross_Join_Spec.spec setup - Zip_Spec.spec setup - Union_Spec.spec setup - Distinct_Spec.spec setup - Cross_Tab_Spec.spec setup + Text_Spec.spec setup Transpose_Spec.spec setup - Add_Row_Number_Spec.spec setup - Integration_Tests.spec setup + Union_Spec.spec setup + Zip_Spec.spec setup main = run_default_backend spec diff --git a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso new file mode 100644 index 000000000000..df37dd8347e4 --- /dev/null +++ b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso @@ -0,0 +1,24 @@ +from Standard.Base import all + +from Standard.Test import Test, Problems +import Standard.Test.Extensions + +from project.Common_Table_Operations.Util import run_default_backend + +main = run_default_backend spec + +spec setup = + prefix = setup.prefix + table_builder = setup.table_builder + + Test.group prefix+"replace" <| + Test.specify "Can do replace" <| + col = table_builder [["x", ["asdfSd", "zsdcSD", "sdsd"]]].at "x" + + col . replace "sd" "qqq" . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqqqq"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive . to_vector . should_equal ["aqqqfqqq", "zqqqcqqq", "qqqqqq"] + + col . replace "sd" "qqq" only_first=True . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqsd"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] From 78343e95d35785b435a857aa6c25aec338b1028a Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Jul 2023 10:23:05 -0400 Subject: [PATCH 02/36] wip --- .../Standard/Database/0.0.0-dev/src/Data/Column.enso | 7 ++++--- .../src/Common_Table_Operations/Text_Spec.enso | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 52fb023af8b7..fac1abcb4a93 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1135,9 +1135,10 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Sensitive only_first=False = - _ = [term, new_text, case_sensitivity, only_first] - msg = "`Column.replace` is not yet implemented." - Error.throw (Unsupported_Database_Operation.Error msg) + _ = [case_sensitivity, only_first] + Value_Type.expect_text self <| + new_name = self.naming_helpers.function_name "replace" [self, term, new_text] + self.make_unary_op "REPLACE" new_name ## Gets the year as a number from the date stored in the column. diff --git a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso index df37dd8347e4..094587b859bd 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso @@ -16,9 +16,10 @@ spec setup = col = table_builder [["x", ["asdfSd", "zsdcSD", "sdsd"]]].at "x" col . replace "sd" "qqq" . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqqqq"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive . to_vector . should_equal ["aqqqfqqq", "zqqqcqqq", "qqqqqq"] + ## + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive . to_vector . should_equal ["aqqqfqqq", "zqqqcqqq", "qqqqqq"] - col . replace "sd" "qqq" only_first=True . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqsd"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] + col . replace "sd" "qqq" only_first=True . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqsd"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] + col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] From 779065ebdd9b4459e97d9f4ff77e6d3299fb5937 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 10 Jul 2023 11:08:32 -0400 Subject: [PATCH 03/36] simple case --- .../Database/0.0.0-dev/src/Data/Column.enso | 17 ++++++++++++++++- .../Database/0.0.0-dev/src/Data/Table.enso | 3 +++ .../0.0.0-dev/src/Internal/Base_Generator.enso | 16 ++++++++++++++++ .../src/Internal/Postgres/Postgres_Dialect.enso | 5 ++++- .../src/Common_Table_Operations/Text_Spec.enso | 6 +++--- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index fac1abcb4a93..0b93654e0348 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -156,6 +156,21 @@ type Column new_type_ref = type_mapping.infer_return_type infer_from_database_callback op_kind [self]+operands new_expr Column.Value new_name self.connection new_type_ref new_expr self.context + ## PRIVATE + + Creates a ternary operation with given kind and operand. + + Arguments: + - op_kind: The kind of ternary operator. + - operand1: The second operand to the binary operator. + - operand2: The third operand to the binary operator. + - new_name: The name of the resulting column. + make_ternary_op : Text -> Column | Any -> Column | Any -> (Text | Nothing) -> Column + make_ternary_op self op_kind operand1 operand2 new_name=Nothing = + effective_new_name = new_name.if_nothing <| + self.naming_helpers.function_name op_kind [self, operand1, operand2] + self.make_op op_kind [operand1, operand2] effective_new_name + ## PRIVATE Creates a binary operation with given kind and operand. @@ -1138,7 +1153,7 @@ type Column _ = [case_sensitivity, only_first] Value_Type.expect_text self <| new_name = self.naming_helpers.function_name "replace" [self, term, new_text] - self.make_unary_op "REPLACE" new_name + self.make_ternary_op "REPLACE" term new_text new_name ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index deab21f8ddce..12293ae47a59 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1790,6 +1790,9 @@ type Table Error.throw (Illegal_Argument.Error "Cannot create a table with no columns.") False -> sql = preprocessed.to_sql + IO.println 'SQL' + IO.println <| sql.unsafe_to_raw_sql + IO.println <| sql.unsafe_to_raw_sql column_type_suggestions = preprocessed.internal_columns.map .sql_type_reference self.connection.read_statement sql column_type_suggestions diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index b0008f53d868..b3df2adb8aa9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -109,6 +109,22 @@ lift_binary_op name function = True -> function (arguments.at 0) (arguments.at 1) [name, generator] +## PRIVATE + + A helper function to create a binary operation from a function. + + Arguments: + - name: Name of the operation, used for error reporting. + - function: A function taking exactly two arguments: the generated SQL code + for the argument of the operation, and returning the generated SQL code for + the whole operation. +lift_ternary_op : Text -> (Builder -> Builder -> Builder -> Builder) -> [Text, (Vector Builder -> Builder)] +lift_ternary_op name function = + generator = arguments -> case arguments.length == 3 of + False -> Error.throw <| Illegal_State.Error ("Invalid amount of arguments for operation " + name + ".") + True -> function (arguments.at 0) (arguments.at 1) (arguments.at 2) + [name, generator] + ## PRIVATE A helper function to create a unary operator which is added to the right of diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 8a0b07fcd60c..5b205d88a5ac 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -230,7 +230,7 @@ type Postgres_Dialect ## PRIVATE make_internal_generator_dialect = cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]] - text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive]+concat_ops+cases+trim_ops + text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive, replace]+concat_ops+cases+trim_ops counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] arith_extensions = [is_nan, is_inf, floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]] bool = [bool_or] @@ -480,6 +480,9 @@ decimal_div = Base_Generator.lift_binary_op "DECIMAL_DIV" x-> y-> decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS decimal) / CAST(" ++ y ++ " AS decimal)) * " ++ y +replace = Base_Generator.lift_ternary_op "REPLACE" t-> p-> r-> + Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" + ## PRIVATE make_extract_as_int enso_name sql_name = Base_Generator.lift_unary_op enso_name arg-> diff --git a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso index 094587b859bd..b98d523b9f9f 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso @@ -11,11 +11,11 @@ spec setup = prefix = setup.prefix table_builder = setup.table_builder - Test.group prefix+"replace" <| + Test.group prefix+"replaceasdfasdf" <| Test.specify "Can do replace" <| - col = table_builder [["x", ["asdfSd", "zsdcSD", "sdsd"]]].at "x" + col = table_builder [["x", ["asdfSd", "zsdcSD", "sdsd"]]] . at "x" - col . replace "sd" "qqq" . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqqqq"] + col . replace "sd" "qqq" . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] ## col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive . to_vector . should_equal ["aqqqfqqq", "zqqqcqqq", "qqqqqq"] From 31883cbda0d2cc37a79122e81d352db8c29d8de1 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Jul 2023 12:25:36 -0400 Subject: [PATCH 04/36] remove text_spec, pg tests --- .../Database/0.0.0-dev/src/Data/Column.enso | 2 +- .../src/Common_Table_Operations/Main.enso | 2 - .../Common_Table_Operations/Text_Spec.enso | 25 ------------ .../src/Database/Postgres_Spec.enso | 38 +++++++++++++++++++ 4 files changed, 39 insertions(+), 28 deletions(-) delete mode 100644 test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 0b93654e0348..2a1920469dfe 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1149,7 +1149,7 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column - replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Sensitive only_first=False = + replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = _ = [case_sensitivity, only_first] Value_Type.expect_text self <| new_name = self.naming_helpers.function_name "replace" [self, term, new_text] diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index f12f19a40c24..904f2f4a73bb 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -19,7 +19,6 @@ import project.Common_Table_Operations.Missing_Values_Spec import project.Common_Table_Operations.Order_By_Spec import project.Common_Table_Operations.Select_Columns_Spec import project.Common_Table_Operations.Take_Drop_Spec -import project.Common_Table_Operations.Text_Spec import project.Common_Table_Operations.Transpose_Spec from project.Common_Table_Operations.Util import run_default_backend @@ -114,7 +113,6 @@ spec setup = Order_By_Spec.spec setup Select_Columns_Spec.spec setup Take_Drop_Spec.spec setup - Text_Spec.spec setup Transpose_Spec.spec setup Union_Spec.spec setup Zip_Spec.spec setup diff --git a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso deleted file mode 100644 index b98d523b9f9f..000000000000 --- a/test/Table_Tests/src/Common_Table_Operations/Text_Spec.enso +++ /dev/null @@ -1,25 +0,0 @@ -from Standard.Base import all - -from Standard.Test import Test, Problems -import Standard.Test.Extensions - -from project.Common_Table_Operations.Util import run_default_backend - -main = run_default_backend spec - -spec setup = - prefix = setup.prefix - table_builder = setup.table_builder - - Test.group prefix+"replaceasdfasdf" <| - Test.specify "Can do replace" <| - col = table_builder [["x", ["asdfSd", "zsdcSD", "sdsd"]]] . at "x" - - col . replace "sd" "qqq" . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] - ## - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqqqq"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive . to_vector . should_equal ["aqqqfqqq", "zqqqcqqq", "qqqqqq"] - - col . replace "sd" "qqq" only_first=True . to_vector . should_equal ["aqqqfqqq", "zqqqcSD", "qqqsd"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Sensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] - col . replace "sd" "qqq" case_sensitivity=Case_Sensitive.Insensitive only_first=True . to_vector . should_equal ["aqqqfSd", "zqqqcSD", "qqqsd"] diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 7fa74af3bfbb..b8ee70e9cf72 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,6 +261,44 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal + Test.group "replace" <| + col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" + col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" + col0.replace 'hello', 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + + col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['hey A[bCd] hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + col0.replace 'hello', 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] + + col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye '] + + col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['hey hey hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" + col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye Hello', 'bye Hello '] + + col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" + col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" + col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" + col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] + + Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" + col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + run_tests connection db_name = prefix = "[PostgreSQL] " name_counter = Ref.new 0 From efd0898b2e8628e346713ebe60a0f68bc5bb4434 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 11 Jul 2023 15:15:03 -0400 Subject: [PATCH 05/36] two tests --- .../Database/0.0.0-dev/src/Data/Column.enso | 15 ++++++- .../Internal/Postgres/Postgres_Dialect.enso | 25 +++++++++-- .../src/Database/Postgres_Spec.enso | 42 ++++++++++++++----- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 2a1920469dfe..3d7fedccadc5 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1150,10 +1150,21 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = - _ = [case_sensitivity, only_first] Value_Type.expect_text self <| new_name = self.naming_helpers.function_name "replace" [self, term, new_text] - self.make_ternary_op "REPLACE" term new_text new_name + use_regex_part = case term of + _ : Text -> "TEXT" + _ : Column -> "COLUMN" + _ : Regex -> "REGEX" + case_sensitivity_part = case case_sensitivity of + Case_Sensitivity.Default -> "CS" + Case_Sensitivity.Sensitive -> "CS" + Case_Sensitivity.Insensitive _ -> "CI" + only_first_part = if only_first then "ONLY_FIRST" else "ALL" + op_name = "REPLACE_" + use_regex_part + "_" + case_sensitivity_part + "_" + only_first_part + IO.println 'AAA' + IO.println op_name + self.make_ternary_op op_name term new_text new_name ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 5b205d88a5ac..b5fddaa59a1e 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -6,6 +6,7 @@ import Standard.Base.Errors.Unimplemented.Unimplemented import Standard.Table.Data.Aggregate_Column.Aggregate_Column import Standard.Table.Internal.Naming_Helpers.Naming_Helpers import Standard.Table.Internal.Problem_Builder.Problem_Builder +import Standard.Table.Internal.Vector_Builder.Vector_Builder from Standard.Table import Value_Type from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all from Standard.Table.Errors import Inexact_Type_Coercion @@ -230,7 +231,8 @@ type Postgres_Dialect ## PRIVATE make_internal_generator_dialect = cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]] - text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive, replace]+concat_ops+cases+trim_ops + text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive]+concat_ops+cases+trim_ops + text_replace = [replace_text_cs_all, replace_text_cs_only_first] counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] arith_extensions = [is_nan, is_inf, floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]] bool = [bool_or] @@ -240,7 +242,7 @@ make_internal_generator_dialect = stats = [agg_median, agg_mode, agg_percentile, stddev_pop, stddev_samp] date_ops = [make_extract_as_int "year" "YEAR", make_extract_as_int "month" "MONTH", make_extract_as_int "day" "DAY"] special_overrides = [is_null, is_empty] - my_mappings = text + counts + stats + first_last_aggregators + arith_extensions + bool + date_ops + special_overrides + my_mappings = text + text_replace + counts + stats + first_last_aggregators + arith_extensions + bool + date_ops + special_overrides Base_Generator.base_dialect . extend_with my_mappings ## PRIVATE @@ -480,9 +482,26 @@ decimal_div = Base_Generator.lift_binary_op "DECIMAL_DIV" x-> y-> decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS decimal) / CAST(" ++ y ++ " AS decimal)) * " ++ y -replace = Base_Generator.lift_ternary_op "REPLACE" t-> p-> r-> +replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" +replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" t-> p-> r-> + escaped_pattern = escape_regex_param p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 1)" + +## PRIVATE + Escape a literal regex param +escape_regex_param : Builder -> Builder +escape_regex_param regex_param = transform_literal_param Regex.escape regex_param + +## PRIVATE + Transform the interpolation value within the Builder +transform_literal_param : (Any -> Any) -> Builder -> Builder +transform_literal_param transformer param = + x = case param of Builder.Value (Vector_Builder.Leaf x) -> case x.at 0 of SQL_Fragment.Interpolation y -> y + transformed_x = transformer x + Builder.Value (Vector_Builder.Leaf [SQL_Fragment.Interpolation transformed_x]) + ## PRIVATE make_extract_as_int enso_name sql_name = Base_Generator.lift_unary_op enso_name arg-> diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index b8ee70e9cf72..eac3f8a4b4bf 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,43 +261,63 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace" <| + Test.group "replace: literal pattern and replacementasdfasdf" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" - col0.replace 'hello', 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello' 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['hey A[bCd] hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col0.replace 'hello', 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] + col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye '] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye '] col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['hey hey hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col0.replace 'hello', 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye Hello', 'bye Hello '] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye Hello', 'bye Hello '] col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" - col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" - col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" - col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] + col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" - col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + + Test.group "replace: pattern and replacement columnsasdfasdf" <| + col = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + patterns = table_builder [["x", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']]] . at "x" + replacements = table_builder [["x", ['bye', 'bye', 'bye', 'hey', 'hey']]] . at "x" + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" + col.replace patterns replacements . to_vector . should_equal ['bye Hello', 'hbyeo hbyeo', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + col.replace patterns replacements only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye ', 'hey hey hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye Hello', 'bye Hello', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] run_tests connection db_name = prefix = "[PostgreSQL] " From 595fea710b4e92220dee33346e55f8c9f5a88d29 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 10:52:36 -0400 Subject: [PATCH 06/36] 8 tests --- .../Base/0.0.0-dev/src/Data/Text/Regex.enso | 6 ++++ .../Database/0.0.0-dev/src/Data/Column.enso | 6 +++- .../Internal/Postgres/Postgres_Dialect.enso | 29 ++++++++++++++++++- .../src/Database/Postgres_Spec.enso | 10 +++---- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso index 96573321aa75..88d79b0a151a 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Regex.enso @@ -376,6 +376,12 @@ type Regex if should_recompile.not then self else Regex.compile self.internal_regex_object.pattern case_insensitive + ## PRIVATE + + Get the original pattern string as a `Text`. + pattern_string : Text + pattern_string self = self.internal_regex_object.pattern + ## PRIVATE Convert the polyglot map to a Map. polyglot_map_to_map : Any -> Map Any Any diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 3d7fedccadc5..0702a6743e7b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1151,7 +1151,11 @@ type Column replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = Value_Type.expect_text self <| - new_name = self.naming_helpers.function_name "replace" [self, term, new_text] + term_name = case term of + _ : Text -> term + _ : Column -> term + _ : Regex -> term.pattern_string + new_name = self.naming_helpers.function_name "replace" [self, term_name, new_text] use_regex_part = case term of _ : Text -> "TEXT" _ : Column -> "COLUMN" diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index b5fddaa59a1e..0d9c2d2e8447 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -232,7 +232,7 @@ type Postgres_Dialect make_internal_generator_dialect = cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]] text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive]+concat_ops+cases+trim_ops - text_replace = [replace_text_cs_all, replace_text_cs_only_first] + text_replace = [replace_text_cs_all, replace_text_cs_only_first, replace_text_ci_all, replace_text_ci_only_first, replace_regex_cs_all, replace_regex_cs_only_first, replace_regex_ci_all, replace_regex_ci_only_first, replace_column_cs_all] counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] arith_extensions = [is_nan, is_inf, floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]] bool = [bool_or] @@ -489,6 +489,33 @@ replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONL escaped_pattern = escape_regex_param p Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 1)" +replace_text_ci_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ALL" t-> p-> r-> + escaped_pattern = escape_regex_param p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 0, 'i')" + +replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" t-> p-> r-> + escaped_pattern = escape_regex_param p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 1, 'i')" + +replace_regex_cs_all = Base_Generator.lift_ternary_op "REPLACE_REGEX_CS_ALL" t-> p-> r-> + pattern_string = transform_literal_param (_.pattern_string) p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 0)" + +replace_regex_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_REGEX_CS_ONLY_FIRST" t-> p-> r-> + pattern_string = transform_literal_param (_.pattern_string) p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 1)" + +replace_regex_ci_all = Base_Generator.lift_ternary_op "REPLACE_REGEX_CI_ALL" t-> p-> r-> + pattern_string = transform_literal_param (_.pattern_string) p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 0, 'i')" + +replace_regex_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_REGEX_CI_ONLY_FIRST" t-> p-> r-> + pattern_string = transform_literal_param (_.pattern_string) p + Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 1, 'i')" + +replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t-> p-> r-> + Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" + ## PRIVATE Escape a literal regex param escape_regex_param : Builder -> Builder diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index eac3f8a4b4bf..e8a130a81315 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -278,12 +278,12 @@ postgres_specific_spec connection db_name setup = col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye '] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye'] col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['hey hey hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye Hello', 'bye Hello '] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO'] col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] @@ -292,13 +292,13 @@ postgres_specific_spec connection db_name setup = col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] Test.group "replace: pattern and replacement columnsasdfasdf" <| col = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" From 5d9985f47366435c46bfb2613bf4aff85a77029a Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 11:02:41 -0400 Subject: [PATCH 07/36] all tests --- .../Table_Tests/src/Database/Postgres_Spec.enso | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index e8a130a81315..55f70fb8e534 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -9,7 +9,7 @@ from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all hiding Fir import Standard.Database.Data.SQL_Type.SQL_Type import Standard.Database.Internal.Postgres.Pgpass from Standard.Database import all -from Standard.Database.Errors import SQL_Error +from Standard.Database.Errors import SQL_Error, Unsupported_Database_Operation from Standard.Test import Test, Test_Suite import Standard.Test.Extensions @@ -301,23 +301,24 @@ postgres_specific_spec connection db_name setup = col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] Test.group "replace: pattern and replacement columnsasdfasdf" <| - col = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" - patterns = table_builder [["x", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']]] . at "x" - replacements = table_builder [["x", ['bye', 'bye', 'bye', 'hey', 'hey']]] . at "x" + table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] + col = table.at "x" + patterns = table.at "patterns" + replacements = table.at "replacements" Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" - col.replace patterns replacements . to_vector . should_equal ['bye Hello', 'hbyeo hbyeo', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + col.replace patterns replacements . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] col.replace patterns replacements case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col.replace patterns replacements only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + col.replace patterns replacements only_first=True . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye ', 'hey hey hey', 'abac ad Ab aCAd'] + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye Hello', 'bye Hello', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation run_tests connection db_name = prefix = "[PostgreSQL] " From 0d6c62e2f9394b8079c38589bfc663bd143853e0 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 11:35:50 -0400 Subject: [PATCH 08/36] sl 12/2 --- .../src/Internal/SQLite/SQLite_Dialect.enso | 31 +++++++- .../Internal/SQLite/SQLite_Type_Mapping.enso | 2 +- .../Table_Tests/src/Database/SQLite_Spec.enso | 70 ++++++++++++++++++- 3 files changed, 100 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index cc10d9e9dfea..ce6538c61a58 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -260,12 +260,13 @@ type SQLite_Dialect ## PRIVATE make_internal_generator_dialect = text = [starts_with, contains, ends_with, make_case_sensitive]+concat_ops+trim_ops + text_replace = [replace_text_cs_all, replace_text_cs_only_first, replace_text_ci_only_first] counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] stats = [agg_stddev_pop, agg_stddev_samp] arith_extensions = [is_inf, floating_point_div, mod_op] bool = [bool_or] - my_mappings = text + counts + stats + arith_extensions + bool + my_mappings = text + text_replace + counts + stats + arith_extensions + bool Base_Generator.base_dialect . extend_with my_mappings ## PRIVATE @@ -397,6 +398,34 @@ floating_point_div = Base_Generator.lift_binary_op "/" x-> y-> mod_op = Base_Generator.lift_binary_op "mod" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS REAL) / CAST(" ++ y ++ " AS REAL)) * " ++ y +replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> + Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" + +replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" <| replace_text_only_first True +replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" <| replace_text_only_first False + +## + , replace(a, p, r) as "cs=s/d r=f of=f" + , case when instr(a, p) = 0 then a + else substr(a, 1, instr(a, p)-1) || + r || + substr(a, instr(a, p)+length(p)) + end as "cs=s/d r=f of=t" + , case when instr(lower(a), p) = 0 then a + else substr(a, 1, instr(lower(a), p)-1) || + r || + substr(a, instr(lower(a), p)+length(p)) + end as "cs=i r=f of=t" + +replace_text_only_first case_sensitive t p r = + search_string = if case_sensitive then t else + Builder.code "LOWER(" ++ t ++ ")" + instr = Builder.code "INSTR(" ++ search_string ++ ", " ++ p ++ ")" + prefix = Builder.code "SUBSTR(" ++ t ++ ", 1," ++ instr ++ "-1)" + suffix = Builder.code "SUBSTR(" ++ t ++ "," ++ instr ++ "+LENGTH(" ++ p ++ "))" + concatenation = prefix ++ " || " ++ r ++ " || " ++ suffix + Builder.code "CASE WHEN " ++ instr ++ "= 0 THEN " ++ t ++ " ELSE " ++ concatenation ++ "END" + ## PRIVATE It will return `Nothing` if the type does not require custom logic. make_custom_cast column target_value_type type_mapping = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso index 7faff3c808e4..101f623757b4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso @@ -181,7 +181,7 @@ operations_map = always_boolean_ops = ["==", "!=", "equals_ignore_case", ">=", "<=", "<", ">", "BETWEEN", "AND", "OR", "NOT", "IS_NULL", "IS_EMPTY", "LIKE", "IS_IN", "IS_IN_COLUMN", "starts_with", "ends_with", "contains", "BOOL_OR", "IS_INF"] always_floating_ops = ["/", "mod", "AVG", "STDDEV_POP", "STDDEV_SAMP", "ROUND"] - always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM"] + always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE_TEXT_CS_ALL", "REPLACE_TEXT_CS_ONLY_FIRST", "REPLACE_TEXT_CI_ONLY_FIRST"] always_integer_ops = ["COUNT", "COUNT_IS_NULL", "COUNT_DISTINCT", "COUNT_DISTINCT_INCLUDE_NULL", "COUNT_EMPTY", "COUNT_NOT_EMPTY", "COUNT_ROWS", "ROW_NUMBER"] same_as_first = ["TRUNCATE", "CEIL", "FLOOR"] arithmetic_ops = ["ADD_NUMBER", "-", "*", "^", "%", "SUM"] diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 2d9c322aee45..786692ed00ca 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -6,7 +6,7 @@ import Standard.Table.Data.Type.Value_Type.Bits from Standard.Table import Table, Value_Type from Standard.Database import all -from Standard.Database.Errors import SQL_Error +from Standard.Database.Errors import SQL_Error, Unsupported_Database_Operation from Standard.Test import Test, Test_Suite import Standard.Test.Extensions @@ -190,6 +190,74 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal + Test.group "zxcvzxcv" <| + col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" + col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] + + Test.group "replace: literal pattern and replacementasdfasdf" <| + col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" + col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + + # * + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" + col0.replace 'hello' 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] + + col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['hey A[bCd] hey', 'abac ad Ab aCAd'] + + # * + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] + + col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye'] + + col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['hey hey hey', 'abac ad Ab aCAd'] + + # * + Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO'] + + col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" + col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" + col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] + + Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + + Test.group "replace: pattern and replacement columnsasdfasdf" <| + table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] + col = table.at "x" + patterns = table.at "patterns" + replacements = table.at "replacements" + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" + col.replace patterns replacements . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + col.replace patterns replacements only_first=True . should_fail_with Unsupported_Database_Operation + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation + sqlite_spec connection prefix = name_counter = Ref.new 0 table_builder columns = From 080591169f276d105254033b9710261b4dcc0279 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 11:37:42 -0400 Subject: [PATCH 09/36] some unsupp --- test/Table_Tests/src/Database/SQLite_Spec.enso | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 786692ed00ca..23b4c6177fd8 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -200,7 +200,6 @@ sqlite_specific_spec prefix connection setup = col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" - # * Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" col0.replace 'hello' 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] @@ -208,18 +207,14 @@ sqlite_specific_spec prefix connection setup = col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['hey A[bCd] hey', 'abac ad Ab aCAd'] - # * Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye'] + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation - col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['hey hey hey', 'abac ad Ab aCAd'] - - # * Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO'] @@ -227,16 +222,16 @@ sqlite_specific_spec prefix connection setup = Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation Test.group "replace: pattern and replacement columnsasdfasdf" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] From 5c34f2879d9c935673da306e95a55448208f49a4 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 11:50:40 -0400 Subject: [PATCH 10/36] sl all --- .../src/Internal/SQLite/SQLite_Dialect.enso | 14 ++++++++++---- .../src/Internal/SQLite/SQLite_Type_Mapping.enso | 2 +- test/Table_Tests/src/Database/SQLite_Spec.enso | 10 ++-------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index ce6538c61a58..c3870016ac0f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -260,7 +260,7 @@ type SQLite_Dialect ## PRIVATE make_internal_generator_dialect = text = [starts_with, contains, ends_with, make_case_sensitive]+concat_ops+trim_ops - text_replace = [replace_text_cs_all, replace_text_cs_only_first, replace_text_ci_only_first] + text_replace = [replace_text_cs_all, replace_text_cs_only_first, replace_text_ci_only_first, replace_column_cs_all, replace_column_cs_only_first, replace_column_ci_only_first] counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] stats = [agg_stddev_pop, agg_stddev_samp] arith_extensions = [is_inf, floating_point_div, mod_op] @@ -401,8 +401,14 @@ mod_op = Base_Generator.lift_binary_op "mod" x-> y-> replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" -replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" <| replace_text_only_first True -replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" <| replace_text_only_first False +replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" <| replace_only_first True +replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" <| replace_only_first False + +replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t-> p-> r-> + Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" + +replace_column_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ONLY_FIRST" <| replace_only_first True +replace_column_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CI_ONLY_FIRST" <| replace_only_first False ## , replace(a, p, r) as "cs=s/d r=f of=f" @@ -417,7 +423,7 @@ replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONL substr(a, instr(lower(a), p)+length(p)) end as "cs=i r=f of=t" -replace_text_only_first case_sensitive t p r = +replace_only_first case_sensitive t p r = search_string = if case_sensitive then t else Builder.code "LOWER(" ++ t ++ ")" instr = Builder.code "INSTR(" ++ search_string ++ ", " ++ p ++ ")" diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso index 101f623757b4..438960a6eb88 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso @@ -181,7 +181,7 @@ operations_map = always_boolean_ops = ["==", "!=", "equals_ignore_case", ">=", "<=", "<", ">", "BETWEEN", "AND", "OR", "NOT", "IS_NULL", "IS_EMPTY", "LIKE", "IS_IN", "IS_IN_COLUMN", "starts_with", "ends_with", "contains", "BOOL_OR", "IS_INF"] always_floating_ops = ["/", "mod", "AVG", "STDDEV_POP", "STDDEV_SAMP", "ROUND"] - always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE_TEXT_CS_ALL", "REPLACE_TEXT_CS_ONLY_FIRST", "REPLACE_TEXT_CI_ONLY_FIRST"] + always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE_TEXT_CS_ALL", "REPLACE_TEXT_CS_ONLY_FIRST", "REPLACE_TEXT_CI_ONLY_FIRST", "REPLACE_COLUMN_CS_ALL", "REPLACE_COLUMN_CS_ONLY_FIRST", "REPLACE_COLUMN_CI_ONLY_FIRST"] always_integer_ops = ["COUNT", "COUNT_IS_NULL", "COUNT_DISTINCT", "COUNT_DISTINCT_INCLUDE_NULL", "COUNT_EMPTY", "COUNT_NOT_EMPTY", "COUNT_ROWS", "ROW_NUMBER"] same_as_first = ["TRUNCATE", "CEIL", "FLOOR"] arithmetic_ops = ["ADD_NUMBER", "-", "*", "^", "%", "SUM"] diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 23b4c6177fd8..7ae42db2f241 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -190,12 +190,6 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "zxcvzxcv" <| - col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" - col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] - Test.group "replace: literal pattern and replacementasdfasdf" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -245,13 +239,13 @@ sqlite_specific_spec prefix connection setup = col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col.replace patterns replacements only_first=True . should_fail_with Unsupported_Database_Operation + col.replace patterns replacements only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] sqlite_spec connection prefix = name_counter = Ref.new 0 From 6ce316930b3451b816591845eb705bee06672a13 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 12:32:26 -0400 Subject: [PATCH 11/36] better errors --- .../Database/0.0.0-dev/src/Data/Column.enso | 15 ++++++++++++--- .../src/Internal/SQLite/SQLite_Dialect.enso | 13 ------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 0702a6743e7b..a04bd17aa3b1 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1166,9 +1166,8 @@ type Column Case_Sensitivity.Insensitive _ -> "CI" only_first_part = if only_first then "ONLY_FIRST" else "ALL" op_name = "REPLACE_" + use_regex_part + "_" + case_sensitivity_part + "_" + only_first_part - IO.println 'AAA' - IO.println op_name - self.make_ternary_op op_name term new_text new_name + if_replace_op_supported self.connection.dialect op_name term case_sensitivity only_first <| + self.make_ternary_op op_name term new_text new_name ## Gets the year as a number from the date stored in the column. @@ -1503,3 +1502,13 @@ adapt_unified_column column expected_type = SQL_Type_Reference.new column.connection column.context expression adapted = dialect.adapt_unified_column column.as_internal expected_type infer_return_type Column.Value name=column.name connection=column.connection sql_type_reference=adapted.sql_type_reference expression=adapted.expression context=column.context + +## PRIVATE + Build a friendly error explaining that the combination of REPLACE options is + not supported for this backend. +if_replace_op_supported : Any -> Text -> Text | Column | Regex -> Case_Sensitivity -> Boolean -> Any -> Any +if_replace_op_supported dialect op_name term case_sensitivity only_first ~action = + if dialect.is_supported op_name then action else + term_type = Meta.type_of term . to_text + msg = "The REPLACE operation on " + dialect.name + " is not supported for the following options: argument type " + term_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text + Error.throw (Unsupported_Database_Operation.Error msg) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index c3870016ac0f..c4d310311bed 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -410,19 +410,6 @@ replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t replace_column_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ONLY_FIRST" <| replace_only_first True replace_column_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CI_ONLY_FIRST" <| replace_only_first False -## - , replace(a, p, r) as "cs=s/d r=f of=f" - , case when instr(a, p) = 0 then a - else substr(a, 1, instr(a, p)-1) || - r || - substr(a, instr(a, p)+length(p)) - end as "cs=s/d r=f of=t" - , case when instr(lower(a), p) = 0 then a - else substr(a, 1, instr(lower(a), p)-1) || - r || - substr(a, instr(lower(a), p)+length(p)) - end as "cs=i r=f of=t" - replace_only_first case_sensitive t p r = search_string = if case_sensitive then t else Builder.code "LOWER(" ++ t ++ ")" From ade4bc9a7eb7c73634f4be03f8db9dfdf84231d3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 12:46:11 -0400 Subject: [PATCH 12/36] in-mem col params test --- .../Common_Table_Operations/Column_Operations_Spec.enso | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 1ee1f0315f42..233b0deffbf5 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1044,6 +1044,14 @@ spec setup = a.replace "[aeiou]".to_regex "#" . to_vector . should_equal ["Alph#", "Br#v#", "Ch#rl##", "D#lt#", "Ech#", "F#xtr#t"] a.replace "([aeiou])(.*?)[aeiou]".to_regex "$1$2$1" . to_vector . should_equal ["Alpha", "Brava", "Charlae", "Delte", "Echo", "Foxtrot"] + Test.specify "should take pattern and replacement string columns" <| + t = table_builder [["x", ["hello", "what", "yes"]], ["patterns", ["ell", "wh", "es"]], ["replacements", ["xyz", "qwer", "asdf"]]] + col = t.at "x" + patterns = t.at "patterns" + replacements = t.at "replacements" + + col.replace patterns replacements . to_vector . should_equal ["hxyzo", "qwerat", "yasdf"] + Test.specify "should only allow replace on Text columns" <| c.replace "a" "#" . should_fail_with Invalid_Value_Type a.replace 1 "#" . should_fail_with Invalid_Value_Type From fdc01856d3f1e7c365217d84d8e7079eb12212a6 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 12:53:27 -0400 Subject: [PATCH 13/36] complex regex --- test/Table_Tests/src/Database/Postgres_Spec.enso | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 55f70fb8e534..e7d657b95efe 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -300,6 +300,11 @@ postgres_specific_spec connection db_name setup = Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + Test.specify "can properly escape complex regexes" <| + regex = "^([^\(]+)|(?\w\d[a-z])+$" + col = table_builder [["x", [regex]]] . at "x" + col.replace regex "asdf" . to_vector . should_equal ["asdf"] + Test.group "replace: pattern and replacement columnsasdfasdf" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" From 96fcdec3258dc4699d27df1236bb95485c4d6e02 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 13:08:23 -0400 Subject: [PATCH 14/36] unicode --- .../Common_Table_Operations/Column_Operations_Spec.enso | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 233b0deffbf5..2c86617e3434 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1044,6 +1044,14 @@ spec setup = a.replace "[aeiou]".to_regex "#" . to_vector . should_equal ["Alph#", "Br#v#", "Ch#rl##", "D#lt#", "Ech#", "F#xtr#t"] a.replace "([aeiou])(.*?)[aeiou]".to_regex "$1$2$1" . to_vector . should_equal ["Alpha", "Brava", "Charlae", "Delte", "Echo", "Foxtrot"] + Test.specify "should handle unicode" <| + table = table_builder [["x", ["śćxx", "ąąasdfąą", "affib"]], ["patterns", ["ć", "ąą", "ffi"]], ["replacements", ["abc", "def", "ghi"]]] + col = table.at "x" + patterns = table.at "patterns" + replacements = table.at "replacements" + + col.replace patterns replacements . to_vector . should_equal ["śabcxx", "defasdfdef", "aghib"] + Test.specify "should take pattern and replacement string columns" <| t = table_builder [["x", ["hello", "what", "yes"]], ["patterns", ["ell", "wh", "es"]], ["replacements", ["xyz", "qwer", "asdf"]]] col = t.at "x" From 972bc130ae4e3de2fa28076d0280205fb4ca0509 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 13:17:50 -0400 Subject: [PATCH 15/36] cleanup --- .../lib/Standard/Database/0.0.0-dev/src/Data/Table.enso | 3 --- test/Table_Tests/src/Database/Postgres_Spec.enso | 5 ++--- test/Table_Tests/src/Database/SQLite_Spec.enso | 5 ++--- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index 12293ae47a59..deab21f8ddce 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -1790,9 +1790,6 @@ type Table Error.throw (Illegal_Argument.Error "Cannot create a table with no columns.") False -> sql = preprocessed.to_sql - IO.println 'SQL' - IO.println <| sql.unsafe_to_raw_sql - IO.println <| sql.unsafe_to_raw_sql column_type_suggestions = preprocessed.internal_columns.map .sql_type_reference self.connection.read_statement sql column_type_suggestions diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index e7d657b95efe..f6c85804dcf8 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,7 +261,7 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacementasdfasdf" <| + Test.group "replace: literal pattern and replacement" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -287,7 +287,6 @@ postgres_specific_spec connection db_name setup = col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] @@ -305,7 +304,7 @@ postgres_specific_spec connection db_name setup = col = table_builder [["x", [regex]]] . at "x" col.replace regex "asdf" . to_vector . should_equal ["asdf"] - Test.group "replace: pattern and replacement columnsasdfasdf" <| + Test.group "replace: pattern and replacement columns" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 7ae42db2f241..a501032ba7da 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -190,7 +190,7 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacementasdfasdf" <| + Test.group "replace: literal pattern and replacement" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -214,7 +214,6 @@ sqlite_specific_spec prefix connection setup = col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" col1.replace 'a[bcd]'.to_regex 'hey' . should_fail_with Unsupported_Database_Operation @@ -227,7 +226,7 @@ sqlite_specific_spec prefix connection setup = Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation - Test.group "replace: pattern and replacement columnsasdfasdf" <| + Test.group "replace: pattern and replacement columns" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" From 76ffbddcc365225233960c166f001b77ceb60564 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 16:26:24 -0400 Subject: [PATCH 16/36] disallow, chart --- .../src/Data/Text/Case_Sensitivity.enso | 10 +++ .../Database/0.0.0-dev/src/Data/Column.enso | 72 ++++++++++++++----- .../Column_Operations_Spec.enso | 7 ++ 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso index 0650dd5b61d9..4892f3f6303e 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso @@ -1,3 +1,4 @@ +import project.Any.Any import project.Data.Locale.Locale import project.Data.Text.Regex.Regex import project.Data.Text.Text @@ -70,3 +71,12 @@ type Case_Sensitivity Case_Sensitivity.Default -> (==) Case_Sensitivity.Sensitive -> (==) Case_Sensitivity.Insensitive locale -> (name-> criterion-> name.equals_ignore_case criterion locale) + + ## PRIVATE + Throws an error if self is Insensitive with a non-default locale + disallow_non_default_locale : Any -> Any + disallow_non_default_locale self ~action = case self of + Case_Sensitivity.Insensitive locale -> if locale == Locale.default then action else + msg = "Custom locales are not supported for this operation." + Error.throw (Illegal_Argument.Error msg) + _ -> action diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 91d2cf8b73c6..fb5cbc7808d9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1131,7 +1131,40 @@ type Column - case_sensitivity: Specifies if the text values should be compared case sensitively. - only_first: If True, only replace the first match. - - use_regex: If true, the term is used as a regular expression. + + ! Backend Support + + Each database backend supports different combinations of options: + + Text: + +----------------+------------+----------+--------+ + | case_sensitive | only_first | postgres | sqlite | + +----------------+------------+----------+--------+ + | t | f | ✓ | ✓ | + | t | t | ✓ | ✓ | + | f | f | ✓ | ✗ | + | f | t | ✓ | ✓ | + +----------------+------------+----------+--------+ + + Regex: + +----------------+------------+----------+--------+ + | case_sensitive | only_first | postgres | sqlite | + +----------------+------------+----------+--------+ + | t | f | ✓ | ✗ | + | t | t | ✓ | ✗ | + | f | f | ✓ | ✗ | + | f | t | ✓ | ✗ | + +----------------+------------+----------+--------+ + + Text Column: + +----------------+------------+----------+--------+ + | case_sensitive | only_first | postgres | sqlite | + +----------------+------------+----------+--------+ + | t | f | ✓ | ✓ | + | t | t | ✗ | ✓ | + | f | f | ✗ | ✗ | + | f | t | ✗ | ✓ | + +----------------+------------+----------+--------+ > Example Replace dashes with underscores. @@ -1149,24 +1182,25 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = - Value_Type.expect_text self <| - term_name = case term of - _ : Text -> term - _ : Column -> term - _ : Regex -> term.pattern_string - new_name = self.naming_helpers.function_name "replace" [self, term_name, new_text] - use_regex_part = case term of - _ : Text -> "TEXT" - _ : Column -> "COLUMN" - _ : Regex -> "REGEX" - case_sensitivity_part = case case_sensitivity of - Case_Sensitivity.Default -> "CS" - Case_Sensitivity.Sensitive -> "CS" - Case_Sensitivity.Insensitive _ -> "CI" - only_first_part = if only_first then "ONLY_FIRST" else "ALL" - op_name = "REPLACE_" + use_regex_part + "_" + case_sensitivity_part + "_" + only_first_part - if_replace_op_supported self.connection.dialect op_name term case_sensitivity only_first <| - self.make_ternary_op op_name term new_text new_name + case_sensitivity.disallow_non_default_locale <| + Value_Type.expect_text self <| + term_name = case term of + _ : Text -> term + _ : Column -> term + _ : Regex -> term.pattern_string + new_name = self.naming_helpers.function_name "replace" [self, term_name, new_text] + use_regex_part = case term of + _ : Text -> "TEXT" + _ : Column -> "COLUMN" + _ : Regex -> "REGEX" + case_sensitivity_part = case case_sensitivity of + Case_Sensitivity.Default -> "CS" + Case_Sensitivity.Sensitive -> "CS" + Case_Sensitivity.Insensitive _ -> "CI" + only_first_part = if only_first then "ONLY_FIRST" else "ALL" + op_name = "REPLACE_" + use_regex_part + "_" + case_sensitivity_part + "_" + only_first_part + if_replace_op_supported self.connection.dialect op_name term case_sensitivity only_first <| + self.make_ternary_op op_name term new_text new_name ## Gets the year as a number from the date stored in the column. diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 40511172ff4c..21e1c036b7da 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1107,6 +1107,13 @@ spec setup = vt3.should_be_a (Value_Type.Char ...) vt3.variable_length.should_be_true + if setup.is_database then + t4 = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]], ["B", ["A","O","a","E","o","O"]], ["C", [1,2,3,4,5,6]], ["D", ['',Nothing,'',Nothing,'','']]] + a = t4.at "A" + Test.specify "should not allow Case_Sensitivity.Insensitive with a non-default locale" <| + locale = Locale.new "en" "GB" "UTF-8" + a.replace 'asdf' 'zxcv' case_sensitivity=(Case_Sensitivity.Insensitive locale) . should_fail_with Illegal_Argument + Test.group prefix+"Column Operations - Text Trim" <| t5 = table_builder [["A", [" A ", ' \t\n\rA\r\n\t ', "xxxAxx"]], ["B", [" ",' \t',"x"]], ["C", [1,2,3]]] a = t5.at "A" From b62a94b29c4855d10907a52348bf9e3bc8d16388 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 16:29:24 -0400 Subject: [PATCH 17/36] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b2d2f34222d..a7184c81e1c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -517,6 +517,7 @@ `Column`, and in-memory `Table` to take a `Regex` in addition to a `Text`.] [7223] - [Added `cross_join` support to database tables.][7234] +- [Implemented `replace` on database columns.][7275] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -745,6 +746,7 @@ [7174]: https://github.com/enso-org/enso/pull/7174 [7223]: https://github.com/enso-org/enso/pull/7223 [7234]: https://github.com/enso-org/enso/pull/7234 +[7275]: https://github.com/enso-org/enso/pull/7275 #### Enso Compiler From 5d739d80d6b285ef7b3652b0276f124c6027b618 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 12 Jul 2023 16:32:04 -0400 Subject: [PATCH 18/36] revert --- .../src/Common_Table_Operations/Main.enso | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index d6f54bf6e536..2f69e8e7672f 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -96,25 +96,25 @@ type Test_Selection Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False take_drop=True allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True distinct_returns_first_row_from_group_if_ordered=True date_time=True fixed_length_text_columns=False supports_decimal_type=False spec setup = - Add_Row_Number_Spec.spec setup - Aggregate_Spec.spec setup - Column_Operations_Spec.spec setup - Conversion_Spec.spec setup Core_Spec.spec setup - Cross_Join_Spec.spec setup - Cross_Tab_Spec.spec setup + Select_Columns_Spec.spec setup + Column_Operations_Spec.spec setup Date_Time_Spec.spec setup - Distinct_Spec.spec setup - Expression_Spec.spec detailed=False setup + Conversion_Spec.spec setup + Aggregate_Spec.spec setup Filter_Spec.spec setup - Integration_Tests.spec setup - Join_Spec.spec setup Missing_Values_Spec.spec setup Order_By_Spec.spec setup - Select_Columns_Spec.spec setup Take_Drop_Spec.spec setup - Transpose_Spec.spec setup - Union_Spec.spec setup + Expression_Spec.spec detailed=False setup + Join_Spec.spec setup + Cross_Join_Spec.spec setup Zip_Spec.spec setup + Union_Spec.spec setup + Distinct_Spec.spec setup + Cross_Tab_Spec.spec setup + Transpose_Spec.spec setup + Add_Row_Number_Spec.spec setup + Integration_Tests.spec setup main = run_default_backend spec From c5f2fe130f3089758b8ca963e383f56a80cec059 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Jul 2023 15:04:23 -0400 Subject: [PATCH 19/36] pg --- .../Database/0.0.0-dev/src/Data/Column.enso | 32 +------ .../Internal/Postgres/Postgres_Dialect.enso | 95 +++++++++++-------- .../src/Database/Postgres_Spec.enso | 2 +- 3 files changed, 58 insertions(+), 71 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 3f649b8ae867..abb607994682 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1190,25 +1190,9 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = - case_sensitivity.disallow_non_default_locale <| - Value_Type.expect_text self <| - term_name = case term of - _ : Text -> term - _ : Column -> term - _ : Regex -> term.pattern_string - new_name = self.naming_helpers.function_name "replace" [self, term_name, new_text] - use_regex_part = case term of - _ : Text -> "TEXT" - _ : Column -> "COLUMN" - _ : Regex -> "REGEX" - case_sensitivity_part = case case_sensitivity of - Case_Sensitivity.Default -> "CS" - Case_Sensitivity.Sensitive -> "CS" - Case_Sensitivity.Insensitive _ -> "CI" - only_first_part = if only_first then "ONLY_FIRST" else "ALL" - op_name = "REPLACE_" + use_regex_part + "_" + case_sensitivity_part + "_" + only_first_part - if_replace_op_supported self.connection.dialect op_name term case_sensitivity only_first <| - self.make_ternary_op op_name term new_text new_name + Value_Type.expect_text self <| case_sensitivity.disallow_non_default_locale <| + input_type = Meta.type_of term + self.make_op "REPLACE" [term, new_text] "new name" [input_type, case_sensitivity, only_first] ## Gets the year as a number from the date stored in the column. @@ -1640,13 +1624,3 @@ adapt_unified_column column expected_type = A shorthand to be able to share the implementations between in-memory and database. simple_unary_op column op_kind = column.make_unary_op op_kind - -## PRIVATE - Build a friendly error explaining that the combination of REPLACE options is - not supported for this backend. -if_replace_op_supported : Any -> Text -> Text | Column | Regex -> Case_Sensitivity -> Boolean -> Any -> Any -if_replace_op_supported dialect op_name term case_sensitivity only_first ~action = - if dialect.is_supported op_name then action else - term_type = Meta.type_of term . to_text - msg = "The REPLACE operation on " + dialect.name + " is not supported for the following options: argument type " + term_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text - Error.throw (Unsupported_Database_Operation.Error msg) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index c4b131d4b220..dab93a838ee7 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -12,6 +12,7 @@ from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all from Standard.Table.Errors import Inexact_Type_Coercion import project.Connection.Connection.Connection +import project.Data.Column.Column import project.Data.Dialect import project.Data.SQL.Builder import project.Data.SQL.SQL_Fragment @@ -243,8 +244,7 @@ type Postgres_Dialect ## PRIVATE make_internal_generator_dialect = cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]] - text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive]+concat_ops+cases+trim_ops - text_replace = [replace_text_cs_all, replace_text_cs_only_first, replace_text_ci_all, replace_text_ci_only_first, replace_regex_cs_all, replace_regex_cs_only_first, replace_regex_ci_all, replace_regex_ci_only_first, replace_column_cs_all] + text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive, ["REPLACE", replace]]+concat_ops+cases+trim_ops counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] arith_extensions = [is_nan, is_inf, floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]] bool = [bool_or] @@ -254,7 +254,7 @@ make_internal_generator_dialect = stats = [agg_median, agg_mode, agg_percentile, stddev_pop, stddev_samp] date_ops = [make_extract_as_int "year", make_extract_as_int "quarter", make_extract_as_int "month", make_extract_as_int "week", make_extract_as_int "day", make_extract_as_int "hour", make_extract_as_int "minute", make_extract_fractional_as_int "second", make_extract_fractional_as_int "millisecond" modulus=1000, make_extract_fractional_as_int "microsecond" modulus=1000, ["date_add", make_date_add], ["date_diff", make_date_diff]] special_overrides = [is_null, is_empty] - my_mappings = text + text_replace + counts + stats + first_last_aggregators + arith_extensions + bool + date_ops + special_overrides + my_mappings = text + counts + stats + first_last_aggregators + arith_extensions + bool + date_ops + special_overrides Base_Generator.base_dialect . extend_with my_mappings ## PRIVATE @@ -494,44 +494,57 @@ decimal_div = Base_Generator.lift_binary_op "DECIMAL_DIV" x-> y-> decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS decimal) / CAST(" ++ y ++ " AS decimal)) * " ++ y -replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> - Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" - -replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" t-> p-> r-> - escaped_pattern = escape_regex_param p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 1)" - -replace_text_ci_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ALL" t-> p-> r-> - escaped_pattern = escape_regex_param p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 0, 'i')" - -replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" t-> p-> r-> - escaped_pattern = escape_regex_param p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ escaped_pattern ++ ", " ++ r ++ ", 1, 1, 'i')" - -replace_regex_cs_all = Base_Generator.lift_ternary_op "REPLACE_REGEX_CS_ALL" t-> p-> r-> - pattern_string = transform_literal_param (_.pattern_string) p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 0)" - -replace_regex_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_REGEX_CS_ONLY_FIRST" t-> p-> r-> - pattern_string = transform_literal_param (_.pattern_string) p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 1)" - -replace_regex_ci_all = Base_Generator.lift_ternary_op "REPLACE_REGEX_CI_ALL" t-> p-> r-> - pattern_string = transform_literal_param (_.pattern_string) p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 0, 'i')" - -replace_regex_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_REGEX_CI_ONLY_FIRST" t-> p-> r-> - pattern_string = transform_literal_param (_.pattern_string) p - Builder.code "REGEXP_REPLACE(" ++ t ++ ", " ++ pattern_string ++ ", " ++ r ++ ", 1, 1, 'i')" - -replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t-> p-> r-> - Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" - -## PRIVATE - Escape a literal regex param -escape_regex_param : Builder -> Builder -escape_regex_param regex_param = transform_literal_param Regex.escape regex_param +replace : Vector Builder -> Any -> Builder +replace args metadata = + input = args.at 0 + pattern = args.at 1 + replacement = args.at 2 + + input_type = metadata.at 0 + case_sensitivity = metadata.at 1 + only_first = metadata.at 2 + + expression = case input_type of + Text -> + ## To use REGEXP_REPLACE on a non-regex, we have to escape it. + escaped_pattern = transform_literal_param Regex.escape pattern + case only_first of + False -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 1, 0, 'i')" + _ -> + Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" + True -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 1, 1, 'i')" + _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 1, 1)" + Regex -> + pattern_string = transform_literal_param (_.pattern_string) pattern + case only_first of + False -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 0, 'i')" + _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 0)" + True -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 1, 'i')" + _ -> + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 1)" + Column -> + case only_first of + False -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> + Nothing + _ -> + Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" + True -> Nothing + case expression of + Nothing -> + msg = "The REPLACE operation is not supported on PostgreSQL for the following options: argument type " + input_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text + Error.throw (Unsupported_Database_Operation.Error msg) + _ -> expression ## PRIVATE Transform the interpolation value within the Builder diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index f6c85804dcf8..c053b7406e49 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,7 +261,7 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacement" <| + Test.group "replace: literal pattern and replacementasdfasdf" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" From 0fcdbe52a52de798862d8515a81a3752ac677f19 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Thu, 13 Jul 2023 16:04:09 -0400 Subject: [PATCH 20/36] sqlite impl no longer called --- .../Database/0.0.0-dev/src/Data/Column.enso | 2 + .../Internal/Postgres/Postgres_Dialect.enso | 7 ++ .../src/Internal/SQLite/SQLite_Dialect.enso | 74 +++++++++++++++---- .../Internal/SQLite/SQLite_Type_Mapping.enso | 2 +- .../src/Database/Postgres_Spec.enso | 2 +- .../Table_Tests/src/Database/SQLite_Spec.enso | 10 ++- 6 files changed, 79 insertions(+), 18 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index abb607994682..f26cc323e677 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1190,8 +1190,10 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = + IO.println 'REPL0' Value_Type.expect_text self <| case_sensitivity.disallow_non_default_locale <| input_type = Meta.type_of term + IO.println 'REPL1' self.make_op "REPLACE" [term, new_text] "new name" [input_type, case_sensitivity, only_first] ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index dab93a838ee7..30f20a463194 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -70,6 +70,8 @@ type Postgres_Dialect according to the specific dialect. generate_sql : Query -> SQL_Statement generate_sql self query = + IO.println 'GENERATE pg' + IO.println query Base_Generator.generate_query self.internal_generator_dialect query . build ## PRIVATE @@ -496,6 +498,9 @@ decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> replace : Vector Builder -> Any -> Builder replace args metadata = + IO.println 'BBB2' + IO.println args + IO.println metadata input = args.at 0 pattern = args.at 1 replacement = args.at 2 @@ -543,6 +548,8 @@ replace args metadata = case expression of Nothing -> msg = "The REPLACE operation is not supported on PostgreSQL for the following options: argument type " + input_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text + IO.println 'ERR postgres' + IO.println msg Error.throw (Unsupported_Database_Operation.Error msg) _ -> expression diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 0d6c6aec3b7d..001e49bc7d6a 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -10,6 +10,7 @@ from Standard.Table import Value_Type from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all import project.Connection.Connection.Connection +import project.Data.Column.Column import project.Data.Dialect import project.Data.SQL.Builder import project.Data.SQL_Statement.SQL_Statement @@ -64,6 +65,8 @@ type SQLite_Dialect according to the specific dialect. generate_sql : Query -> SQL_Statement generate_sql self query = + IO.println 'GENERATE sl' + IO.println query Base_Generator.generate_query self.internal_generator_dialect query . build ## PRIVATE @@ -267,14 +270,13 @@ type SQLite_Dialect ## PRIVATE make_internal_generator_dialect = - text = [starts_with, contains, ends_with, make_case_sensitive]+concat_ops+trim_ops - text_replace = [replace_text_cs_all, replace_text_cs_only_first, replace_text_ci_only_first, replace_column_cs_all, replace_column_cs_only_first, replace_column_ci_only_first] + text = [starts_with, contains, ends_with, make_case_sensitive, ["REPLACE", replaceq]]+concat_ops+trim_ops counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] stats = [agg_stddev_pop, agg_stddev_samp] arith_extensions = [is_inf, floating_point_div, mod_op] bool = [bool_or] - my_mappings = text + text_replace + counts + stats + arith_extensions + bool + my_mappings = text + counts + stats + arith_extensions + bool Base_Generator.base_dialect . extend_with my_mappings ## PRIVATE @@ -406,17 +408,61 @@ floating_point_div = Base_Generator.lift_binary_op "/" x-> y-> mod_op = Base_Generator.lift_binary_op "mod" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS REAL) / CAST(" ++ y ++ " AS REAL)) * " ++ y -replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> - Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" - -replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" <| replace_only_first True -replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" <| replace_only_first False - -replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t-> p-> r-> - Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" - -replace_column_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ONLY_FIRST" <| replace_only_first True -replace_column_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CI_ONLY_FIRST" <| replace_only_first False +replaceq a b = + IO.println 'UMM' + Error.throw (Unsupported_Database_Operation.Error 'zxcv') + +replace : Vector Builder -> Any -> Builder +replace args metadata = + IO.println 'BBB' + IO.println args + IO.println metadata + + input = args.at 0 + pattern = args.at 1 + replacement = args.at 2 + + input_type = metadata.at 0 + case_sensitivity = metadata.at 1 + only_first = metadata.at 2 + + expression = case input_type == Text || input_type == Column of + True -> + ## To use REGEXP_REPLACE on a non-regex, we have to escape it. + case only_first of + False -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> Nothing + _ -> + Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" + True -> case case_sensitivity of + Case_Sensitivity.Insensitive _ -> + replace_only_first False input pattern replacement + _ -> + replace_only_first True input pattern replacement + False -> Nothing + IO.println 'AAAA' + IO.println expression + IO.println args + IO.println metadata + case expression of + Nothing -> + msg = "The REPLACE operation is not supported on SQLite for the following options: argument type " + input_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text + IO.println 'ERR sqlite' + IO.println msg + Error.throw (Unsupported_Database_Operation.Error msg) + _ -> expression + +#replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> +# Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" + +#replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" <| replace_only_first True +#replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" <| replace_only_first False +# +#replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t-> p-> r-> +# Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" +# +#replace_column_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ONLY_FIRST" <| replace_only_first True +#replace_column_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CI_ONLY_FIRST" <| replace_only_first False replace_only_first case_sensitive t p r = search_string = if case_sensitive then t else diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso index 438960a6eb88..a87f5c199eef 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso @@ -181,7 +181,7 @@ operations_map = always_boolean_ops = ["==", "!=", "equals_ignore_case", ">=", "<=", "<", ">", "BETWEEN", "AND", "OR", "NOT", "IS_NULL", "IS_EMPTY", "LIKE", "IS_IN", "IS_IN_COLUMN", "starts_with", "ends_with", "contains", "BOOL_OR", "IS_INF"] always_floating_ops = ["/", "mod", "AVG", "STDDEV_POP", "STDDEV_SAMP", "ROUND"] - always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE_TEXT_CS_ALL", "REPLACE_TEXT_CS_ONLY_FIRST", "REPLACE_TEXT_CI_ONLY_FIRST", "REPLACE_COLUMN_CS_ALL", "REPLACE_COLUMN_CS_ONLY_FIRST", "REPLACE_COLUMN_CI_ONLY_FIRST"] + always_text_ops = ["ADD_TEXT", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MAKE_CASE_SENSITIVE", "FOLD_CASE", "TRIM", "LTRIM", "RTRIM", "REPLACE"] always_integer_ops = ["COUNT", "COUNT_IS_NULL", "COUNT_DISTINCT", "COUNT_DISTINCT_INCLUDE_NULL", "COUNT_EMPTY", "COUNT_NOT_EMPTY", "COUNT_ROWS", "ROW_NUMBER"] same_as_first = ["TRUNCATE", "CEIL", "FLOOR"] arithmetic_ops = ["ADD_NUMBER", "-", "*", "^", "%", "SUM"] diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index c053b7406e49..98741e2c2440 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -304,7 +304,7 @@ postgres_specific_spec connection db_name setup = col = table_builder [["x", [regex]]] . at "x" col.replace regex "asdf" . to_vector . should_equal ["asdf"] - Test.group "replace: pattern and replacement columns" <| + Test.group "replace: pattern and replacement columnsasdfasdf" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index a501032ba7da..c72bad3c3471 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -190,7 +190,13 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacement" <| + Test.group 'qwerqwer' <| + col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" + col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation + + Test.group "replace: literal pattern and replacementasdfasdf" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -226,7 +232,7 @@ sqlite_specific_spec prefix connection setup = Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation - Test.group "replace: pattern and replacement columns" <| + Test.group "replace: pattern and replacement columnsasdfasdf" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" From 306a3319444d954dcad68b6618acb8beb11c03ea Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Jul 2023 10:03:13 -0400 Subject: [PATCH 21/36] metadata --- .../Database/0.0.0-dev/src/Data/Column.enso | 2 -- .../Internal/Postgres/Postgres_Dialect.enso | 9 +----- .../src/Internal/SQLite/SQLite_Dialect.enso | 32 ++----------------- .../src/Database/Postgres_Spec.enso | 6 ++-- .../Table_Tests/src/Database/SQLite_Spec.enso | 18 ++++------- 5 files changed, 12 insertions(+), 55 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index f26cc323e677..abb607994682 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1190,10 +1190,8 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' replace : Text | Column | Regex -> Text | Column -> Case_Sensitivity -> Boolean -> Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = - IO.println 'REPL0' Value_Type.expect_text self <| case_sensitivity.disallow_non_default_locale <| input_type = Meta.type_of term - IO.println 'REPL1' self.make_op "REPLACE" [term, new_text] "new name" [input_type, case_sensitivity, only_first] ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 30f20a463194..68b4528dcb8b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -70,8 +70,6 @@ type Postgres_Dialect according to the specific dialect. generate_sql : Query -> SQL_Statement generate_sql self query = - IO.println 'GENERATE pg' - IO.println query Base_Generator.generate_query self.internal_generator_dialect query . build ## PRIVATE @@ -498,9 +496,6 @@ decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> replace : Vector Builder -> Any -> Builder replace args metadata = - IO.println 'BBB2' - IO.println args - IO.println metadata input = args.at 0 pattern = args.at 1 replacement = args.at 2 @@ -547,9 +542,7 @@ replace args metadata = True -> Nothing case expression of Nothing -> - msg = "The REPLACE operation is not supported on PostgreSQL for the following options: argument type " + input_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text - IO.println 'ERR postgres' - IO.println msg + msg = "The REPLACE operation is not supported on PostgreSQL for the following options: argument type " + input_type.to_text + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text Error.throw (Unsupported_Database_Operation.Error msg) _ -> expression diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 001e49bc7d6a..744c335c56b4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -65,8 +65,6 @@ type SQLite_Dialect according to the specific dialect. generate_sql : Query -> SQL_Statement generate_sql self query = - IO.println 'GENERATE sl' - IO.println query Base_Generator.generate_query self.internal_generator_dialect query . build ## PRIVATE @@ -270,7 +268,7 @@ type SQLite_Dialect ## PRIVATE make_internal_generator_dialect = - text = [starts_with, contains, ends_with, make_case_sensitive, ["REPLACE", replaceq]]+concat_ops+trim_ops + text = [starts_with, contains, ends_with, make_case_sensitive, ["REPLACE", replace]]+concat_ops+trim_ops counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]] stats = [agg_stddev_pop, agg_stddev_samp] arith_extensions = [is_inf, floating_point_div, mod_op] @@ -408,16 +406,8 @@ floating_point_div = Base_Generator.lift_binary_op "/" x-> y-> mod_op = Base_Generator.lift_binary_op "mod" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS REAL) / CAST(" ++ y ++ " AS REAL)) * " ++ y -replaceq a b = - IO.println 'UMM' - Error.throw (Unsupported_Database_Operation.Error 'zxcv') - replace : Vector Builder -> Any -> Builder replace args metadata = - IO.println 'BBB' - IO.println args - IO.println metadata - input = args.at 0 pattern = args.at 1 replacement = args.at 2 @@ -440,30 +430,12 @@ replace args metadata = _ -> replace_only_first True input pattern replacement False -> Nothing - IO.println 'AAAA' - IO.println expression - IO.println args - IO.println metadata case expression of Nothing -> - msg = "The REPLACE operation is not supported on SQLite for the following options: argument type " + input_type + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text - IO.println 'ERR sqlite' - IO.println msg + msg = "The REPLACE operation is not supported on SQLite for the following options: argument type " + input_type.to_text + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text Error.throw (Unsupported_Database_Operation.Error msg) _ -> expression -#replace_text_cs_all = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ALL" t-> p-> r-> -# Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" - -#replace_text_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CS_ONLY_FIRST" <| replace_only_first True -#replace_text_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_TEXT_CI_ONLY_FIRST" <| replace_only_first False -# -#replace_column_cs_all = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ALL" t-> p-> r-> -# Builder.code "REPLACE(" ++ t ++ ", " ++ p ++ ", " ++ r ++ ")" -# -#replace_column_cs_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CS_ONLY_FIRST" <| replace_only_first True -#replace_column_ci_only_first = Base_Generator.lift_ternary_op "REPLACE_COLUMN_CI_ONLY_FIRST" <| replace_only_first False - replace_only_first case_sensitive t p r = search_string = if case_sensitive then t else Builder.code "LOWER(" ++ t ++ ")" diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 98741e2c2440..328e78b3667b 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -316,13 +316,13 @@ postgres_specific_spec connection db_name setup = col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col.replace patterns replacements only_first=True . should_fail_with Unsupported_Database_Operation + col.replace patterns replacements only_first=True . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation run_tests connection db_name = prefix = "[PostgreSQL] " diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index c72bad3c3471..3293e82f6a10 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -190,12 +190,6 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group 'qwerqwer' <| - col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" - col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" - Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation - Test.group "replace: literal pattern and replacementasdfasdf" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -213,7 +207,7 @@ sqlite_specific_spec prefix connection setup = col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation + col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO'] @@ -221,16 +215,16 @@ sqlite_specific_spec prefix connection setup = col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' . should_fail_with Unsupported_Database_Operation + col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . should_fail_with Unsupported_Database_Operation + col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . should_fail_with Unsupported_Database_Operation + col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation Test.group "replace: pattern and replacement columnsasdfasdf" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] @@ -247,7 +241,7 @@ sqlite_specific_spec prefix connection setup = col.replace patterns replacements only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . should_fail_with Unsupported_Database_Operation + col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] From 21c5762bcd7670a7e56e1ad7abe5b637ab84077c Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Jul 2023 10:06:16 -0400 Subject: [PATCH 22/36] review --- .../Database/0.0.0-dev/src/Data/Column.enso | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index abb607994682..87dea5596eaf 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -158,21 +158,6 @@ type Column new_type_ref = type_mapping.infer_return_type infer_from_database_callback op_kind [self]+operands new_expr Column.Value new_name self.connection new_type_ref new_expr self.context - ## PRIVATE - - Creates a ternary operation with given kind and operand. - - Arguments: - - op_kind: The kind of ternary operator. - - operand1: The second operand to the binary operator. - - operand2: The third operand to the binary operator. - - new_name: The name of the resulting column. - make_ternary_op : Text -> Column | Any -> Column | Any -> (Text | Nothing) -> Column - make_ternary_op self op_kind operand1 operand2 new_name=Nothing = - effective_new_name = new_name.if_nothing <| - self.naming_helpers.function_name op_kind [self, operand1, operand2] - self.make_op op_kind [operand1, operand2] effective_new_name - ## PRIVATE Creates a binary operation with given kind and operand. From ec355d2ae6cba204378d971afa7cc08e3da87dd9 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Jul 2023 10:07:42 -0400 Subject: [PATCH 23/36] review --- .../0.0.0-dev/src/Internal/Base_Generator.enso | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 1f58ae209461..bc42f1f3d9c0 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -111,22 +111,6 @@ lift_binary_op name function = True -> function (arguments.at 0) (arguments.at 1) [name, generator] -## PRIVATE - - A helper function to create a binary operation from a function. - - Arguments: - - name: Name of the operation, used for error reporting. - - function: A function taking exactly two arguments: the generated SQL code - for the argument of the operation, and returning the generated SQL code for - the whole operation. -lift_ternary_op : Text -> (Builder -> Builder -> Builder -> Builder) -> [Text, (Vector Builder -> Builder)] -lift_ternary_op name function = - generator = arguments -> case arguments.length == 3 of - False -> Error.throw <| Illegal_State.Error ("Invalid amount of arguments for operation " + name + ".") - True -> function (arguments.at 0) (arguments.at 1) (arguments.at 2) - [name, generator] - ## PRIVATE A helper function to create a unary operator which is added to the right of From 64f6125f6242c088a7d74dd9f9b9f8c2afbbe44e Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Jul 2023 10:09:18 -0400 Subject: [PATCH 24/36] cleanup --- test/Table_Tests/src/Database/Postgres_Spec.enso | 4 ++-- test/Table_Tests/src/Database/SQLite_Spec.enso | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 328e78b3667b..022b9f942e1e 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,7 +261,7 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacementasdfasdf" <| + Test.group "replace: literal pattern and replacement" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -304,7 +304,7 @@ postgres_specific_spec connection db_name setup = col = table_builder [["x", [regex]]] . at "x" col.replace regex "asdf" . to_vector . should_equal ["asdf"] - Test.group "replace: pattern and replacement columnsasdfasdf" <| + Test.group "replace: pattern and replacement columns" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 3293e82f6a10..12525211600e 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -190,7 +190,7 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacementasdfasdf" <| + Test.group "replace: literal pattern and replacement" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -226,7 +226,7 @@ sqlite_specific_spec prefix connection setup = Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - Test.group "replace: pattern and replacement columnsasdfasdf" <| + Test.group "replace: pattern and replacement columns" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" From fa1eeb1d836ccdc341e3daebc5c3a22670e253cf Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Fri, 14 Jul 2023 14:48:55 -0400 Subject: [PATCH 25/36] prefix --- test/Table_Tests/src/Database/Postgres_Spec.enso | 4 ++-- test/Table_Tests/src/Database/SQLite_Spec.enso | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 022b9f942e1e..a09ac32a59d1 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,7 +261,7 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacement" <| + Test.group "[PostgreSQL] replace: literal pattern and replacementasdfasdf" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -304,7 +304,7 @@ postgres_specific_spec connection db_name setup = col = table_builder [["x", [regex]]] . at "x" col.replace regex "asdf" . to_vector . should_equal ["asdf"] - Test.group "replace: pattern and replacement columns" <| + Test.group "[PostgreSQL] replace: pattern and replacement columns" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 12525211600e..16e1ce5826d0 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -190,7 +190,7 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "replace: literal pattern and replacement" <| + Test.group prefix+"replace: literal pattern and replacement" <| col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" @@ -226,7 +226,7 @@ sqlite_specific_spec prefix connection setup = Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - Test.group "replace: pattern and replacement columns" <| + Test.group prefix+"replace: pattern and replacement columns" <| table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] col = table.at "x" patterns = table.at "patterns" From fb25cb1f908f22d8adc770f4913213b63b748613 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 17 Jul 2023 10:37:45 -0400 Subject: [PATCH 26/36] use pg 14 api --- .../src/Internal/Postgres/Postgres_Dialect.enso | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 68b4528dcb8b..069106540b27 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -511,27 +511,27 @@ replace args metadata = case only_first of False -> case case_sensitivity of Case_Sensitivity.Insensitive _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 1, 0, 'i')" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 'ig')" _ -> Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" True -> case case_sensitivity of Case_Sensitivity.Insensitive _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 1, 1, 'i')" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 'i')" _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 1, 1)" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ")" Regex -> pattern_string = transform_literal_param (_.pattern_string) pattern case only_first of False -> case case_sensitivity of Case_Sensitivity.Insensitive _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 0, 'i')" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 'ig')" _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 0)" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 'g')" True -> case case_sensitivity of Case_Sensitivity.Insensitive _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 1, 'i')" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 'i')" _ -> - Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 1, 1)" + Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ")" Column -> case only_first of False -> case case_sensitivity of From c58a27e46e8bb65d26d56d5d7a7d836822a61aea Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 17 Jul 2023 11:02:15 -0400 Subject: [PATCH 27/36] pass original term as metadata --- .../Database/0.0.0-dev/src/Data/Column.enso | 2 +- .../Internal/Postgres/Postgres_Dialect.enso | 21 +++++++------------ .../src/Internal/SQLite/SQLite_Dialect.enso | 6 +++--- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 87dea5596eaf..8afc787b7407 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1177,7 +1177,7 @@ type Column replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = Value_Type.expect_text self <| case_sensitivity.disallow_non_default_locale <| input_type = Meta.type_of term - self.make_op "REPLACE" [term, new_text] "new name" [input_type, case_sensitivity, only_first] + self.make_op "REPLACE" [term, new_text] "new name" [term, input_type, case_sensitivity, only_first] ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 069106540b27..82ed10771708 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -500,14 +500,17 @@ replace args metadata = pattern = args.at 1 replacement = args.at 2 - input_type = metadata.at 0 - case_sensitivity = metadata.at 1 - only_first = metadata.at 2 + ## `raw_pattern` is a `Text1 or `Regex`; it's the same value as `input`, but not + embedded in IR. + raw_pattern = metadata.at 0 + input_type = metadata.at 1 + case_sensitivity = metadata.at 2 + only_first = metadata.at 3 expression = case input_type of Text -> ## To use REGEXP_REPLACE on a non-regex, we have to escape it. - escaped_pattern = transform_literal_param Regex.escape pattern + escaped_pattern = Builder.interpolation (Regex.escape raw_pattern) case only_first of False -> case case_sensitivity of Case_Sensitivity.Insensitive _ -> @@ -520,7 +523,7 @@ replace args metadata = _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ")" Regex -> - pattern_string = transform_literal_param (_.pattern_string) pattern + pattern_string = Builder.interpolation raw_pattern.pattern_string case only_first of False -> case case_sensitivity of Case_Sensitivity.Insensitive _ -> @@ -546,14 +549,6 @@ replace args metadata = Error.throw (Unsupported_Database_Operation.Error msg) _ -> expression -## PRIVATE - Transform the interpolation value within the Builder -transform_literal_param : (Any -> Any) -> Builder -> Builder -transform_literal_param transformer param = - x = case param of Builder.Value (Vector_Builder.Leaf x) -> case x.at 0 of SQL_Fragment.Interpolation y -> y - transformed_x = transformer x - Builder.Value (Vector_Builder.Leaf [SQL_Fragment.Interpolation transformed_x]) - ## PRIVATE make_extract_as_int enso_name sql_name=enso_name = Base_Generator.lift_unary_op enso_name arg-> diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 744c335c56b4..acc6557d8d60 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -412,9 +412,9 @@ replace args metadata = pattern = args.at 1 replacement = args.at 2 - input_type = metadata.at 0 - case_sensitivity = metadata.at 1 - only_first = metadata.at 2 + input_type = metadata.at 1 + case_sensitivity = metadata.at 2 + only_first = metadata.at 3 expression = case input_type == Text || input_type == Column of True -> From f5692640e3fb758451303e446413bdf053869309 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 17 Jul 2023 12:05:04 -0400 Subject: [PATCH 28/36] unused columns --- .../src/Common_Table_Operations/Column_Operations_Spec.enso | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 21e1c036b7da..c3f1d69d54e3 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1108,11 +1108,10 @@ spec setup = vt3.variable_length.should_be_true if setup.is_database then - t4 = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]], ["B", ["A","O","a","E","o","O"]], ["C", [1,2,3,4,5,6]], ["D", ['',Nothing,'',Nothing,'','']]] - a = t4.at "A" + col = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]]] . at 'A' Test.specify "should not allow Case_Sensitivity.Insensitive with a non-default locale" <| locale = Locale.new "en" "GB" "UTF-8" - a.replace 'asdf' 'zxcv' case_sensitivity=(Case_Sensitivity.Insensitive locale) . should_fail_with Illegal_Argument + col.replace 'asdf' 'zxcv' case_sensitivity=(Case_Sensitivity.Insensitive locale) . should_fail_with Illegal_Argument Test.group prefix+"Column Operations - Text Trim" <| t5 = table_builder [["A", [" A ", ' \t\n\rA\r\n\t ', "xxxAxx"]], ["B", [" ",' \t',"x"]], ["C", [1,2,3]]] From e0da4fc8afbb01a23b4004bc61f259842db8c53d Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 17 Jul 2023 17:14:13 -0400 Subject: [PATCH 29/36] do supported check early --- .../src/Data/Text/Case_Sensitivity.enso | 9 ++ .../Database/0.0.0-dev/src/Data/Column.enso | 12 ++- .../Database/0.0.0-dev/src/Data/Dialect.enso | 8 ++ .../Internal/Postgres/Postgres_Dialect.enso | 45 ++++++---- .../src/Internal/Replace_Params.enso | 17 ++++ .../src/Internal/SQLite/SQLite_Dialect.enso | 28 +++--- .../Column_Operations_Spec.enso | 89 +++++++++++++++++-- .../src/Database/Postgres_Spec.enso | 63 ------------- .../Table_Tests/src/Database/SQLite_Spec.enso | 20 ----- 9 files changed, 174 insertions(+), 117 deletions(-) create mode 100644 distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso index 59f27500618c..48c4baac383c 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Case_Sensitivity.enso @@ -61,3 +61,12 @@ type Case_Sensitivity to_explicit_sensitivity_in_memory self = case self of Case_Sensitivity.Default -> Case_Sensitivity.Sensitive _ -> self + + ## PRIVATE + Throws an error if self is Insensitive with a non-default locale + disallow_non_default_locale : Any -> Any + disallow_non_default_locale self ~action = case self of + Case_Sensitivity.Insensitive locale -> if locale == Locale.default then action else + msg = "Custom locales are not supported for this operation." + Error.throw (Illegal_Argument.Error msg) + _ -> action diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 7a093349e3ab..1453accad14f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -23,6 +23,7 @@ import project.Internal.IR.Context.Context import project.Internal.IR.Internal_Column.Internal_Column import project.Internal.IR.Query.Query import project.Internal.IR.SQL_Expression.SQL_Expression +import project.Internal.Replace_Params.Replace_Params import project.Internal.SQL_Type_Reference.SQL_Type_Reference from project.Data.Table import freshen_columns, Table from project.Errors import Integrity_Error, Unsupported_Database_Operation, Unsupported_Name @@ -1174,11 +1175,18 @@ type Column column.replace '"(.*?)"'.to_regex '($1)' @term make_regex_text_widget - replace : Text | Regex | Column -> Text | Column -> Case_Sensitivity -> Boolean -> Column + replace : Text | Regex | Column -> Text | Column -> Case_Sensitivity -> Boolean -> Column ! Unsupported_Database_Operation replace self term="" new_text="" case_sensitivity=Case_Sensitivity.Default only_first=False = Value_Type.expect_text self <| case_sensitivity.disallow_non_default_locale <| input_type = Meta.type_of term - self.make_op "REPLACE" [term, new_text] "new name" [term, input_type, case_sensitivity, only_first] + params = Replace_Params.Value input_type case_sensitivity only_first + case self.connection.dialect.are_replace_params_supported params of + False -> params.throw_unsupported + True -> + raw_term = case term of + _ : Regex -> term.pattern_string + _ -> term + self.make_op "REPLACE" [raw_term, new_text] "new name" [term, params] ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso index b2683b79f345..4de23ab8e662 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso @@ -19,6 +19,7 @@ import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.Query.Query import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.Postgres.Postgres_Dialect +import project.Internal.Replace_Params.Replace_Params import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.SQLite.SQLite_Dialect @@ -231,6 +232,13 @@ type Dialect _ = [period, operation_input_type] Unimplemented.throw "This is an interface only." + ## PRVIATE + Returns true if the `replace` parameters are supported by this backend. + are_replace_params_supported : Replace_Params -> Boolean + are_replace_params_supported self replace_params = + _ = [replace_params] + Unimplemented.throw "This is an interface only." + ## PRIVATE The dialect of SQLite databases. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 82ed10771708..554c222e8844 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -35,6 +35,7 @@ import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind import project.Internal.Postgres.Postgres_Error_Mapper.Postgres_Error_Mapper import project.Internal.Postgres.Postgres_Type_Mapping.Postgres_Type_Mapping +import project.Internal.Replace_Params.Replace_Params import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.Statement_Setter.Statement_Setter @@ -241,6 +242,11 @@ type Postgres_Dialect _ -> Date_Period_Metadata.Value period operation_input_type + ## PRVIATE + Returns true if the `replace` parameters are supported by this backend. + are_replace_params_supported : Replace_Params -> Boolean + are_replace_params_supported self replace_params = supported_replace_params.contains replace_params + ## PRIVATE make_internal_generator_dialect = cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]] @@ -494,6 +500,17 @@ decimal_div = Base_Generator.lift_binary_op "DECIMAL_DIV" x-> y-> decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS decimal) / CAST(" ++ y ++ " AS decimal)) * " ++ y +## PRIVATE +supported_replace_params : Set Replace_Params +supported_replace_params = + e0 = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive False] + e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] + e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] + e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Default True, Replace_Params.Value Column Case_Sensitivity.Sensitive False] + e5 = [Replace_Params.Value Column Case_Sensitivity.Sensitive True, Replace_Params.Value Column Case_Sensitivity.Insensitive False, Replace_Params.Value Column Case_Sensitivity.Insensitive True] + Set.from_vector <| (e0 + e1 + e2 + e3 + e4 + e5).map x-> [x, 0] + replace : Vector Builder -> Any -> Builder replace args metadata = input = args.at 0 @@ -503,51 +520,47 @@ replace args metadata = ## `raw_pattern` is a `Text1 or `Regex`; it's the same value as `input`, but not embedded in IR. raw_pattern = metadata.at 0 - input_type = metadata.at 1 - case_sensitivity = metadata.at 2 - only_first = metadata.at 3 + replace_params = metadata.at 1 - expression = case input_type of + expression = case replace_params.input_type of Text -> ## To use REGEXP_REPLACE on a non-regex, we have to escape it. escaped_pattern = Builder.interpolation (Regex.escape raw_pattern) - case only_first of - False -> case case_sensitivity of + case replace_params.only_first of + False -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 'ig')" _ -> Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" - True -> case case_sensitivity of + True -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ", 'i')" _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ escaped_pattern ++ ", " ++ replacement ++ ")" Regex -> pattern_string = Builder.interpolation raw_pattern.pattern_string - case only_first of - False -> case case_sensitivity of + case replace_params.only_first of + False -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 'ig')" _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 'g')" - True -> case case_sensitivity of + True -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ", 'i')" _ -> Builder.code "REGEXP_REPLACE(" ++ input ++ ", " ++ pattern_string ++ ", " ++ replacement ++ ")" Column -> - case only_first of - False -> case case_sensitivity of + case replace_params.only_first of + False -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> Nothing _ -> Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" True -> Nothing case expression of - Nothing -> - msg = "The REPLACE operation is not supported on PostgreSQL for the following options: argument type " + input_type.to_text + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text - Error.throw (Unsupported_Database_Operation.Error msg) - _ -> expression + Nothing -> replace_params.throw_unsupported + _ -> expression ## PRIVATE make_extract_as_int enso_name sql_name=enso_name = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso new file mode 100644 index 000000000000..597a36c4f172 --- /dev/null +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso @@ -0,0 +1,17 @@ +import Standard.Base.Data.Boolean.Boolean +import Standard.Base.Error.Error +import Standard.Base.Meta.Type +import Standard.Base.Nothing.Nothing +import Standard.Base.Data.Text.Case_Sensitivity.Case_Sensitivity + +from project.Errors import Unsupported_Database_Operation + +# Specifies a set of parameters to Table.replace +type Replace_Params + Value input_type (case_sensitivity : Case_Sensitivity) (only_first : Boolean) + + ## Raise an exception for an unsupported combination of parameters + throw_unsupported : Nothing ! Unsupported_Database_Operation + throw_unsupported self = + msg = "The REPLACE operation is not supported on PostgreSQL for the following options: argument type " + self.input_type.to_text + ", case_sensitivity " + self.case_sensitivity.to_display_text + ", only_first " + self.only_first.to_text + Error.throw (Unsupported_Database_Operation.Error msg) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index acc6557d8d60..e5412fc8e2a0 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -29,6 +29,7 @@ import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.Query.Query import project.Internal.IR.SQL_Expression.SQL_Expression import project.Internal.IR.SQL_Join_Kind.SQL_Join_Kind +import project.Internal.Replace_Params.Replace_Params import project.Internal.SQL_Type_Mapping.SQL_Type_Mapping import project.Internal.SQL_Type_Reference.SQL_Type_Reference import project.Internal.SQLite.SQLite_Error_Mapper.SQLite_Error_Mapper @@ -266,6 +267,11 @@ type SQLite_Dialect _ = [period, operation_input_type] Error.throw (Unsupported_Database_Operation.Error "SQLite backend does not support date/time operations.") + ## PRVIATE + Returns true if the `replace` parameters are suppoerted by this backend. + are_replace_params_supported : Replace_Params -> Boolean + are_replace_params_supported self replace_params = supported_replace_params.contains replace_params + ## PRIVATE make_internal_generator_dialect = text = [starts_with, contains, ends_with, make_case_sensitive, ["REPLACE", replace]]+concat_ops+trim_ops @@ -406,34 +412,36 @@ floating_point_div = Base_Generator.lift_binary_op "/" x-> y-> mod_op = Base_Generator.lift_binary_op "mod" x-> y-> x ++ " - FLOOR(CAST(" ++ x ++ " AS REAL) / CAST(" ++ y ++ " AS REAL)) * " ++ y +## PRIVATE +supported_replace_params : Set Replace_Params +supported_replace_params = + e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + Set.from_vector <| e.map x-> [x, 0] + replace : Vector Builder -> Any -> Builder replace args metadata = input = args.at 0 pattern = args.at 1 replacement = args.at 2 - input_type = metadata.at 1 - case_sensitivity = metadata.at 2 - only_first = metadata.at 3 + replace_params = metadata.at 1 - expression = case input_type == Text || input_type == Column of + expression = case replace_params.input_type == Text || replace_params.input_type == Column of True -> ## To use REGEXP_REPLACE on a non-regex, we have to escape it. - case only_first of - False -> case case_sensitivity of + case replace_params.only_first of + False -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> Nothing _ -> Builder.code "REPLACE(" ++ input ++ ", " ++ pattern ++ ", " ++ replacement ++ ")" - True -> case case_sensitivity of + True -> case replace_params.case_sensitivity of Case_Sensitivity.Insensitive _ -> replace_only_first False input pattern replacement _ -> replace_only_first True input pattern replacement False -> Nothing case expression of - Nothing -> - msg = "The REPLACE operation is not supported on SQLite for the following options: argument type " + input_type.to_text + ", case_sensitivity " + case_sensitivity.to_display_text + ", only_first " + only_first.to_text - Error.throw (Unsupported_Database_Operation.Error msg) + Nothing -> replace_params.throw_unsupported _ -> expression replace_only_first case_sensitive t p r = diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index c3f1d69d54e3..63c10882b4d8 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1,6 +1,9 @@ from Standard.Base import all + import Standard.Base.Errors.Common.Arithmetic_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Meta.Type +import Standard.Database.Internal.Replace_Params.Replace_Params from Standard.Table import Value_Type from Standard.Table.Data.Type.Value_Type import Bits @@ -1038,6 +1041,86 @@ spec setup = op a True . should_fail_with Invalid_Value_Type Test.group prefix+"Column Operations - Text Replace" <| + do_replace column term new_text case_sensitivity=Case_Sensitivity.Default only_first=False expected = + case setup.is_database of + True -> + input_type = Meta.type_of term + params = Replace_Params.Value input_type case_sensitivity only_first + case column.connection.dialect.are_replace_params_supported params of + True -> column.replace term new_text case_sensitivity only_first . to_vector . should_equal expected + False -> column.replace term new_text case_sensitivity only_first . should_fail_with Unsupported_Database_Operation + False -> + column.replace term new_text case_sensitivity only_first . to_vector . should_equal expected + + Test.group prefix+"replace: literal text pattern and replacement" <| + col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" + col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" + do_replace col0 'hello' 'bye' expected=['bye Hello', 'bye bye', 'HELLO HELLO'] + do_replace col0 'hello' 'bye' case_sensitivity=Case_Sensitivity.Default expected=['bye Hello', 'bye bye', 'HELLO HELLO'] + do_replace col0 'hello' 'bye' case_sensitivity=Case_Sensitivity.Sensitive expected=['bye Hello', 'bye bye', 'HELLO HELLO'] + do_replace col1 'a[bcd]' 'hey' expected=['hey A[bCd] hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + do_replace col0 'hello' 'bye' only_first=True expected=['bye Hello', 'bye hello', 'HELLO HELLO'] + do_replace col1 'a[bcd]' 'hey' only_first=True expected=['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + do_replace col0 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive expected=['bye bye', 'bye bye', 'bye bye'] + do_replace col1 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive expected=['hey hey hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" + do_replace col0 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True expected=['bye Hello', 'bye hello', 'bye HELLO'] + do_replace col1 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True expected=['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.group prefix+"replace: literal regex pattern and replacement" <| + col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" + + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" + do_replace col1 'a[bcd]'.to_regex 'hey' expected=['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" + do_replace col1 'a[bcd]'.to_regex 'hey' only_first=True expected=['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" + do_replace col1 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive expected=['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] + + Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" + do_replace col1 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True expected=['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] + + Test.specify "can properly escape complex regexes" <| + regex = "^([^\(]+)|(?\w\d[a-z])+$" + col = table_builder [["x", [regex]]] . at "x" + do_replace col regex "asdf" ["asdf"] + + Test.group prefix+"replace: pattern and replacement columns" <| + table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] + col = table.at "x" + patterns = table.at "patterns" + replacements = table.at "replacements" + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" + do_replace col patterns replacements expected=['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Default expected=['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Sensitive expected=['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" + do_replace col patterns replacements only_first=True expected=['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" + do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Insensitive expected=['bye bye', 'bye bye', 'HELLO bye', 'hey hey hey', 'abac ad Ab aCAd'] + + Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" + do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True expected=['bye Hello', 'bye hello', 'bye HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] + + if setup.is_database then + col = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]]] . at 'A' + Test.specify "should not allow Case_Sensitivity.Insensitive with a non-default locale" <| + locale = Locale.new "en" "GB" "UTF-8" + col.replace 'asdf' 'zxcv' case_sensitivity=(Case_Sensitivity.Insensitive locale) . should_fail_with Illegal_Argument + + Test.group prefix+"Column Operations - Text Replace (in-memory only)" <| if setup.is_database.not then t4 = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]], ["B", ["A","O","a","E","o","O"]], ["C", [1,2,3,4,5,6]], ["D", ['',Nothing,'',Nothing,'','']]] a = t4.at "A" @@ -1107,12 +1190,6 @@ spec setup = vt3.should_be_a (Value_Type.Char ...) vt3.variable_length.should_be_true - if setup.is_database then - col = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]]] . at 'A' - Test.specify "should not allow Case_Sensitivity.Insensitive with a non-default locale" <| - locale = Locale.new "en" "GB" "UTF-8" - col.replace 'asdf' 'zxcv' case_sensitivity=(Case_Sensitivity.Insensitive locale) . should_fail_with Illegal_Argument - Test.group prefix+"Column Operations - Text Trim" <| t5 = table_builder [["A", [" A ", ' \t\n\rA\r\n\t ', "xxxAxx"]], ["B", [" ",' \t',"x"]], ["C", [1,2,3]]] a = t5.at "A" diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index a09ac32a59d1..103834d9892f 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -261,69 +261,6 @@ postgres_specific_spec connection db_name setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group "[PostgreSQL] replace: literal pattern and replacementasdfasdf" <| - col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" - col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" - col0.replace 'hello' 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - - col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['hey A[bCd] hey', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] - - col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['bye bye', 'bye bye', 'bye bye'] - - col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['hey hey hey', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO'] - - col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] - - Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyhey hey hey heyhey'] - - Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['a[bcd] A[bCd] a[bcd]', 'heyac ad Ab aCAd'] - - Test.specify "can properly escape complex regexes" <| - regex = "^([^\(]+)|(?\w\d[a-z])+$" - col = table_builder [["x", [regex]]] . at "x" - col.replace regex "asdf" . to_vector . should_equal ["asdf"] - - Test.group "[PostgreSQL] replace: pattern and replacement columns" <| - table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] - col = table.at "x" - patterns = table.at "patterns" - replacements = table.at "replacements" - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" - col.replace patterns replacements . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col.replace patterns replacements only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - run_tests connection db_name = prefix = "[PostgreSQL] " name_counter = Ref.new 0 diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 16e1ce5826d0..d6a98d820e80 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -226,26 +226,6 @@ sqlite_specific_spec prefix connection setup = Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - Test.group prefix+"replace: pattern and replacement columns" <| - table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] - col = table.at "x" - patterns = table.at "patterns" - replacements = table.at "replacements" - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" - col.replace patterns replacements . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO', 'hey A[bCd] hey', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col.replace patterns replacements only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col.replace patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - sqlite_spec connection prefix = name_counter = Ref.new 0 table_builder columns = From 1c2384699e6e2d3ae5772b342e8f3085321f08ce Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 18 Jul 2023 10:36:47 -0400 Subject: [PATCH 30/36] redundant supported lists --- .../Internal/Postgres/Postgres_Dialect.enso | 5 +- .../src/Internal/SQLite/SQLite_Dialect.enso | 2 +- .../Column_Operations_Spec.enso | 5 +- .../src/Common_Table_Operations/Main.enso | 6 ++- .../src/Database/Postgres_Spec.enso | 14 +++++- .../Table_Tests/src/Database/SQLite_Spec.enso | 46 ++++--------------- 6 files changed, 34 insertions(+), 44 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 554c222e8844..65854c96857f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -507,9 +507,8 @@ supported_replace_params = e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] - e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Default True, Replace_Params.Value Column Case_Sensitivity.Sensitive False] - e5 = [Replace_Params.Value Column Case_Sensitivity.Sensitive True, Replace_Params.Value Column Case_Sensitivity.Insensitive False, Replace_Params.Value Column Case_Sensitivity.Insensitive True] - Set.from_vector <| (e0 + e1 + e2 + e3 + e4 + e5).map x-> [x, 0] + e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Sensitive False] + Set.from_vector <| e0 + e1 + e2 + e3 + e4 replace : Vector Builder -> Any -> Builder replace args metadata = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index e5412fc8e2a0..1f1ab8400dc8 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -416,7 +416,7 @@ mod_op = Base_Generator.lift_binary_op "mod" x-> y-> supported_replace_params : Set Replace_Params supported_replace_params = e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] - Set.from_vector <| e.map x-> [x, 0] + Set.from_vector e replace : Vector Builder -> Any -> Builder replace args metadata = diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 63c10882b4d8..ce298f2d7566 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1046,7 +1046,10 @@ spec setup = True -> input_type = Meta.type_of term params = Replace_Params.Value input_type case_sensitivity only_first - case column.connection.dialect.are_replace_params_supported params of + supported_replace_params = setup.test_selection.supported_replace_params + supported_replace_params . should_be_a Set + are_params_supported = supported_replace_params.contains params + case are_params_supported of True -> column.replace term new_text case_sensitivity only_first . to_vector . should_equal expected False -> column.replace term new_text case_sensitivity only_first . should_fail_with Unsupported_Database_Operation False -> diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index c3ad6f7a7183..ccf74c82b8c9 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -1,5 +1,7 @@ from Standard.Base import all +import Standard.Database.Internal.Replace_Params.Replace_Params + import project.Common_Table_Operations.Add_Row_Number_Spec import project.Common_Table_Operations.Aggregate_Spec import project.Common_Table_Operations.Column_Operations_Spec @@ -97,7 +99,9 @@ type Test_Selection `Duration`/`Period` type. - supports_nanoseconds_in_time: Specifies if the backend supports nanosecond precision in time values. - Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False take_drop=True allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True distinct_returns_first_row_from_group_if_ordered=True date_time=True fixed_length_text_columns=False supports_decimal_type=False supports_time_duration=False supports_nanoseconds_in_time=False + - supported_replace_params: Specifies the possible values of + Replace_Params that a backend supports. + Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False take_drop=True allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True distinct_returns_first_row_from_group_if_ordered=True date_time=True fixed_length_text_columns=False supports_decimal_type=False supports_time_duration=False supports_nanoseconds_in_time=False supported_replace_params=Nothing spec setup = Core_Spec.spec setup diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 103834d9892f..60acfd0f8882 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -6,8 +6,10 @@ import Standard.Table.Data.Type.Value_Type.Bits from Standard.Table import Table, Value_Type from Standard.Table.Data.Aggregate_Column.Aggregate_Column import all hiding First, Last +import Standard.Database.Data.Column.Column import Standard.Database.Data.SQL_Type.SQL_Type import Standard.Database.Internal.Postgres.Pgpass +import Standard.Database.Internal.Replace_Params.Replace_Params from Standard.Database import all from Standard.Database.Errors import SQL_Error, Unsupported_Database_Operation @@ -276,7 +278,7 @@ run_tests connection db_name = Common_Spec.spec prefix connection - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True take_drop=False allows_mixed_type_comparisons=False fixed_length_text_columns=True supports_decimal_type=True + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True take_drop=False allows_mixed_type_comparisons=False fixed_length_text_columns=True supports_decimal_type=True supported_replace_params=supported_replace_params aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config first_last_row_order=False aggregation_problems=False agg_in_memory_table = (enso_project.data / "data.csv") . read agg_table = agg_in_memory_table.select_into_database_table connection (Name_Generator.random_name "Agg1") primary_key=Nothing temporary=True @@ -288,6 +290,16 @@ run_tests connection db_name = postgres_specific_spec connection db_name setup Common_Table_Operations.Main.spec setup +## PRIVATE +supported_replace_params : Set Replace_Params +supported_replace_params = + e0 = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive False] + e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] + e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] + e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Sensitive False] + Set.from_vector <| e0 + e1 + e2 + e3 + e4 + table_spec = db_name = Environment.get "ENSO_DATABASE_TEST_DB_NAME" db_host_port = (Environment.get "ENSO_DATABASE_TEST_HOST").if_nothing "localhost" . split ':' diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index d6a98d820e80..429254162ed3 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -5,6 +5,8 @@ import Standard.Base.Errors.File_Error.File_Error import Standard.Table.Data.Type.Value_Type.Bits from Standard.Table import Table, Value_Type +import Standard.Database.Data.Column.Column +import Standard.Database.Internal.Replace_Params.Replace_Params from Standard.Database import all from Standard.Database.Errors import SQL_Error, Unsupported_Database_Operation @@ -190,42 +192,6 @@ sqlite_specific_spec prefix connection setup = do_round 231 . should_be_a Decimal do_round 231 -1 . should_be_a Decimal - Test.group prefix+"replace: literal pattern and replacement" <| - col0 = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO']]] . at "x" - col1 = table_builder [["x", ['a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']]] . at "x" - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=false" - col0.replace 'hello' 'bye' . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Default . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Sensitive . to_vector . should_equal ['bye Hello', 'bye bye', 'HELLO HELLO'] - - col1.replace 'a[bcd]' 'hey' . to_vector . should_equal ['hey A[bCd] hey', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=false only_first=true" - col0.replace 'hello' 'bye' only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'HELLO HELLO'] - - col1.replace 'a[bcd]' 'hey' only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" - col0.replace 'hello' 'bye' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['bye Hello', 'bye hello', 'bye HELLO'] - - col1.replace 'a[bcd]' 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_equal ['hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] - - Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=sensitive/default use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=insensitive use_regex=True only_first=false" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive . to_vector . should_fail_with Unsupported_Database_Operation - - Test.specify "case_sensitivity=insensitive use_regex=True only_first=true" - col1.replace 'a[bcd]'.to_regex 'hey' case_sensitivity=Case_Sensitivity.Insensitive only_first=True . to_vector . should_fail_with Unsupported_Database_Operation - sqlite_spec connection prefix = name_counter = Ref.new 0 table_builder columns = @@ -239,7 +205,7 @@ sqlite_spec connection prefix = Common_Spec.spec prefix connection - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=False order_by=True natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True take_drop=False is_nan_and_nothing_distinct=False date_time=False + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=False order_by=True natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True take_drop=False is_nan_and_nothing_distinct=False date_time=False supported_replace_params=supported_replace_params ## For now `advanced_stats`, `first_last`, `text_shortest_longest` and `multi_distinct` remain disabled, because SQLite does not provide the @@ -260,6 +226,12 @@ sqlite_spec connection prefix = connection.close +## PRIVATE +supported_replace_params : Set Replace_Params +supported_replace_params = + e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + Set.from_vector e + spec = enso_project.data.create_directory file = enso_project.data / "transient" / "sqlite_test.db" From b8a0d71355363a0fcddff17fc5c8210900b19fad Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 18 Jul 2023 11:24:46 -0400 Subject: [PATCH 31/36] unused import --- .../Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso | 1 - 1 file changed, 1 deletion(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso index 597a36c4f172..1456f25bade8 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Replace_Params.enso @@ -1,6 +1,5 @@ import Standard.Base.Data.Boolean.Boolean import Standard.Base.Error.Error -import Standard.Base.Meta.Type import Standard.Base.Nothing.Nothing import Standard.Base.Data.Text.Case_Sensitivity.Case_Sensitivity From 75f710a22171f474fd11b5ad486de50b845a088f Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Tue, 18 Jul 2023 14:51:45 -0400 Subject: [PATCH 32/36] indentation --- .../src/Internal/Postgres/Postgres_Dialect.enso | 12 ++++++------ .../src/Internal/SQLite/SQLite_Dialect.enso | 4 ++-- test/Table_Tests/src/Database/Postgres_Spec.enso | 12 ++++++------ test/Table_Tests/src/Database/SQLite_Spec.enso | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 65854c96857f..7604f0defff4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -503,12 +503,12 @@ decimal_mod = Base_Generator.lift_binary_op "DECIMAL_MOD" x-> y-> ## PRIVATE supported_replace_params : Set Replace_Params supported_replace_params = - e0 = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive False] - e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] - e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] - e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] - e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Sensitive False] - Set.from_vector <| e0 + e1 + e2 + e3 + e4 + e0 = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive False] + e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] + e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] + e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Sensitive False] + Set.from_vector <| e0 + e1 + e2 + e3 + e4 replace : Vector Builder -> Any -> Builder replace args metadata = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 1f1ab8400dc8..70cc75829bb9 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -415,8 +415,8 @@ mod_op = Base_Generator.lift_binary_op "mod" x-> y-> ## PRIVATE supported_replace_params : Set Replace_Params supported_replace_params = - e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] - Set.from_vector e + e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + Set.from_vector e replace : Vector Builder -> Any -> Builder replace args metadata = diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 60acfd0f8882..ca2c2c40aa8d 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -293,12 +293,12 @@ run_tests connection db_name = ## PRIVATE supported_replace_params : Set Replace_Params supported_replace_params = - e0 = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive False] - e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] - e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] - e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] - e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Sensitive False] - Set.from_vector <| e0 + e1 + e2 + e3 + e4 + e0 = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive False] + e1 = [Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive False, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + e2 = [Replace_Params.Value Regex Case_Sensitivity.Default False, Replace_Params.Value Regex Case_Sensitivity.Default True, Replace_Params.Value Regex Case_Sensitivity.Sensitive False] + e3 = [Replace_Params.Value Regex Case_Sensitivity.Sensitive True, Replace_Params.Value Regex Case_Sensitivity.Insensitive False, Replace_Params.Value Regex Case_Sensitivity.Insensitive True] + e4 = [Replace_Params.Value Column Case_Sensitivity.Default False, Replace_Params.Value Column Case_Sensitivity.Sensitive False] + Set.from_vector <| e0 + e1 + e2 + e3 + e4 table_spec = db_name = Environment.get "ENSO_DATABASE_TEST_DB_NAME" diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 429254162ed3..4ed125c16a12 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -229,8 +229,8 @@ sqlite_spec connection prefix = ## PRIVATE supported_replace_params : Set Replace_Params supported_replace_params = - e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] - Set.from_vector e + e = [Replace_Params.Value Text Case_Sensitivity.Default False, Replace_Params.Value Text Case_Sensitivity.Sensitive False, Replace_Params.Value Text Case_Sensitivity.Default True, Replace_Params.Value Text Case_Sensitivity.Sensitive True, Replace_Params.Value Text Case_Sensitivity.Insensitive True] + Set.from_vector e spec = enso_project.data.create_directory From 72466225ae9b7ba76515871d06bf0b98a51aaf12 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 19 Jul 2023 13:48:27 -0400 Subject: [PATCH 33/36] name --- .../Database/0.0.0-dev/src/Data/Column.enso | 3 ++- .../Column_Operations_Spec.enso | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index 1453accad14f..cc1ab083fdf2 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1186,7 +1186,8 @@ type Column raw_term = case term of _ : Regex -> term.pattern_string _ -> term - self.make_op "REPLACE" [raw_term, new_text] "new name" [term, params] + new_name = self.naming_helpers.function_name "replace" [self, raw_term, new_text] + self.make_op "REPLACE" [raw_term, new_text] new_name [term, params] ## Gets the year as a number from the date stored in the column. diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index ce298f2d7566..fde69eda9482 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -3,6 +3,7 @@ from Standard.Base import all import Standard.Base.Errors.Common.Arithmetic_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Meta.Type +import Standard.Database.Data.Column.Column import Standard.Database.Internal.Replace_Params.Replace_Params from Standard.Table import Value_Type @@ -1123,6 +1124,20 @@ spec setup = locale = Locale.new "en" "GB" "UTF-8" col.replace 'asdf' 'zxcv' case_sensitivity=(Case_Sensitivity.Insensitive locale) . should_fail_with Illegal_Argument + Test.specify "column name" <| + table = table_builder [["x", ['hello Hello', 'hello hello', 'HELLO HELLO', 'a[bcd] A[bCd] a[bcd]', 'abac ad Ab aCAd']], ["patterns", ['hello', 'hello', 'hello', 'a[bcd]', 'a[bcd]']], ["replacements", ['bye', 'bye', 'bye', 'hey', 'hey']]] + col = table.at "x" + patterns = table.at "patterns" + replacements = table.at "replacements" + + supported_replace_params = setup.test_selection.supported_replace_params + if supported_replace_params.contains (Replace_Params.Value Text Case_Sensitivity.Default False) then + col.replace 'hello' 'bye' . name . should_equal 'replace([x], \'hello\', \'bye\')' + if supported_replace_params.contains (Replace_Params.Value Regex Case_Sensitivity.Default False) then + col.replace 'a[bcd]'.to_regex 'hey' . name . should_equal 'replace([x], \'a[bcd]\', \'hey\')' + if supported_replace_params.contains (Replace_Params.Value Column Case_Sensitivity.Default False) then + col.replace patterns replacements . name . should_equal 'replace([x], [patterns], [replacements])' + Test.group prefix+"Column Operations - Text Replace (in-memory only)" <| if setup.is_database.not then t4 = table_builder [["A", ["Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot"]], ["B", ["A","O","a","E","o","O"]], ["C", [1,2,3,4,5,6]], ["D", ['',Nothing,'',Nothing,'','']]] From 36a431ce7968ad00d1b9e16447c17adb1a2aea5e Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 19 Jul 2023 14:02:26 -0400 Subject: [PATCH 34/36] if_replace_params_supports --- .../Database/0.0.0-dev/src/Data/Column.enso | 14 ++++++-------- .../Database/0.0.0-dev/src/Data/Dialect.enso | 6 +++--- .../src/Internal/Postgres/Postgres_Dialect.enso | 5 +++-- .../src/Internal/SQLite/SQLite_Dialect.enso | 5 +++-- .../Column_Operations_Spec.enso | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso index cc1ab083fdf2..7c722eb19106 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso @@ -1180,14 +1180,12 @@ type Column Value_Type.expect_text self <| case_sensitivity.disallow_non_default_locale <| input_type = Meta.type_of term params = Replace_Params.Value input_type case_sensitivity only_first - case self.connection.dialect.are_replace_params_supported params of - False -> params.throw_unsupported - True -> - raw_term = case term of - _ : Regex -> term.pattern_string - _ -> term - new_name = self.naming_helpers.function_name "replace" [self, raw_term, new_text] - self.make_op "REPLACE" [raw_term, new_text] new_name [term, params] + self.connection.dialect.if_replace_params_supports params <| + raw_term = case term of + _ : Regex -> term.pattern_string + _ -> term + new_name = self.naming_helpers.function_name "replace" [self, raw_term, new_text] + self.make_op "REPLACE" [raw_term, new_text] new_name [term, params] ## Gets the year as a number from the date stored in the column. diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso index 4de23ab8e662..524731be16f0 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso @@ -234,9 +234,9 @@ type Dialect ## PRVIATE Returns true if the `replace` parameters are supported by this backend. - are_replace_params_supported : Replace_Params -> Boolean - are_replace_params_supported self replace_params = - _ = [replace_params] + if_replace_params_supports : Replace_Params -> Any -> Any + if_replace_params_supports self replace_params ~action = + _ = [replace_params, action] Unimplemented.throw "This is an interface only." ## PRIVATE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index 7604f0defff4..3513d93e4673 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -244,8 +244,9 @@ type Postgres_Dialect ## PRVIATE Returns true if the `replace` parameters are supported by this backend. - are_replace_params_supported : Replace_Params -> Boolean - are_replace_params_supported self replace_params = supported_replace_params.contains replace_params + if_replace_params_supports : Replace_Params -> Any -> Any + if_replace_params_supports self replace_params ~action = + if supported_replace_params.contains replace_params then action else replace_params.throw_unsupported ## PRIVATE make_internal_generator_dialect = diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso index 70cc75829bb9..1ff1affbccec 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso @@ -269,8 +269,9 @@ type SQLite_Dialect ## PRVIATE Returns true if the `replace` parameters are suppoerted by this backend. - are_replace_params_supported : Replace_Params -> Boolean - are_replace_params_supported self replace_params = supported_replace_params.contains replace_params + if_replace_params_supports : Replace_Params -> Any -> Any + if_replace_params_supports self replace_params ~action = + if supported_replace_params.contains replace_params then action else replace_params.throw_unsupported ## PRIVATE make_internal_generator_dialect = diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index fde69eda9482..565ac5836bc8 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1041,7 +1041,7 @@ spec setup = op c 1 . should_fail_with Invalid_Value_Type op a True . should_fail_with Invalid_Value_Type - Test.group prefix+"Column Operations - Text Replace" <| + Test.group prefix+"Column Operations - Text Replaceasdfasdf" <| do_replace column term new_text case_sensitivity=Case_Sensitivity.Default only_first=False expected = case setup.is_database of True -> From 6a27f70330a418ede0865ce90ad226000c013df3 Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Wed, 19 Jul 2023 14:03:18 -0400 Subject: [PATCH 35/36] cleanup --- .../src/Common_Table_Operations/Column_Operations_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 565ac5836bc8..fde69eda9482 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1041,7 +1041,7 @@ spec setup = op c 1 . should_fail_with Invalid_Value_Type op a True . should_fail_with Invalid_Value_Type - Test.group prefix+"Column Operations - Text Replaceasdfasdf" <| + Test.group prefix+"Column Operations - Text Replace" <| do_replace column term new_text case_sensitivity=Case_Sensitivity.Default only_first=False expected = case setup.is_database of True -> From 7a4bf24752b2cecaa9bd9dee931421cc573b2dee Mon Sep 17 00:00:00 2001 From: Gregory Travis Date: Mon, 24 Jul 2023 16:28:30 -0400 Subject: [PATCH 36/36] fix test --- .../src/Common_Table_Operations/Column_Operations_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index fde69eda9482..2a5094c6e95a 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -1113,7 +1113,7 @@ spec setup = do_replace col patterns replacements only_first=True expected=['bye Hello', 'bye hello', 'HELLO HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=false" - do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Insensitive expected=['bye bye', 'bye bye', 'HELLO bye', 'hey hey hey', 'abac ad Ab aCAd'] + do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Insensitive expected=['bye bye', 'bye bye', 'bye bye', 'hey hey hey', 'abac ad Ab aCAd'] Test.specify "case_sensitivity=insensitive use_regex=false only_first=true" do_replace col patterns replacements case_sensitivity=Case_Sensitivity.Insensitive only_first=True expected=['bye Hello', 'bye hello', 'bye HELLO', 'hey A[bCd] a[bcd]', 'abac ad Ab aCAd']