diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e923c9ccb7c..fa10d80b5bf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,7 @@ representing `Delimited` files without storing them on the filesystem.][3478] - [Added rank data, correlation and covariance statistics for `Vector`][3484] - [Implemented `Table.order_by` for the SQLite backend.][3502] +- [Implemented `Table.order_by` for the PostgreSQL backend.][3514] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -208,6 +209,7 @@ [3478]: https://github.com/enso-org/enso/pull/3478 [3484]: https://github.com/enso-org/enso/pull/3484 [3502]: https://github.com/enso-org/enso/pull/3502 +[3514]: https://github.com/enso-org/enso/pull/3514 #### Enso Compiler diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Text_Ordering.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Text_Ordering.enso index 112ef0aab534..1ea26a45e230 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Text_Ordering.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Text_Ordering.enso @@ -5,5 +5,8 @@ from Standard.Base import all Arguments: - sort_digits_as_numbers: Sort digits in the text as numbers. Setting this to `True` results in a "Natural" ordering. - - case_sensitive: Specifies if the ordering should be case case sensitive. -type Text_Ordering (sort_digits_as_numbers:Boolean=False) (case_sensitive:(True|Case_Insensitive)=True) + - case_sensitive: Specifies if the ordering should be case case sensitive. If + set to `Nothing` (the default), it chooses the default ordering for a given + backend. For the In-memory backend, the default ordering is case sensitive. + In databases, the default ordering depends on the database configuration. +type Text_Ordering (sort_digits_as_numbers:Boolean=False) (case_sensitive:(Nothing|True|Case_Insensitive)=Nothing) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso index bab6865dc3b5..2c9bb1c1250f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect.enso @@ -45,7 +45,7 @@ type Dialect One of the purposes of this method is to verify if the expected ordering settings are supported by the given database backend. - prepare_order_descriptor : Connection -> IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor + prepare_order_descriptor : IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor prepare_order_descriptor = Errors.unimplemented "This is an interface only." ## PRIVATE diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Postgres.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Postgres.enso index a58611e4635f..420f1798c4d1 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Postgres.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Postgres.enso @@ -6,6 +6,8 @@ from Standard.Database.Data.Sql import Sql_Type import Standard.Database.Data.Dialect import Standard.Database.Data.Dialect.Helpers import Standard.Database.Data.Internal.Base_Generator +import Standard.Database.Data.Internal.IR +from Standard.Database.Error as Database_Errors import Unsupported_Database_Operation_Error ## PRIVATE @@ -49,14 +51,14 @@ type Postgresql_Dialect One of the purposes of this method is to verify if the expected ordering settings are supported by the given database backend. - prepare_order_descriptor : Connection -> IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor - prepare_order_descriptor connection internal_column sort_direction text_ordering = - _ = [connection, internal_column, sort_direction, text_ordering] - Errors.unimplemented "TODO" + prepare_order_descriptor : IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor + prepare_order_descriptor internal_column sort_direction text_ordering = + here.make_order_descriptor internal_column sort_direction text_ordering ## PRIVATE make_internal_generator_dialect = - text = [here.starts_with, here.contains, here.ends_with, here.agg_shortest, here.agg_longest]+here.concat_ops + cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]] + text = [here.starts_with, here.contains, here.ends_with, here.agg_shortest, here.agg_longest]+here.concat_ops+cases counts = [here.agg_count_is_null, here.agg_count_empty, here.agg_count_not_empty, ["COUNT_DISTINCT", here.agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", here.agg_count_distinct_include_null]] stddev_pop = ["STDDEV_POP", Base_Generator.make_function "stddev_pop"] @@ -218,3 +220,27 @@ make_contains_expr expr substring = ## PRIVATE contains = Base_Generator.lift_binary_op "contains" here.make_contains_expr + +## PRIVATE +make_order_descriptor internal_column sort_direction text_ordering = + nulls = case sort_direction of + Sort_Direction.Ascending -> IR.Nulls_First + Sort_Direction.Descending -> IR.Nulls_Last + case internal_column.sql_type.is_likely_text of + True -> + ## In the future we can modify this error to suggest using a custom defined collation. + if text_ordering.sort_digits_as_numbers then Error.throw (Unsupported_Database_Operation_Error "Natural ordering is currently not supported. You may need to materialize the Table to perform this operation.") else + case text_ordering.case_sensitive of + Nothing -> + IR.Order_Descriptor internal_column.expression sort_direction nulls_order=nulls collation=Nothing + True -> + IR.Order_Descriptor internal_column.expression sort_direction nulls_order=nulls collation="ucs_basic" + Case_Insensitive locale -> case Locale.default.java_locale.equals locale.java_locale of + False -> + Error.throw (Unsupported_Database_Operation_Error "Case insensitive ordering with custom locale is currently not supported. You may need to materialize the Table to perform this operation.") + True -> + upper = IR.Operation "UPPER" [internal_column.expression] + folded_expression = IR.Operation "LOWER" [upper] + IR.Order_Descriptor folded_expression sort_direction nulls_order=nulls collation=Nothing + False -> + IR.Order_Descriptor internal_column.expression sort_direction nulls_order=nulls collation=Nothing diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Redshift.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Redshift.enso index a6ffdc3fbe10..2c21f365a1ef 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Redshift.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Redshift.enso @@ -49,7 +49,6 @@ type Redshift_Dialect One of the purposes of this method is to verify if the expected ordering settings are supported by the given database backend. - prepare_order_descriptor : Connection -> IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor - prepare_order_descriptor connection internal_column sort_direction text_ordering = - _ = [connection, internal_column, sort_direction, text_ordering] - Errors.unimplemented "TODO" \ No newline at end of file + prepare_order_descriptor : IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor + prepare_order_descriptor internal_column sort_direction text_ordering = + Postgres.make_order_descriptor internal_column sort_direction text_ordering diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Sqlite.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Sqlite.enso index 0e2d484837a3..5316c962fe8e 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Sqlite.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Dialect/Sqlite.enso @@ -49,12 +49,20 @@ type Sqlite_Dialect One of the purposes of this method is to verify if the expected ordering settings are supported by the given database backend. - prepare_order_descriptor : Connection -> IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor - prepare_order_descriptor _ internal_column sort_direction text_ordering = case internal_column.sql_type.is_likely_text of + prepare_order_descriptor : IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor + prepare_order_descriptor internal_column sort_direction text_ordering = case internal_column.sql_type.is_likely_text of True -> if text_ordering.sort_digits_as_numbers then Error.throw (Unsupported_Database_Operation_Error "Natural ordering is not supported by the SQLite backend. You may need to materialize the Table to perform this operation.") else - if text_ordering.case_sensitive != True then Error.throw (Unsupported_Database_Operation_Error "Case insensitive ordering is not supported by the SQLite backend. You may need to materialize the Table to perform this operation.") else - IR.Order_Descriptor internal_column.expression sort_direction collation=Nothing + case text_ordering.case_sensitive of + Nothing -> + IR.Order_Descriptor internal_column.expression sort_direction collation=Nothing + True -> + IR.Order_Descriptor internal_column.expression sort_direction collation="BINARY" + Case_Insensitive locale -> case Locale.default.java_locale.equals locale.java_locale of + False -> + Error.throw (Unsupported_Database_Operation_Error "Case insensitive ordering with custom locale is not supported by the SQLite backend. You may need to materialize the Table to perform this operation.") + True -> + IR.Order_Descriptor internal_column.expression sort_direction collation="NOCASE" False -> IR.Order_Descriptor internal_column.expression sort_direction collation=Nothing diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso index e23d51604a05..8329515bc009 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Data/Table.enso @@ -465,6 +465,8 @@ type Table - If values do not implement an ordering, an `Incomparable_Values_Error`. + Missing (`Nothing`) values are sorted as less than any other object. + > Example Order the table by the column "alpha" in ascending order. @@ -486,7 +488,7 @@ type Table propagate dataflow errors correctly. See: https://www.pivotaltracker.com/story/show/181057718 Panic.throw_wrapped_if_error <| - this.connection.dialect.prepare_order_descriptor this.connection internal_column associated_selector.direction text_ordering + this.connection.dialect.prepare_order_descriptor internal_column associated_selector.direction text_ordering new_ctx = this.context.add_orders new_order_descriptors this.updated_context new_ctx diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Helpers.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Helpers.enso index 3cf1a7298e73..44134e741ded 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Helpers.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Helpers.enso @@ -206,7 +206,8 @@ rename_columns internal_columns mapping on_problems = - text_ordering: The sort methodology to use. sort_columns : Vector -> Sort_Direction -> Text_Ordering -> Vector sort_columns internal_columns direction text_ordering = - mapper = case text_ordering.case_sensitive of + case_sensitive = text_ordering.case_sensitive.if_nothing True + mapper = case case_sensitive of True -> _.name Case_Insensitive locale -> col -> col.name.to_case_insensitive_key locale=locale diff --git a/test/Table_Tests/src/Common_Table_Spec.enso b/test/Table_Tests/src/Common_Table_Spec.enso index 2dba769d8bf4..3147a51efabc 100644 --- a/test/Table_Tests/src/Common_Table_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Spec.enso @@ -11,7 +11,7 @@ from Standard.Table.Data.Position as Position_Module import all import Standard.Table.Data.Sort_Column_Selector import Standard.Table.Data.Sort_Column -type Test_Selection supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=False order_by_unicode_normalization_by_default=False +type Test_Selection supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False ## A common test suite for shared operations on the Table API. @@ -527,14 +527,15 @@ spec prefix table_builder test_selection pending=Nothing = col7 = ["psi", [Nothing, "c01", "c10", "C2"]] col8 = ["phi", ["śc", Nothing, 's\u0301b', "śa"]] col9 = ["tau", [32.0, 0.5, -0.1, 1.6]] - table_builder [col1, col2, col3, col4, col5, col6, col7, col8, col9] + col10 = ["rho", ["BB", Nothing, Nothing, "B"]] + table_builder [col1, col2, col3, col4, col5, col6, col7, col8, col9, col10] Test.specify "should work as shown in the doc examples" <| t1 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "alpha"]) t1.at "alpha" . to_vector . should_equal [0, 1, 2, 3] t1.at "gamma" . to_vector . should_equal [4, 3, 2, 1] - t2 = table.order_by (Sort_Column_Selector.By_Index [Sort_Column.Index 1, Sort_Column.Index -7 Sort_Direction.Descending]) + t2 = table.order_by (Sort_Column_Selector.By_Index [Sort_Column.Index 1, Sort_Column.Index -8 Sort_Direction.Descending]) t2.at "beta" . to_vector . should_equal ["a", "a", "b", "b"] t2.at "gamma" . to_vector . should_equal [3, 1, 4, 2] t2.at "alpha" . to_vector . should_equal [1, 3, 0, 2] @@ -564,13 +565,13 @@ spec prefix table_builder test_selection pending=Nothing = Problems.test_problem_handling action problems tester Test.specify "should correctly handle problems: aliased indices" <| - selector = Sort_Column_Selector.By_Index [Sort_Column.Index 1, Sort_Column.Index -8 Sort_Direction.Descending, Sort_Column.Index -7 Sort_Direction.Descending, Sort_Column.Index 2 Sort_Direction.Ascending] + selector = Sort_Column_Selector.By_Index [Sort_Column.Index 1, Sort_Column.Index -9 Sort_Direction.Descending, Sort_Column.Index -8 Sort_Direction.Descending, Sort_Column.Index 2 Sort_Direction.Ascending] action = table.order_by selector on_problems=_ tester table = table.at "beta" . to_vector . should_equal ["a", "a", "b", "b"] table.at "gamma" . to_vector . should_equal [3, 1, 4, 2] table.at "alpha" . to_vector . should_equal [1, 3, 0, 2] - problems = [Input_Indices_Already_Matched [Sort_Column.Index -8 Sort_Direction.Descending, Sort_Column.Index 2]] + problems = [Input_Indices_Already_Matched [Sort_Column.Index -9 Sort_Direction.Descending, Sort_Column.Index 2]] Problems.test_problem_handling action problems tester Test.specify "should correctly handle problems: duplicate names" <| @@ -679,9 +680,11 @@ spec prefix table_builder test_selection pending=Nothing = t1.at "xi" . to_vector . should_equal [Nothing, 0.5, 1.0, 1.5] t1.at "alpha" . to_vector . should_equal [1, 0, 3, 2] - t2 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "psi"]) - t2.at "psi" . to_vector . should_equal [Nothing, "C2", "c01", "c10"] - t2.at "alpha" . to_vector . should_equal [3, 0, 2, 1] + t2 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "rho"]) + t2.at "rho" . to_vector . should_equal [Nothing, Nothing, "B", "BB"] + + t3 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "rho" Sort_Direction.Descending]) + t3.at "rho" . to_vector . should_equal ["BB", "B", Nothing, Nothing] Test.specify "should behave as expected with Unicode normalization, depending on the defaults settings" <| t1 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "phi"]) @@ -702,16 +705,21 @@ spec prefix table_builder test_selection pending=Nothing = t2.at "delta" . to_vector . should_equal ["a03", "a1", "a10", "a2"] t2.at "alpha" . to_vector . should_equal [0, 2, 3, 1] - # TODO [RW] This test must actually be verified in practice. Test.specify "should support case insensitive ordering" pending=(if test_selection.case_insensitive_ordering.not then "Case insensitive ordering is not supported.") <| t1 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "eta"]) text_ordering=(Text_Ordering case_sensitive=Case_Insensitive) - t1.at "eta" . to_vector . should_equal ["Aleph", "alpha", "Beta", "bądź"] + expected = case test_selection.case_insensitive_ascii_only of + True -> ["Aleph", "alpha", "Beta", "bądź"] + False -> ["Aleph", "alpha", "bądź", "Beta"] + t1.at "eta" . to_vector . should_equal expected + + t2 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "eta"]) text_ordering=(Text_Ordering case_sensitive=True) + t2.at "eta" . to_vector . should_equal ["Aleph", "Beta", "alpha", "bądź"] - t2 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "eta"]) text_ordering=(Text_Ordering case_sensitive=Case_Insensitive (Locale.new "pl" "PL")) - t2.at "eta" . to_vector . should_equal ["Aleph", "alpha", "bądź", "Beta"] + t3 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "psi"]) text_ordering=(Text_Ordering case_sensitive=Case_Insensitive) + t3.at "psi" . to_vector . should_equal [Nothing, "c01", "c10", "C2"] - t3 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "eta"]) - t3.at "eta" . to_vector . should_equal ["Aleph", "Beta", "alpha", "bądź"] + t4 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "psi" Sort_Direction.Descending]) text_ordering=(Text_Ordering case_sensitive=True) + t4.at "psi" . to_vector . should_equal ["c10", "c01", "C2", Nothing] Test.specify "should support natural and case insensitive ordering at the same time" pending=(if (test_selection.natural_ordering.not || test_selection.case_insensitive_ordering.not) then "Natural ordering or case sensitive ordering is not supported.") <| t1 = table.order_by (Sort_Column_Selector.By_Name [Sort_Column.Name "psi"]) text_ordering=(Text_Ordering sort_digits_as_numbers=True case_sensitive=Case_Insensitive) diff --git a/test/Table_Tests/src/Database/Postgresql_Spec.enso b/test/Table_Tests/src/Database/Postgresql_Spec.enso index 886b90bd82cc..fff8991e960d 100644 --- a/test/Table_Tests/src/Database/Postgresql_Spec.enso +++ b/test/Table_Tests/src/Database/Postgresql_Spec.enso @@ -99,7 +99,7 @@ run_tests connection pending=Nothing = Common_Spec.spec prefix connection pending=pending here.postgres_specific_spec connection pending=pending - common_selection = Common_Table_Spec.Test_Selection supports_case_sensitive_columns=True order_by=False + common_selection = Common_Table_Spec.Test_Selection supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True Common_Table_Spec.spec prefix table_builder test_selection=common_selection pending=pending selection = Aggregate_Spec.Test_Selection first_last_row_order=False aggregation_problems=False diff --git a/test/Table_Tests/src/Database/Sqlite_Spec.enso b/test/Table_Tests/src/Database/Sqlite_Spec.enso index d2114081952a..1469c1634aa4 100644 --- a/test/Table_Tests/src/Database/Sqlite_Spec.enso +++ b/test/Table_Tests/src/Database/Sqlite_Spec.enso @@ -62,7 +62,7 @@ spec = Common_Spec.spec prefix connection here.sqlite_specific_spec connection - common_selection = Common_Table_Spec.Test_Selection supports_case_sensitive_columns=False order_by=True natural_ordering=False case_insensitive_ordering=False + common_selection = Common_Table_Spec.Test_Selection supports_case_sensitive_columns=False order_by=True natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True Common_Table_Spec.spec prefix table_builder test_selection=common_selection ## For now `advanced_stats`, `first_last`, `text_shortest_longest` and diff --git a/test/Table_Tests/src/Table_Spec.enso b/test/Table_Tests/src/Table_Spec.enso index 79b1479620ea..7daeb13ab16b 100644 --- a/test/Table_Tests/src/Table_Spec.enso +++ b/test/Table_Tests/src/Table_Spec.enso @@ -635,7 +635,7 @@ spec = t_3 = Table.new [c_3_1, c_3_2, c_3_3] t_3.default_visualization.should_equal Visualization.Id.table - selection = Common_Table_Spec.Test_Selection supports_case_sensitive_columns=True order_by=False + selection = Common_Table_Spec.Test_Selection supports_case_sensitive_columns=True order_by=False natural_ordering=True case_insensitive_ordering=True order_by_unicode_normalization_by_default=True Common_Table_Spec.spec "[In-Memory] " table_builder=Table.new test_selection=selection Test.group "Use First Row As Names" <|