PostgreSQL Support in Database Library (#1565)

Co-authored-by: Marcin Kostrzewa <[email protected]>

.github/workflows/scala.yml

@@ -211,6 +211,23 @@ jobs:
           gu install python
           gu install r
 
+      - name: Prepare configuration for the Database tests
+        shell: bash
+        if: runner.os == 'Linux'
+        run: |
+          echo "ENSO_DATABASE_TEST_HOST=127.0.0.1" >> $GITHUB_ENV
+          echo "ENSO_DATABASE_TEST_DB_NAME=enso_test_db" >> $GITHUB_ENV
+          echo "ENSO_DATABASE_TEST_DB_USER=enso_test_user" >> $GITHUB_ENV
+          echo "ENSO_DATABASE_TEST_DB_PASSWORD=enso_test_password" >> $GITHUB_ENV
+      - name: Configure PostgreSQL for the Database tests
+        uses: harmon758/postgresql-action@v1
+        if: runner.os == 'Linux'
+        with:
+          postgresql version: "latest"
+          postgresql db: ${{ env.ENSO_DATABASE_TEST_DB_NAME }}
+          postgresql user: ${{ env.ENSO_DATABASE_TEST_DB_USER }}
+          postgresql password: ${{ env.ENSO_DATABASE_TEST_DB_PASSWORD }}
+
       - name: Test Engine Distribution (Unix)
         shell: bash
         if: runner.os != 'Windows'

build.sbt

@@ -70,7 +70,7 @@ GatherLicenses.distributions := Seq(
     "Standard",
     file("distribution/std-lib/Standard/THIRD-PARTY"),
     Distribution.sbtProjects(`std-bits`)
-  ),
+  )
 )
 GatherLicenses.licenseConfigurations := Set("compile")
 GatherLicenses.configurationRoot := file("tools/legal-review")
@@ -1252,9 +1252,10 @@ lazy val `std-bits` = project
     Compile / packageBin / artifactPath :=
       `standard-polyglot-root` / "std-bits.jar",
     libraryDependencies ++= Seq(
-      "com.ibm.icu"   % "icu4j"             % icuVersion,
-      "com.univocity" % "univocity-parsers" % "2.9.0",
-      "org.xerial"    % "sqlite-jdbc"       % "3.34.0"
+      "com.ibm.icu"    % "icu4j"             % icuVersion,
+      "com.univocity"  % "univocity-parsers" % "2.9.0",
+      "org.xerial"     % "sqlite-jdbc"       % "3.34.0",
+      "org.postgresql" % "postgresql"        % "42.2.19"
     ),
     Compile / packageBin := Def.task {
       val result = (Compile / packageBin).value

distribution/std-lib/Standard/THIRD-PARTY/NOTICE

@@ -1,6 +1,11 @@
 Enso
 Copyright 2020 - 2021 New Byte Order sp. z o. o.
 
+'univocity-parsers', licensed under the Apache 2, is distributed with the Standard.
+The license file can be found at `licenses/APACHE2.0`.
+Copyright notices related to this dependency can be found in the directory `com.univocity.univocity-parsers-2.9.0`.
+
+
 'sqlite-jdbc', licensed under the The Apache Software License, Version 2.0, is distributed with the Standard.
 The license information can be found along with the copyright notices.
 Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.34.0`.
@@ -11,7 +16,12 @@ The license information can be found along with the copyright notices.
 Copyright notices related to this dependency can be found in the directory `com.ibm.icu.icu4j-67.1`.
 
 
-'univocity-parsers', licensed under the Apache 2, is distributed with the Standard.
-The license file can be found at `licenses/APACHE2.0`.
-Copyright notices related to this dependency can be found in the directory `com.univocity.univocity-parsers-2.9.0`.
+'checker-qual', licensed under the The MIT License, is distributed with the Standard.
+The license information can be found along with the copyright notices.
+Copyright notices related to this dependency can be found in the directory `org.checkerframework.checker-qual-3.5.0`.
+
+
+'postgresql', licensed under the BSD-2-Clause, is distributed with the Standard.
+The license information can be found along with the copyright notices.
+Copyright notices related to this dependency can be found in the directory `org.postgresql.postgresql-42.2.19`.
 

distribution/std-lib/Standard/THIRD-PARTY/org.checkerframework.checker-qual-3.5.0/LICENSE.txt

@@ -0,0 +1,22 @@
+Checker Framework qualifiers
+Copyright 2004-present by the Checker Framework developers
+
+MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

distribution/std-lib/Standard/THIRD-PARTY/org.postgresql.postgresql-42.2.19/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 1997, PostgreSQL Global Development Group
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

distribution/std-lib/Standard/THIRD-PARTY/org.postgresql.postgresql-42.2.19/NOTICES

@@ -0,0 +1,43 @@
+Copyright (c) 2015, PostgreSQL Global Development Group
+
+/*
+ * Copyright (c) 2003, PostgreSQL Global Development Group
+ * See the LICENSE file in the project root for more information.
+ */
+
+Copyright (c) 2006, PostgreSQL Global Development Group
+
+Copyright (c) 2009, PostgreSQL Global Development Group
+
+Copyright (c) 2014, PostgreSQL Global Development Group
+
+Copyright (c) 2017, PostgreSQL Global Development Group
+
+Copyright (c) 2007, PostgreSQL Global Development Group
+
+Copyright (c) 2019, PostgreSQL Global Development Group
+
+// Copyright (c) 2004, Open Cloud Limited.
+
+Copyright (c) 2016, PostgreSQL Global Development Group
+
+Copyright (c) 2020, PostgreSQL Global Development Group
+
+Copyright (c) 2004, PostgreSQL Global Development Group
+
+Copyright (c) 2018, PostgreSQL Global Development Group
+
+# Just Czech translation ;o)
+# Copyright (C) 2005
+# This file is distributed under the same license as the PACKAGE package.
+# Petr Dittrich <[email protected]>, 2005
+
+Copyright (c) 2008, PostgreSQL Global Development Group
+
+Copyright (c) 2005, PostgreSQL Global Development Group
+
+Copyright (c) 2021, PostgreSQL Global Development Group
+
+Copyright (c) 2011, PostgreSQL Global Development Group
+
+Copyright (c) 2012, PostgreSQL Global Development Group

distribution/std-lib/Standard/src/Base/Meta.enso

@@ -130,3 +130,20 @@ meta value = if Builtins.Meta.is_atom value then Atom value else
    This is a power-user feature, that is only useful for certain optimizations.
 is_same_object : Any -> Any -> Boolean
 is_same_object value_1 value_2 = Builtins.Meta.is_same_object value_1 value_2
+
+## PRIVATE
+
+   Returns a Text representing the source location of a stack frame above
+   the call.
+
+   Arguments:
+   - skip_frames: how many frames on the stack to skip.
+
+   If the function is called with 0 it will return the location of that call.
+   Note that not only function calls, but also things like pattern matching add
+   frames to the stack and there is no guarantee that the amount of frames that
+   need to be skipped will not change between versions, so this method should be
+   used carefully.
+get_source_location : Integer -> Text
+get_source_location skip_frames =
+    Builtins.Meta.get_source_location skip_frames+1

distribution/std-lib/Standard/src/Base/System/Environment.enso

@@ -0,0 +1,11 @@
+from Standard.Base import all
+
+polyglot java import java.lang.System
+
+## UNSTABLE
+
+   Returns a value of a specified environment variable or Nothing if such
+   variable is not defined.
+get : Text -> Text | Nothing
+get key =
+    System.getenv key

distribution/std-lib/Standard/src/Database/Connection/Connection.enso

@@ -9,13 +9,15 @@ import Standard.Table.Internal.Java_Exports
 
 from Standard.Database.Data.Sql import Sql_Type
 
+polyglot java import java.lang.UnsupportedOperationException
 polyglot java import java.util.ArrayList
+polyglot java import java.util.Properties
 polyglot java import java.sql.DriverManager
 polyglot java import java.sql.PreparedStatement
 polyglot java import java.sql.Types
-polyglot java import java.lang.UnsupportedOperationException
 polyglot java import java.sql.SQLException
 polyglot java import java.sql.SQLTimeoutException
+polyglot java import org.enso.database.JDBCProxy
 
 type Connection
 
@@ -54,13 +56,6 @@ type Connection
 
        Executes a raw query and returns the result as an in-memory Table.
 
-       Currently the SQL decimal type is mapped to a floating-point
-       representation which is not exact. While operations in the database
-       itself may (or may not, depending on the database engine) be done with
-       exact precision, the materialized values may loose precision, so extra
-       care is needed when working with decimal computations that need to be
-       exact.
-
        Arguments:
        - query: either raw SQL code as Text or an instance of Sql.Statement
          representing the query to execute.
@@ -150,15 +145,12 @@ type Connection
 
    Creates a builder for a column based on a provided SQL type, trying to infer
    the best type for the builder.
-
-   WARNING: Currently it coerces decimals into floating point numbers.
 create_builder : Sql_Type -> Builder
 create_builder sql_type =
     initial_size = 10
     if sql_type.is_definitely_boolean then Builder_Boolean (Java_Exports.make_bool_builder) else
         if sql_type.is_definitely_integer then Builder_Long (Java_Exports.make_long_builder initial_size) else
-            is_double =  sql_type.is_definitely_double || sql_type==Sql_Type.decimal
-            if is_double then Builder_Double (Java_Exports.make_double_builder initial_size) else
+            if sql_type.is_definitely_double then Builder_Double (Java_Exports.make_double_builder initial_size) else
                 Builder_Inferred (Java_Exports.make_inferred_builder initial_size)
 
 type Builder
@@ -219,14 +211,13 @@ type Unsupported_Dialect url
 
    Creates a JDBC connection based on a URL and optionally username and
    password.
-create_jdbc_connection : Text -> Nothing | Text -> Nothing | Text -> Connection
-create_jdbc_connection url user=Nothing password=Nothing = here.wrap_sql_errors <|
+create_jdbc_connection : Text -> Vector -> Connection
+create_jdbc_connection url properties = here.wrap_sql_errors <|
+    java_props = Properties.new
+    properties.each pair->
+        java_props.setProperty pair.first pair.second
     dialect = Dialect.supported_dialects.find (d -> url.starts_with "jdbc:"+d.name) . map_error (_ -> Unsupported_Dialect url)
-    java_connection = case user.is_nothing && password.is_nothing of
-        True ->
-            DriverManager.getConnection url
-        False ->
-            DriverManager.getConnection url user password
+    java_connection = JDBCProxy.getConnection url java_props
     resource = Managed_Resource.register java_connection here.close_connection
     Connection resource dialect
 

distribution/std-lib/Standard/src/Database/Connection/Database.enso

@@ -2,24 +2,42 @@ from Standard.Base import all
 
 from Standard.Database.Connection.Connection import all
 
-# TODO [RW] finalize Postgres support
 ## UNSTABLE
 
    Tries to connect to the database under a provided URL.
 
-   Currently only SQLite databases are supported.
+   Currently SQLite and PostgreSQL databases are supported.
 
    The exact URL depends on the database engine.
-   For SQLite it has format `sqlite:/path/to/database/file`.
+   For SQLite the expected format is `sqlite:/path/to/database/file`.
+   For PostgreSQL it can be one of:
+   - `postgresql:database_name` - which will connect to the database with the
+     given name on the local machine;
+   - `postgresql:/` - which will connect to the default database
+     (which is the same as the username) on the local machine;
+   - `postgresql://host/database_name` - which will connect to the specified
+     database on a specified host, the `host` can consist of an IP address or a
+     hostname, optionally followed by colon and a port number, so values like
+     `db.example.com`, `127.0.0.1`, `example.com:1234`, `127.0.0.1:1234` are
+     allowed;
+   - `postgresql://host/` - which will connect to the same database as the
+     username on a specified host, the `host`` is defined as above.
 
    Arguments:
    - url: the URL to connect to.
    - user: (optional) an username for authentication.
    - password: (optional) a password for authentication.
-connect : Text -> Nothing | Text -> Nothing | Text -> Connection ! Sql_Error
-connect url user=Nothing password=Nothing =
+   - custom_properties: (optional) a vector of key-value Text pairs which can
+     set any other properties that can be used to configure the connection or
+     for authentication. Supported properties depend on the database engine that
+     the connection is made to.
+connect : Text -> Nothing | Text -> Nothing | Text -> Vector -> Connection ! Sql_Error
+connect url user=Nothing password=Nothing custom_properties=[] =
     full_url = if url.starts_with "jdbc:" then url else "jdbc:"+url
-    Connection.create_jdbc_connection full_url user password
+    user_prop = if user.is_nothing then [] else [["user", user]]
+    pass_prop = if user.is_nothing then [] else [["password", password]]
+    properties = user_prop + pass_prop + custom_properties
+    Connection.create_jdbc_connection full_url properties
 
 ## UNSTABLE
 

distribution/std-lib/Standard/src/Database/Data/Column.enso

@@ -64,13 +64,6 @@ type Column
 
        Returns a materialized dataframe containing rows of this table.
 
-       Currently the SQL decimal type is mapped to a floating-point
-       representation which is not exact. While operations in the database
-       itself may (or may not, depending on the database engine) be done with
-       exact precision, the materialized values may loose precision, so extra
-       care is needed when working with decimal computations that need to be
-       exact.
-
        Arguments:
        - max_rows: specifies a maximum amount of rows to fetch; if not set, all
          available rows are fetched.
@@ -82,13 +75,6 @@ type Column
     ## UNSTABLE
 
        Returns a vector containing all the elements in this column.
-
-       Currently the SQL decimal type is mapped to a floating-point
-       representation which is not exact. While operations in the database
-       itself may (or may not, depending on the database engine) be done with
-       exact precision, the materialized values may loose precision, so extra
-       care is needed when working with decimal computations that need to be
-       exact.
     to_vector : Vector
     to_vector =
         ## We remove the index to avoid fetching index data that will not be
@@ -376,6 +362,9 @@ type Column
        Checks for each element of the column if it starts with `other`.
        If `other` is a column, the operation is performed pairwise between
        corresponding elements of `this` and `other`.
+
+       If the argument is a missing value (a Nothing or a column with missing
+       values), the behaviour on these missing values is vendor specific.
     starts_with : Column | Text -> Column
     starts_with other = this.make_binary_op "starts_with" other new_type=Sql_Type.boolean
 
@@ -384,6 +373,9 @@ type Column
        Checks for each element of the column if it ends with `other`.
        If `other` is a column, the operation is performed pairwise between
        corresponding elements of `this` and `other`.
+
+       If the argument is a missing value (a Nothing or a column with missing
+       values), the behaviour on these missing values is vendor specific.
     ends_with : Column | Text -> Column
     ends_with other = this.make_binary_op "ends_with" other new_type=Sql_Type.boolean
 
@@ -392,6 +384,9 @@ type Column
        Checks for each element of the column if it contains `other`.
        If `other` is a column, the operation is performed pairwise between
        corresponding elements of `this` and `other`.
+
+       If the argument is a missing value (a Nothing or a column with missing
+       values), the behaviour on these missing values is vendor specific.
     contains : Column | Text -> Column
     contains other = this.make_binary_op "contains" other new_type=Sql_Type.boolean
 

distribution/std-lib/Standard/src/Database/Data/Sql.enso

@@ -24,6 +24,10 @@ type Sql_Type
     decimal : Sql_Type
     decimal = Sql_Type Types.DECIMAL
 
+    ## The SQL type representing a general numeric type.
+    numeric : Sql_Type
+    numeric = Sql_Type Types.NUMERIC
+
     ## PRIVATE
 
        Returns True if this type represents an integer. It only handles the

distribution/std-lib/Standard/src/Database/Data/Table.enso

@@ -392,13 +392,6 @@ type Table
 
        Returns a materialized dataframe containing rows of this table.
 
-       Currently the SQL decimal type is mapped to a floating-point
-       representation which is not exact. While operations in the database
-       itself may (or may not, depending on the database engine) be done with
-       exact precision, the materialized values may loose precision, so extra
-       care is needed when working with decimal computations that need to be
-       exact.
-
        Arguments:
        - max_rows: specifies a maximum amount of rows to fetch; if not set, all
          available rows are fetched.