Add support for Databricks runtime 14.3 & 15.4 LTS

.github/workflows/ci.yml

@@ -881,6 +881,8 @@ jobs:
             - suite-delta-lake-databricks113
             - suite-delta-lake-databricks122
             - suite-delta-lake-databricks133
+            - suite-delta-lake-databricks143
+            - suite-delta-lake-databricks154
             - suite-databricks-unity-http-hms
             - suite-gcs
             - suite-clients
@@ -929,6 +931,12 @@ jobs:
             - suite: suite-delta-lake-databricks133
               ignore exclusion if: >-
                 ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
+            - suite: suite-delta-lake-databricks143
+              ignore exclusion if: >-
+                ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
+            - suite: suite-delta-lake-databricks154
+              ignore exclusion if: >-
+                ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
             - suite: suite-databricks-unity-http-hms
               config: hdp3
             - suite: suite-databricks-unity-http-hms
@@ -987,6 +995,8 @@ jobs:
           DATABRICKS_113_JDBC_URL: ""
           DATABRICKS_122_JDBC_URL: ""
           DATABRICKS_133_JDBC_URL: ""
+          DATABRICKS_143_JDBC_URL: ""
+          DATABRICKS_154_JDBC_URL: ""
           DATABRICKS_UNITY_JDBC_URL: ""
           DATABRICKS_UNITY_CATALOG_NAME: ""
           DATABRICKS_UNITY_EXTERNAL_LOCATION: ""
@@ -1063,6 +1073,8 @@ jobs:
           DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }}
           DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }}
           DATABRICKS_133_JDBC_URL: ${{ vars.DATABRICKS_133_JDBC_URL }}
+          DATABRICKS_143_JDBC_URL: ${{ vars.DATABRICKS_143_JDBC_URL }}
+          DATABRICKS_154_JDBC_URL: ${{ vars.DATABRICKS_154_JDBC_URL }}
           DATABRICKS_UNITY_JDBC_URL: ${{ vars.DATABRICKS_UNITY_JDBC_URL }}
           DATABRICKS_UNITY_CATALOG_NAME: ${{ vars.DATABRICKS_UNITY_CATALOG_NAME }}
           DATABRICKS_UNITY_EXTERNAL_LOCATION: ${{ vars.DATABRICKS_UNITY_EXTERNAL_LOCATION }}

docs/src/main/sphinx/connector/delta-lake.md

@@ -13,7 +13,7 @@ data.
 To connect to Databricks Delta Lake, you need:
 
 - Tables written by Databricks Runtime 7.3 LTS, 9.1 LTS, 10.4 LTS, 11.3 LTS,
-  12.2 LTS and 13.3 LTS are supported.
+  12.2 LTS, 13.3 LTS, 14.3 LTS and 15.4 LTS are supported.
 - Deployments using AWS, HDFS, Azure Storage, and Google Cloud Storage (GCS) are
   fully supported.
 - Network access from the coordinator and workers to the Delta Lake storage.

testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java

@@ -98,6 +98,8 @@ public final class TestGroups
     public static final String DELTA_LAKE_DATABRICKS_104 = "delta-lake-databricks-104";
     public static final String DELTA_LAKE_DATABRICKS_113 = "delta-lake-databricks-113";
     public static final String DELTA_LAKE_DATABRICKS_122 = "delta-lake-databricks-122";
+    public static final String DELTA_LAKE_DATABRICKS_133 = "delta-lake-databricks-133";
+    public static final String DELTA_LAKE_DATABRICKS_143 = "delta-lake-databricks-143";
     public static final String DATABRICKS_UNITY_HTTP_HMS = "databricks-unity-http-hms";
     public static final String DELTA_LAKE_EXCLUDE_91 = "delta-lake-exclude-91";
     public static final String DELTA_LAKE_ALLUXIO_CACHING = "delta-lake-alluxio-caching";

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.env.environment;
+
+import com.google.inject.Inject;
+import io.trino.tests.product.launcher.docker.DockerFiles;
+import io.trino.tests.product.launcher.env.common.Standard;
+import io.trino.tests.product.launcher.env.common.TestsEnvironment;
+
+import static java.util.Objects.requireNonNull;
+
+@TestsEnvironment
+public class EnvSinglenodeDeltaLakeDatabricks143
+        extends AbstractSinglenodeDeltaLakeDatabricks
+{
+    @Inject
+    public EnvSinglenodeDeltaLakeDatabricks143(Standard standard, DockerFiles dockerFiles)
+    {
+        super(standard, dockerFiles);
+    }
+
+    @Override
+    String databricksTestJdbcUrl()
+    {
+        return requireNonNull(System.getenv("DATABRICKS_143_JDBC_URL"), "Environment DATABRICKS_143_JDBC_URL was not set");
+    }
+}

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.env.environment;
+
+import com.google.inject.Inject;
+import io.trino.tests.product.launcher.docker.DockerFiles;
+import io.trino.tests.product.launcher.env.common.Standard;
+import io.trino.tests.product.launcher.env.common.TestsEnvironment;
+
+import static java.util.Objects.requireNonNull;
+
+@TestsEnvironment
+public class EnvSinglenodeDeltaLakeDatabricks154
+        extends AbstractSinglenodeDeltaLakeDatabricks
+{
+    @Inject
+    public EnvSinglenodeDeltaLakeDatabricks154(Standard standard, DockerFiles dockerFiles)
+    {
+        super(standard, dockerFiles);
+    }
+
+    @Override
+    String databricksTestJdbcUrl()
+    {
+        return requireNonNull(System.getenv("DATABRICKS_154_JDBC_URL"), "Environment DATABRICKS_154_JDBC_URL was not set");
+    }
+}

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks133.java

@@ -22,7 +22,7 @@
 import java.util.List;
 
 import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES;
-import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS;
+import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133;
 import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment;
 
 public class SuiteDeltaLakeDatabricks133
@@ -33,7 +33,7 @@ public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
     {
         return ImmutableList.of(
                 testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks133.class)
-                        .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS)
+                        .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS_133)
                         .withExcludedTests(getExcludedTests())
                         .build());
     }

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks143.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.suite.suites;
+
+import com.google.common.collect.ImmutableList;
+import io.trino.tests.product.launcher.env.EnvironmentConfig;
+import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks143;
+import io.trino.tests.product.launcher.suite.SuiteDeltaLakeDatabricks;
+import io.trino.tests.product.launcher.suite.SuiteTestRun;
+
+import java.util.List;
+
+import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES;
+import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143;
+import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment;
+
+public class SuiteDeltaLakeDatabricks143
+        extends SuiteDeltaLakeDatabricks
+{
+    @Override
+    public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
+    {
+        return ImmutableList.of(
+                testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks143.class)
+                        .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS_143)
+                        .withExcludedTests(getExcludedTests())
+                        .build());
+    }
+}

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks154.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.tests.product.launcher.suite.suites;
+
+import com.google.common.collect.ImmutableList;
+import io.trino.tests.product.launcher.env.EnvironmentConfig;
+import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks154;
+import io.trino.tests.product.launcher.suite.SuiteDeltaLakeDatabricks;
+import io.trino.tests.product.launcher.suite.SuiteTestRun;
+
+import java.util.List;
+
+import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES;
+import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS;
+import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment;
+
+public class SuiteDeltaLakeDatabricks154
+        extends SuiteDeltaLakeDatabricks
+{
+    @Override
+    public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
+    {
+        return ImmutableList.of(
+                testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks154.class)
+                        .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS)
+                        .withExcludedTests(getExcludedTests())
+                        .build());
+    }
+}

testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java

@@ -28,6 +28,7 @@
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91;
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
 import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
+import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION;
 import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_91_RUNTIME_VERSION;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH;
@@ -274,6 +275,11 @@ public void testTrinoPreservesTableFeature()
     @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
     public void testTrinoAlterTablePreservesGeneratedColumn()
     {
+        if (getDatabricksRuntimeVersion().orElseThrow().isAtLeast(DATABRICKS_143_RUNTIME_VERSION)) {
+            // The following COMMENT statement throws an exception (expected) because version >= 14.3 stores 'generatedColumns' writer feature
+            return;
+        }
+
         String tableName = "test_trino_alter_table_preserves_generated_column_" + randomNameSuffix();
         String tableDirectory = "databricks-compatibility-test-" + tableName;
 

testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCaseInsensitiveMapping.java

@@ -27,14 +27,18 @@
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_104;
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_113;
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122;
+import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133;
+import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143;
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_91;
 import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS;
 import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS;
+import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnDelta;
 import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnTrino;
+import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getDatabricksRuntimeVersion;
 import static io.trino.tests.product.utils.QueryExecutors.onDelta;
 import static io.trino.tests.product.utils.QueryExecutors.onTrino;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -72,7 +76,7 @@ public void testNonLowercaseColumnNames()
         }
     }
 
-    @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
+    @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_104, DELTA_LAKE_DATABRICKS_113, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS})
     @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
     public void testNonLowercaseFieldNames()
     {
@@ -161,6 +165,11 @@ public void testNotNullColumnWithNonLowerCaseColumnName()
     @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH)
     public void testGeneratedColumnWithNonLowerCaseColumnName()
     {
+        if (getDatabricksRuntimeVersion().orElseThrow().isAtLeast(DATABRICKS_143_RUNTIME_VERSION)) {
+            // The following COMMENT statement throws an exception (expected) because version >= 14.3 stores 'generatedColumns' writer feature
+            return;
+        }
+
         String tableName = "test_dl_generated_column_uppercase_name" + randomNameSuffix();
 
         onDelta().executeQuery("" +
@@ -210,6 +219,11 @@ public void testIdentityColumnWithNonLowerCaseColumnName()
                     "ON (t.data = s.data) WHEN MATCHED THEN UPDATE SET data = 1"))
                     .hasMessageContaining("Writing to tables with identity columns is not supported");
 
+            if (getDatabricksRuntimeVersion().orElseThrow().isAtLeast(DATABRICKS_143_RUNTIME_VERSION)) {
+                // The following statement throws an exception (expected) because version >= 14.3 stores 'identityColumns' writer feature
+                return;
+            }
+
             // Verify column operations preserves the identify column name and property
             onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " ADD COLUMN new_col integer");
             onTrino().executeQuery("ALTER TABLE delta.default." + tableName + " RENAME COLUMN new_col TO renamed_col");