From e2c4cdae327ecd343cbb4762291996f5450bb9ab Mon Sep 17 00:00:00 2001 From: Zhou Kang Date: Sat, 16 Mar 2024 00:00:57 +0800 Subject: [PATCH] Add a basic skeleton for Doris catalog. --- build.gradle.kts | 1 + catalogs/catalog-jdbc-doris/build.gradle.kts | 87 +++++++++++++++++++ .../gravitino/catalog/doris/DorisCatalog.java | 51 +++++++++++ .../DorisColumnDefaultValueConverter.java | 23 +++++ .../converter/DorisExceptionConverter.java | 20 +++++ .../doris/converter/DorisTypeConverter.java | 24 +++++ .../operation/DorisDatabaseOperations.java | 31 +++++++ .../doris/operation/DorisTableOperations.java | 56 ++++++++++++ .../com.datastrato.gravitino.CatalogProvider | 5 ++ .../src/main/resources/jdbc-doris.conf | 8 ++ settings.gradle.kts | 7 +- 11 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 catalogs/catalog-jdbc-doris/build.gradle.kts create mode 100644 catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java create mode 100644 catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java create mode 100644 catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java create mode 100644 catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java create mode 100644 catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java create mode 100644 catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java create mode 100644 catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider create mode 100644 catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf diff --git a/build.gradle.kts b/build.gradle.kts index 818e06a8a97..d0781ab8939 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -671,6 +671,7 @@ tasks { dependsOn( ":catalogs:catalog-hive:copyLibAndConfig", ":catalogs:catalog-lakehouse-iceberg:copyLibAndConfig", + ":catalogs:catalog-jdbc-doris:copyLibAndConfig", ":catalogs:catalog-jdbc-mysql:copyLibAndConfig", ":catalogs:catalog-jdbc-postgresql:copyLibAndConfig", ":catalogs:catalog-hadoop:copyLibAndConfig", diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts new file mode 100644 index 00000000000..7adcce632ca --- /dev/null +++ b/catalogs/catalog-jdbc-doris/build.gradle.kts @@ -0,0 +1,87 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +description = "catalog-jdbc-doris" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":api")) + implementation(project(":catalogs:catalog-jdbc-common")) + implementation(project(":common")) + implementation(project(":core")) + + implementation(libs.bundles.log4j) + implementation(libs.commons.collections4) + implementation(libs.commons.lang3) + implementation(libs.guava) + implementation(libs.jsqlparser) + implementation(libs.slf4j.api) + + testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts")) + testImplementation(project(":clients:client-java")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) + + testImplementation(libs.commons.lang3) + testImplementation(libs.guava) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mysql.driver) + testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.mysql) + + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + val copyCatalogLibs by registering(Copy::class) { + dependsOn("jar", "runtimeJars") + from("build/libs") + into("$rootDir/distribution/package/catalogs/jdbc-doris/libs") + } + + val copyCatalogConfig by registering(Copy::class) { + from("src/main/resources") + into("$rootDir/distribution/package/catalogs/jdbc-doris/conf") + + include("jdbc-doris.conf") + + exclude { details -> + details.file.isDirectory() + } + } + + register("copyLibAndConfig", Copy::class) { + dependsOn(copyCatalogLibs, copyCatalogConfig) + } +} + +tasks.test { + val skipUTs = project.hasProperty("skipTests") + if (skipUTs) { + // Only run integration tests + include("**/integration/**") + } + + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/**") + } else { + dependsOn(tasks.jar) + + val init = project.extra.get("initIntegrationTest") as (Test) -> Unit + init(this) + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java new file mode 100644 index 00000000000..69fa0b39cb2 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/DorisCatalog.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris; + +import com.datastrato.gravitino.catalog.doris.converter.DorisColumnDefaultValueConverter; +import com.datastrato.gravitino.catalog.doris.converter.DorisExceptionConverter; +import com.datastrato.gravitino.catalog.doris.converter.DorisTypeConverter; +import com.datastrato.gravitino.catalog.doris.operation.DorisDatabaseOperations; +import com.datastrato.gravitino.catalog.doris.operation.DorisTableOperations; +import com.datastrato.gravitino.catalog.jdbc.JdbcCatalog; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; + +/** Implementation of a Doris catalog in Gravitino. */ +public class DorisCatalog extends JdbcCatalog { + + @Override + public String shortName() { + return "jdbc-doris"; + } + + @Override + protected JdbcExceptionConverter createExceptionConverter() { + return new DorisExceptionConverter(); + } + + @Override + protected JdbcTypeConverter createJdbcTypeConverter() { + return new DorisTypeConverter(); + } + + @Override + protected JdbcDatabaseOperations createJdbcDatabaseOperations() { + return new DorisDatabaseOperations(); + } + + @Override + protected JdbcTableOperations createJdbcTableOperations() { + return new DorisTableOperations(); + } + + @Override + protected JdbcColumnDefaultValueConverter createJdbcColumnDefaultValueConverter() { + return new DorisColumnDefaultValueConverter(); + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java new file mode 100644 index 00000000000..429270537ce --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisColumnDefaultValueConverter.java @@ -0,0 +1,23 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.converter; + +import static com.datastrato.gravitino.rel.Column.DEFAULT_VALUE_NOT_SET; + +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcColumnDefaultValueConverter; +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import com.datastrato.gravitino.rel.expressions.Expression; + +public class DorisColumnDefaultValueConverter extends JdbcColumnDefaultValueConverter { + @Override + public Expression toGravitino( + JdbcTypeConverter.JdbcTypeBean columnType, + String columnDefaultValue, + boolean isExpression, + boolean nullable) { + // TODO: add implementation for doris catalog + return DEFAULT_VALUE_NOT_SET; + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java new file mode 100644 index 00000000000..26121a1b4de --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisExceptionConverter.java @@ -0,0 +1,20 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.converter; + +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter; +import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import java.sql.SQLException; + +/** Exception converter to gravitino exception for Doris. */ +public class DorisExceptionConverter extends JdbcExceptionConverter { + @SuppressWarnings("FormatStringAnnotation") + @Override + public GravitinoRuntimeException toGravitinoException(SQLException se) { + // TODO: add implementation for doris catalog + + return super.toGravitinoException(se); + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java new file mode 100644 index 00000000000..08b75e20b92 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/converter/DorisTypeConverter.java @@ -0,0 +1,24 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.converter; + +import com.datastrato.gravitino.catalog.jdbc.converter.JdbcTypeConverter; +import com.datastrato.gravitino.rel.types.Type; +import com.datastrato.gravitino.rel.types.Types; + +/** Type converter for Doris. */ +public class DorisTypeConverter extends JdbcTypeConverter { + // TODO: add implementation for doris catalog + + @Override + public Type toGravitinoType(JdbcTypeBean typeBean) { + return Types.UnparsedType.of(typeBean.getTypeName()); + } + + @Override + public String fromGravitinoType(Type type) { + throw new IllegalArgumentException("unsupported type: " + type.simpleString()); + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java new file mode 100644 index 00000000000..82d4b1cac09 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisDatabaseOperations.java @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.operation; + +import com.datastrato.gravitino.catalog.jdbc.JdbcSchema; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; +import com.datastrato.gravitino.exceptions.NoSuchSchemaException; +import java.util.Map; + +/** Database operations for Doris. */ +public class DorisDatabaseOperations extends JdbcDatabaseOperations { + // TODO: add implementation for doris catalog + + @Override + public JdbcSchema load(String databaseName) throws NoSuchSchemaException { + throw new UnsupportedOperationException(); + } + + @Override + protected String generateCreateDatabaseSql( + String databaseName, String comment, Map properties) { + throw new UnsupportedOperationException(); + } + + @Override + protected String generateDropDatabaseSql(String databaseName, boolean cascade) { + throw new UnsupportedOperationException(); + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java new file mode 100644 index 00000000000..6ce6bf5d9f6 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/java/com/datastrato/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.doris.operation; + +import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; +import com.datastrato.gravitino.catalog.jdbc.JdbcTable; +import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.datastrato.gravitino.rel.indexes.Index; +import java.util.Map; + +/** Table operations for Doris. */ +public class DorisTableOperations extends JdbcTableOperations { + // TODO: add implementation for doris catalog + + @Override + protected String generateCreateTableSql( + String tableName, + JdbcColumn[] columns, + String comment, + Map properties, + Transform[] partitioning, + Index[] indexes) { + throw new UnsupportedOperationException(); + } + + @Override + protected String generateRenameTableSql(String oldTableName, String newTableName) { + throw new UnsupportedOperationException(); + } + + @Override + protected String generateDropTableSql(String tableName) { + throw new UnsupportedOperationException(); + } + + @Override + protected String generatePurgeTableSql(String tableName) { + throw new UnsupportedOperationException(); + } + + @Override + protected String generateAlterTableSql( + String databaseName, String tableName, TableChange... changes) { + throw new UnsupportedOperationException(); + } + + @Override + protected JdbcTable getOrCreateTable( + String databaseName, String tableName, JdbcTable lazyLoadCreateTable) { + throw new UnsupportedOperationException(); + } +} diff --git a/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider b/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider new file mode 100644 index 00000000000..94219d8a5d2 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/resources/META-INF/services/com.datastrato.gravitino.CatalogProvider @@ -0,0 +1,5 @@ +# +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. +# +com.datastrato.gravitino.catalog.doris.DorisCatalog \ No newline at end of file diff --git a/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf b/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf new file mode 100644 index 00000000000..56262130029 --- /dev/null +++ b/catalogs/catalog-jdbc-doris/src/main/resources/jdbc-doris.conf @@ -0,0 +1,8 @@ +# +# Copyright 2024 Datastrato Pvt Ltd. +# This software is licensed under the Apache License version 2. +# +# jdbc-url: jdbc:mysql://localhost:9030/ +# jdbc-user: strato +# jdbc-password: strato +# jdbc-driver: com.mysql.jdbc.Driver \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 53066813eb6..7e39ae3fd50 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -12,7 +12,12 @@ include("api", "common", "core", "meta", "server", "integration-test", "server-c include("catalogs:bundled-catalog") include("catalogs:catalog-hive") include("catalogs:catalog-lakehouse-iceberg") -include("catalogs:catalog-jdbc-common", "catalogs:catalog-jdbc-mysql", "catalogs:catalog-jdbc-postgresql") +include( + "catalogs:catalog-jdbc-common", + "catalogs:catalog-jdbc-mysql", + "catalogs:catalog-jdbc-postgresql", + "catalogs:catalog-jdbc-doris" +) include("catalogs:catalog-hadoop") include("catalogs:catalog-messaging-kafka") include("clients:client-java", "clients:client-java-runtime")