diff --git a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java index 8a625957024..e3f391162fc 100644 --- a/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java +++ b/api/src/main/java/com/datastrato/gravitino/exceptions/NoSuchEntityException.java @@ -9,6 +9,8 @@ /** This exception is thrown when an entity is not found. */ public class NoSuchEntityException extends RuntimeException { + /** The no such an entity message for the exception. */ + public static final String NO_SUCH_AN_ENTITY_MESSAGE = "No such an entity: %s"; /** * Constructs a new NoSuchEntityException. diff --git a/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java b/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java index 82799ad6b49..2a235fe13e3 100644 --- a/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/utils/MapUtils.java @@ -11,6 +11,7 @@ /** Utility class for working with maps. */ public class MapUtils { + private MapUtils() {} /** * Returns a map with all keys that start with the given prefix. diff --git a/core/build.gradle.kts b/core/build.gradle.kts index a327db37eb6..6f6fb3b7266 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -18,7 +18,6 @@ dependencies { implementation(libs.caffeine) implementation(libs.commons.dbcp2) implementation(libs.commons.io) - implementation(libs.commons.dbcp2) implementation(libs.commons.lang3) implementation(libs.guava) implementation(libs.mybatis) @@ -27,7 +26,6 @@ dependencies { .because("Brings in Guava for Android, which we don't want (and breaks multimaps).") } implementation(libs.rocksdbjni) - implementation(libs.mybatis) annotationProcessor(libs.lombok) compileOnly(libs.lombok) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java index 69731e0d393..652cf52cadd 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -12,6 +12,14 @@ import org.apache.ibatis.annotations.Select; import org.apache.ibatis.annotations.Update; +/** + * A MyBatis Mapper for catalog meta operation SQLs. + * + *

This interface class is a specification defined by MyBatis. It requires this interface class + * to identify the corresponding SQLs for execution. We can write SQLs in an additional XML file, or + * write SQLs with annotations in this interface Mapper. See: + */ public interface CatalogMetaMapper { String TABLE_NAME = "catalog_meta"; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index 9239e20da30..2a9f9236e54 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -23,6 +23,9 @@ import java.util.Objects; import java.util.function.Function; +/** + * The service class for catalog metadata. It provides the basic database operations for catalog. + */ public class CatalogMetaService { private static final CatalogMetaService INSTANCE = new CatalogMetaService(); @@ -40,14 +43,16 @@ public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) { SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { - throw new NoSuchEntityException("No such an entity: %s", identifier.namespace().toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.namespace().toString()); } CatalogPO catalogPO = SessionUtils.getWithoutCommit( CatalogMetaMapper.class, mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName)); if (catalogPO == null) { - throw new NoSuchEntityException("No such entity: %s", identifier.toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.toString()); } return POConverters.fromCatalogPO(catalogPO, identifier.namespace()); } @@ -59,7 +64,8 @@ public List listCatalogsByNamespace(Namespace namespace) { SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { - throw new NoSuchEntityException("No such an entity: %s", namespace.toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, namespace.toString()); } List catalogPOS = SessionUtils.getWithoutCommit( @@ -76,7 +82,7 @@ public void insertCatalog(CatalogEntity catalogEntity, boolean overwrite) { mapper -> mapper.selectMetalakeIdMetaByName(catalogEntity.namespace().level(0))); if (metalakeId == null) { throw new NoSuchEntityException( - "No such an entity: %s", catalogEntity.namespace().toString()); + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, catalogEntity.namespace().toString()); } SessionUtils.doWithCommit( CatalogMetaMapper.class, @@ -113,7 +119,8 @@ public CatalogEntity updateCatalog( SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { - throw new NoSuchEntityException("No such an entity: %s", identifier.namespace().toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.namespace().toString()); } CatalogPO oldCatalogPO = @@ -121,7 +128,8 @@ public CatalogEntity updateCatalog( CatalogMetaMapper.class, mapper -> mapper.selectCatalogMetaByMetalakeIdAndName(metalakeId, catalogName)); if (oldCatalogPO == null) { - throw new NoSuchEntityException("No such an entity: %s", identifier.toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.toString()); } CatalogEntity oldCatalogEntity = @@ -156,7 +164,8 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); if (metalakeId == null) { - throw new NoSuchEntityException("No such an entity: %s", identifier.namespace().toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, identifier.namespace().toString()); } Long catalogId = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 1b43d41355c..0ac0dfe7f60 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -26,6 +26,9 @@ import java.util.Objects; import java.util.function.Function; +/** + * The service class for metalake metadata. It provides the basic database operations for metalake. + */ public class MetalakeMetaService { private static final MetalakeMetaService INSTANCE = new MetalakeMetaService(); @@ -47,7 +50,8 @@ public BaseMetalake getMetalakeByIdentifier(NameIdentifier ident) { SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); if (metalakePO == null) { - throw new NoSuchEntityException("No such an entity: %s", ident.toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, ident.toString()); } return POConverters.fromMetalakePO(metalakePO); } @@ -86,7 +90,8 @@ public BaseMetalake updateMetalake( SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); if (oldMetalakePO == null) { - throw new NoSuchEntityException("No such an entity: %s", ident.toString()); + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_AN_ENTITY_MESSAGE, ident.toString()); } BaseMetalake oldMetalakeEntity = POConverters.fromMetalakePO(oldMetalakePO); @@ -111,36 +116,34 @@ public BaseMetalake updateMetalake( } public boolean deleteMetalake(NameIdentifier ident, boolean cascade) { - MetalakePO metalakePO = + Long metalakeId = SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeMetaByName(ident.name())); - if (metalakePO != null) { + MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(ident.name())); + if (metalakeId != null) { if (cascade) { SessionUtils.doMultipleWithCommit( () -> SessionUtils.doWithoutCommit( MetalakeMetaMapper.class, - mapper -> - mapper.softDeleteMetalakeMetaByMetalakeId(metalakePO.getMetalakeId())), + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)), () -> SessionUtils.doWithoutCommit( CatalogMetaMapper.class, - mapper -> - mapper.softDeleteCatalogMetasByMetalakeId(metalakePO.getMetalakeId())), + mapper -> mapper.softDeleteCatalogMetasByMetalakeId(metalakeId)), () -> { // TODO We will cascade delete the metadata of sub-resources under the metalake }); } else { List catalogEntities = CatalogMetaService.getInstance() - .listCatalogsByNamespace(Namespace.ofCatalog(metalakePO.getMetalakeName())); + .listCatalogsByNamespace(Namespace.ofCatalog(ident.name())); if (!catalogEntities.isEmpty()) { throw new NonEmptyEntityException( "Entity %s has sub-entities, you should remove sub-entities first", ident); } SessionUtils.doWithCommit( MetalakeMetaMapper.class, - mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakePO.getMetalakeId())); + mapper -> mapper.softDeleteMetalakeMetaByMetalakeId(metalakeId)); } } return true; diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java index a19ea0eacaf..4aa72329d73 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/POConverters.java @@ -15,7 +15,6 @@ import com.datastrato.gravitino.storage.relational.po.CatalogPO; import com.datastrato.gravitino.storage.relational.po.MetalakePO; import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.annotations.VisibleForTesting; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -26,13 +25,12 @@ public class POConverters { private POConverters() {} /** - * Convert {@link BaseMetalake} to {@link MetalakePO} + * Initialize MetalakePO * * @param baseMetalake BaseMetalake object - * @return MetalakePO object from BaseMetalake object + * @return MetalakePO object with version initialized */ - @VisibleForTesting - static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { + public static MetalakePO initializeMetalakePOWithVersion(BaseMetalake baseMetalake) { try { return new MetalakePO.Builder() .withMetalakeId(baseMetalake.id()) @@ -42,33 +40,15 @@ static MetalakePO toMetalakePO(BaseMetalake baseMetalake) { .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.auditInfo())) .withSchemaVersion( JsonUtils.anyFieldMapper().writeValueAsString(baseMetalake.getVersion())) + .withCurrentVersion(1L) + .withLastVersion(1L) + .withDeletedAt(0L) .build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to serialize json object:", e); } } - /** - * Initialize MetalakePO - * - * @param baseMetalake BaseMetalake object - * @return MetalakePO object with version initialized - */ - public static MetalakePO initializeMetalakePOWithVersion(BaseMetalake baseMetalake) { - MetalakePO metalakePO = toMetalakePO(baseMetalake); - return new MetalakePO.Builder() - .withMetalakeId(metalakePO.getMetalakeId()) - .withMetalakeName(metalakePO.getMetalakeName()) - .withMetalakeComment(metalakePO.getMetalakeComment()) - .withProperties(metalakePO.getProperties()) - .withAuditInfo(metalakePO.getAuditInfo()) - .withSchemaVersion(metalakePO.getSchemaVersion()) - .withCurrentVersion(1L) - .withLastVersion(1L) - .withDeletedAt(0L) - .build(); - } - /** * Update MetalakePO version * @@ -78,21 +58,25 @@ public static MetalakePO initializeMetalakePOWithVersion(BaseMetalake baseMetala */ public static MetalakePO updateMetalakePOWithVersion( MetalakePO oldMetalakePO, BaseMetalake newMetalake) { - MetalakePO newMetalakePO = toMetalakePO(newMetalake); Long lastVersion = oldMetalakePO.getLastVersion(); // Will set the version to the last version + 1 when having some fields need be multiple version Long nextVersion = lastVersion; - return new MetalakePO.Builder() - .withMetalakeId(newMetalakePO.getMetalakeId()) - .withMetalakeName(newMetalakePO.getMetalakeName()) - .withMetalakeComment(newMetalakePO.getMetalakeComment()) - .withProperties(newMetalakePO.getProperties()) - .withAuditInfo(newMetalakePO.getAuditInfo()) - .withSchemaVersion(newMetalakePO.getSchemaVersion()) - .withCurrentVersion(nextVersion) - .withLastVersion(nextVersion) - .withDeletedAt(0L) - .build(); + try { + return new MetalakePO.Builder() + .withMetalakeId(newMetalake.id()) + .withMetalakeName(newMetalake.name()) + .withMetalakeComment(newMetalake.comment()) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(newMetalake.properties())) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(newMetalake.auditInfo())) + .withSchemaVersion( + JsonUtils.anyFieldMapper().writeValueAsString(newMetalake.getVersion())) + .withCurrentVersion(nextVersion) + .withLastVersion(nextVersion) + .withDeletedAt(0L) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } } /** @@ -131,14 +115,13 @@ public static List fromMetalakePOs(List metalakePOS) { } /** - * Convert {@link CatalogEntity} to {@link CatalogPO} + * Initialize CatalogPO * - * @param catalogEntity CatalogEntity object to be converted - * @param metalakeId Metalake id to be associated with the catalog - * @return CatalogPO object from CatalogEntity object + * @param catalogEntity CatalogEntity object + * @return CatalogPO object with version initialized */ - @VisibleForTesting - static CatalogPO toCatalogPO(CatalogEntity catalogEntity, Long metalakeId) { + public static CatalogPO initializeCatalogPOWithVersion( + CatalogEntity catalogEntity, Long metalakeId) { try { return new CatalogPO.Builder() .withCatalogId(catalogEntity.id()) @@ -150,36 +133,15 @@ static CatalogPO toCatalogPO(CatalogEntity catalogEntity, Long metalakeId) { .withProperties( JsonUtils.anyFieldMapper().writeValueAsString(catalogEntity.getProperties())) .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(catalogEntity.auditInfo())) + .withCurrentVersion(1L) + .withLastVersion(1L) + .withDeletedAt(0L) .build(); } catch (JsonProcessingException e) { throw new RuntimeException("Failed to serialize json object:", e); } } - /** - * Initialize CatalogPO - * - * @param catalogEntity CatalogEntity object - * @return CatalogPO object with version initialized - */ - public static CatalogPO initializeCatalogPOWithVersion( - CatalogEntity catalogEntity, Long metalakeId) { - CatalogPO catalogPO = toCatalogPO(catalogEntity, metalakeId); - return new CatalogPO.Builder() - .withCatalogId(catalogPO.getCatalogId()) - .withCatalogName(catalogPO.getCatalogName()) - .withMetalakeId(metalakeId) - .withType(catalogPO.getType()) - .withProvider(catalogPO.getProvider()) - .withCatalogComment(catalogPO.getCatalogComment()) - .withProperties(catalogPO.getProperties()) - .withAuditInfo(catalogPO.getAuditInfo()) - .withCurrentVersion(1L) - .withLastVersion(1L) - .withDeletedAt(0L) - .build(); - } - /** * Update CatalogPO version * @@ -189,23 +151,26 @@ public static CatalogPO initializeCatalogPOWithVersion( */ public static CatalogPO updateCatalogPOWithVersion( CatalogPO oldCatalogPO, CatalogEntity newCatalog, Long metalakeId) { - CatalogPO newCatalogPO = toCatalogPO(newCatalog, metalakeId); Long lastVersion = oldCatalogPO.getLastVersion(); // Will set the version to the last version + 1 when having some fields need be multiple version Long nextVersion = lastVersion; - return new CatalogPO.Builder() - .withCatalogId(newCatalogPO.getCatalogId()) - .withCatalogName(newCatalogPO.getCatalogName()) - .withMetalakeId(metalakeId) - .withType(newCatalogPO.getType()) - .withProvider(newCatalogPO.getProvider()) - .withCatalogComment(newCatalogPO.getCatalogComment()) - .withProperties(newCatalogPO.getProperties()) - .withAuditInfo(newCatalogPO.getAuditInfo()) - .withCurrentVersion(nextVersion) - .withLastVersion(nextVersion) - .withDeletedAt(0L) - .build(); + try { + return new CatalogPO.Builder() + .withCatalogId(newCatalog.id()) + .withCatalogName(newCatalog.name()) + .withMetalakeId(metalakeId) + .withType(newCatalog.getType().name()) + .withProvider(newCatalog.getProvider()) + .withCatalogComment(newCatalog.getComment()) + .withProperties(JsonUtils.anyFieldMapper().writeValueAsString(newCatalog.getProperties())) + .withAuditInfo(JsonUtils.anyFieldMapper().writeValueAsString(newCatalog.auditInfo())) + .withCurrentVersion(nextVersion) + .withLastVersion(nextVersion) + .withDeletedAt(0L) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to serialize json object:", e); + } } /** diff --git a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java index 378f39918af..544b5e713e6 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/relational/utils/TestPOConverters.java @@ -177,43 +177,6 @@ public void testUpdateCatalogPOVersion() { assertEquals("this is test2", updatePO.getCatalogComment()); } - @Test - public void testToMetalakePO() throws JsonProcessingException { - BaseMetalake metalake = createMetalake(1L, "test", "this is test"); - - MetalakePO expectedMetalakePO = createMetalakePO(1L, "test", "this is test"); - - MetalakePO actualMetalakePO = POConverters.toMetalakePO(metalake); - - // Assert - assertEquals(expectedMetalakePO.getMetalakeId(), actualMetalakePO.getMetalakeId()); - assertEquals(expectedMetalakePO.getMetalakeName(), actualMetalakePO.getMetalakeName()); - assertEquals(expectedMetalakePO.getMetalakeComment(), actualMetalakePO.getMetalakeComment()); - assertEquals(expectedMetalakePO.getProperties(), actualMetalakePO.getProperties()); - assertEquals(expectedMetalakePO.getAuditInfo(), actualMetalakePO.getAuditInfo()); - assertEquals(expectedMetalakePO.getSchemaVersion(), actualMetalakePO.getSchemaVersion()); - } - - @Test - public void testToCatalogPO() throws JsonProcessingException { - CatalogEntity catalog = - createCatalog(1L, "test", Namespace.ofCatalog("test_metalake"), "this is test"); - - CatalogPO expectedCatalogPO = createCatalogPO(1L, "test", 1L, "this is test"); - - CatalogPO actualCatalogPO = POConverters.toCatalogPO(catalog, 1L); - - // Assert - assertEquals(expectedCatalogPO.getCatalogId(), actualCatalogPO.getCatalogId()); - assertEquals(expectedCatalogPO.getMetalakeId(), actualCatalogPO.getMetalakeId()); - assertEquals(expectedCatalogPO.getCatalogName(), actualCatalogPO.getCatalogName()); - assertEquals(expectedCatalogPO.getType(), actualCatalogPO.getType()); - assertEquals(expectedCatalogPO.getProvider(), actualCatalogPO.getProvider()); - assertEquals(expectedCatalogPO.getCatalogComment(), actualCatalogPO.getCatalogComment()); - assertEquals(expectedCatalogPO.getProperties(), actualCatalogPO.getProperties()); - assertEquals(expectedCatalogPO.getAuditInfo(), actualCatalogPO.getAuditInfo()); - } - private static BaseMetalake createMetalake(Long id, String name, String comment) { AuditInfo auditInfo = AuditInfo.builder().withCreator("creator").withCreateTime(FIX_INSTANT).build(); diff --git a/dev/docker/trino/Dockerfile b/dev/docker/trino/Dockerfile index 46d56e23c95..16790b0f4d7 100644 --- a/dev/docker/trino/Dockerfile +++ b/dev/docker/trino/Dockerfile @@ -59,11 +59,11 @@ RUN if [ "$IMAGE_NAME" = "datastrato/trino" ] ; then \ fi - -# Use ARGs to update trino conf and start Trino server -ARG GRAVITINO_HOST_IP -ARG GRAVITINO_HOST_PORT -ARG GRAVITINO_METALAKE_NAME -ARG HIVE_HOST_IP +# The following ENV are needed to update configuration file before starting Trino server, you need to +# pass these ENV variables to the container when starting it like `docker run --env GRAVITINO_HOST_IP=xxx --env GRAVITINO_HOST_PORT=xxxx` +#ENV GRAVITINO_HOST_IP +#ENV GRAVITINO_HOST_PORT +#ENV GRAVITINO_METALAKE_NAME +#ENV HADOOP_USER_NAME ENTRYPOINT ["/bin/bash", "-c", "/etc/trino/update-trino-conf.sh && /usr/lib/trino/bin/run-trino"] diff --git a/dev/docker/trino/conf/update-trino-conf.sh b/dev/docker/trino/conf/update-trino-conf.sh index d800883d485..890dc38ced4 100755 --- a/dev/docker/trino/conf/update-trino-conf.sh +++ b/dev/docker/trino/conf/update-trino-conf.sh @@ -13,3 +13,10 @@ sed "s/GRAVITINO_HOST_IP:GRAVITINO_HOST_PORT/${GRAVITINO_HOST_IP}:${GRAVITINO_HO # Update `gravitino.metalake = GRAVITINO_METALAKE_NAME` in the `conf/catalog/gravitino.properties` sed "s/GRAVITINO_METALAKE_NAME/${GRAVITINO_METALAKE_NAME}/g" "${trino_conf_dir}/catalog/gravitino.properties.tmp" > "${trino_conf_dir}/catalog/gravitino.properties" rm "${trino_conf_dir}/catalog/gravitino.properties.tmp" + +# Check the number of Gravitino connector plugins present in the Trino plugin directory +num_of_gravitino_connector=$(ls /usr/lib/trino/plugin/gravitino | grep gravitino-trino-connector-* | wc -l) +if [[ "${num_of_gravitino_connector}" -ne 1 ]]; then + echo "Multiple versions of the Gravitino connector plugin found or none present." + exit 1 +fi \ No newline at end of file diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index 6c3436a5f8a..116d81f6ae4 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -139,6 +139,9 @@ You can use this image to test Trino. Changelog +- gravitino-ci-trino:0.1.5 + - Add check for the version of gravitino-trino-connector + - gravitino-ci-trino:0.1.4 - Change `-Xmx1G` to `-Xmx2G` in the config file `/etc/trino/jvm.config` diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 952469e33be..e4231462fb9 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -267,6 +267,26 @@ tasks.test { if (skipITs) { exclude("**/integration/test/**") } else { + // Get current project version + val version = project.version.toString() + println("Current project version: $version") + // Check whether this module has already built + val buildDir = project.buildDir + if (!buildDir.exists()) { + dependsOn(":trino-connector:jar") + } else { + // Check the version gravitino related jars in build equal to the current project version + val gravitinoJars = buildDir.resolve("libs").listFiles { _, name -> name.startsWith("gravitino") }?.filter { + val jarVersion = name.substringAfterLast("-").substringBeforeLast(".") + jarVersion != version + } + + if (gravitinoJars != null && gravitinoJars.isNotEmpty()) { + delete(project(":trino-connector").buildDir) + dependsOn(":trino-connector:jar") + } + } + doFirst { printDockerCheckInfo() @@ -297,7 +317,7 @@ tasks.test { // Gravitino CI Docker image environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.8") - environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.3") + environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.5") // Change poll image pause time from 30s to 60s environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60") diff --git a/integration-test/trino-it/docker-compose.yaml b/integration-test/trino-it/docker-compose.yaml index 19fc71ca64a..ed783265b12 100644 --- a/integration-test/trino-it/docker-compose.yaml +++ b/integration-test/trino-it/docker-compose.yaml @@ -64,7 +64,7 @@ services: retries: 5 trino: - image: datastrato/gravitino-ci-trino:0.1.4 + image: datastrato/gravitino-ci-trino:0.1.5 networks: - trino-net container_name: trino-ci-trino diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java index 663dad459c5..2fa6b286c3e 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java @@ -20,6 +20,7 @@ public GravitinoSplitManager(ConnectorSplitManager internalSplitManager) { this.internalSplitManager = internalSplitManager; } + @Override public ConnectorSplitSource getSplits( ConnectorTransactionHandle transaction, ConnectorSession session, diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index 2fc5ffb4d4d..28cc2762088 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -30,6 +30,7 @@ public HiveConnectorAdapter() { this.catalogConverter = new HiveCatalogPropertyConverter(); } + @Override public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { Map config = new HashMap<>(); @@ -69,6 +70,7 @@ public List> getSchemaProperties() { return propertyMetadata.getSchemaPropertyMetadata(); } + @Override public CatalogConnectorMetadataAdapter getMetadataAdapter() { // TODO yuhui Need to improve schema table and column properties return new HiveMetadataAdapter( diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java index f7e276d028b..4176ce901b1 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/iceberg/IcebergConnectorAdapter.java @@ -28,6 +28,7 @@ public IcebergConnectorAdapter() { this.catalogConverter = new IcebergCatalogPropertyConverter(); } + @Override public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { Map config = new HashMap<>(); @@ -42,6 +43,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog return config; } + @Override public CatalogConnectorMetadataAdapter getMetadataAdapter() { // TODO yuhui Need to improve schema table and column properties return new IcebergMetadataAdapter(getSchemaProperties(), getTableProperties(), emptyList()); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java index 06941d90ef4..ee7b7bae7fd 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLConnectorAdapter.java @@ -30,6 +30,7 @@ public MySQLConnectorAdapter() { this.propertyMetadata = new MySQLPropertyMeta(); } + @Override public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { Map config = new HashMap<>(); @@ -44,6 +45,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog return config; } + @Override public CatalogConnectorMetadataAdapter getMetadataAdapter() { // TODO yuhui Need to improve schema table and column properties return new MySQLMetadataAdapter(getSchemaProperties(), getTableProperties(), emptyList()); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLMetadataAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLMetadataAdapter.java index 48f9b795eb1..04527dc74fd 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLMetadataAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/mysql/MySQLMetadataAdapter.java @@ -44,6 +44,7 @@ public Map toTrinoTableProperties(Map properties } /** Transform trino ConnectorTableMetadata to gravitino table metadata */ + @Override public GravitinoTable createTable(ConnectorTableMetadata tableMetadata) { String tableName = tableMetadata.getTableSchema().getTable().getTableName(); String schemaName = tableMetadata.getTableSchema().getTable().getSchemaName(); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java index 4b9b90e1190..feb12fdb0c5 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLConnectorAdapter.java @@ -24,6 +24,7 @@ public PostgreSQLConnectorAdapter() { this.catalogConverter = new JDBCCatalogPropertyConverter(); } + @Override public Map buildInternalConnectorConfig(GravitinoCatalog catalog) throws Exception { Map config = new HashMap<>(); @@ -38,6 +39,7 @@ public Map buildInternalConnectorConfig(GravitinoCatalog catalog return config; } + @Override public CatalogConnectorMetadataAdapter getMetadataAdapter() { // TODO yuhui Need to improve schema table and column properties return new PostgreSQLMetadataAdapter(getSchemaProperties(), getTableProperties(), emptyList());