diff --git a/catalog-lakehouse/build.gradle.kts b/catalog-lakehouse/build.gradle.kts new file mode 100644 index 00000000000..f067d30709d --- /dev/null +++ b/catalog-lakehouse/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +description = "catalog-lakehouse" + +plugins { + `maven-publish` + id("java") + id("idea") + id("com.diffplug.spotless") +} + +dependencies { + implementation(project(":common")) + implementation(project(":core")) + implementation(libs.jackson.databind) + implementation(libs.jackson.annotations) + implementation(libs.jackson.datatype.jdk8) + implementation(libs.jackson.datatype.jsr310) + implementation(libs.guava) + implementation(libs.bundles.log4j) + implementation(libs.bundles.jetty) + implementation(libs.bundles.jersey) + implementation(libs.bundles.iceberg) + + compileOnly(libs.lombok) + annotationProcessor(libs.lombok) + + testImplementation(libs.slf4j.jdk14) + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testRuntimeOnly(libs.junit.jupiter.engine) + testImplementation(libs.mockito.core) + testImplementation(libs.jersey.test.framework.core) { + exclude(group = "org.junit.jupiter") + } + testImplementation(libs.jersey.test.framework.provider.jetty) { + exclude(group = "org.junit.jupiter") + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/ops/IcebergTableOps.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/ops/IcebergTableOps.java new file mode 100644 index 00000000000..a2c30b77de6 --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/ops/IcebergTableOps.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.ops; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.utils.IcebergCatalogUtil; +import com.google.common.base.Preconditions; +import java.util.Optional; +import javax.ws.rs.NotSupportedException; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; +import org.apache.iceberg.rest.responses.CreateNamespaceResponse; +import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IcebergTableOps { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergTableOps.class); + + private Catalog catalog; + private SupportsNamespaces asNamespaceCatalog; + private final String DEFAULT_ICEBERG_CATALOG_TYPE = "memory"; + + public IcebergTableOps() { + catalog = IcebergCatalogUtil.loadIcebergCatalog(DEFAULT_ICEBERG_CATALOG_TYPE); + if (catalog instanceof SupportsNamespaces) { + asNamespaceCatalog = (SupportsNamespaces) catalog; + } + } + + private void validateNamespace(Optional namespace) { + namespace.ifPresent( + n -> + Preconditions.checkArgument( + n.toString().isEmpty() == false, "Namespace couldn't be empty")); + if (asNamespaceCatalog == null) { + throw new NotSupportedException("The underlying catalog doesn't support namespace operation"); + } + } + + public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { + validateNamespace(Optional.of(request.namespace())); + return CatalogHandlers.createNamespace(asNamespaceCatalog, request); + } + + public void dropNamespace(Namespace namespace) { + validateNamespace(Optional.of(namespace)); + CatalogHandlers.dropNamespace(asNamespaceCatalog, namespace); + } + + public GetNamespaceResponse loadNamespace(Namespace namespace) { + validateNamespace(Optional.of(namespace)); + return CatalogHandlers.loadNamespace(asNamespaceCatalog, namespace); + } + + public ListNamespacesResponse listNamespace(Namespace parent) { + validateNamespace(Optional.empty()); + return CatalogHandlers.listNamespaces(asNamespaceCatalog, parent); + } + + public UpdateNamespacePropertiesResponse updateNamespaceProperties( + Namespace namespace, UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest) { + validateNamespace(Optional.of(namespace)); + return CatalogHandlers.updateNamespaceProperties( + asNamespaceCatalog, namespace, updateNamespacePropertiesRequest); + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/utils/IcebergCatalogUtil.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/utils/IcebergCatalogUtil.java new file mode 100644 index 00000000000..afcf0463840 --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/utils/IcebergCatalogUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.utils; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.inmemory.InMemoryCatalog; + +public class IcebergCatalogUtil { + + private static InMemoryCatalog loadMemoryCatalog() { + InMemoryCatalog memoryCatalog = new InMemoryCatalog(); + Map properties = new HashMap<>(); + + memoryCatalog.initialize("memory", properties); + return memoryCatalog; + } + + public static Catalog loadIcebergCatalog(String catalogType) { + switch (catalogType.toLowerCase(Locale.ENGLISH)) { + case "memory": + return loadMemoryCatalog(); + // todo: add hive, jdbc catalog + default: + throw new RuntimeException( + catalogType + + " catalog is not supported yet, supported catalogs: [memory]" + + catalogType); + } + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergExceptionMapper.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergExceptionMapper.java new file mode 100644 index 00000000000..cfc1e570267 --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergExceptionMapper.java @@ -0,0 +1,65 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.exceptions.NamespaceNotEmptyException; +import org.apache.iceberg.exceptions.NoSuchIcebergTableException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.exceptions.UnprocessableEntityException; +import org.apache.iceberg.exceptions.ValidationException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Provider +public class IcebergExceptionMapper implements ExceptionMapper { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergExceptionMapper.class); + + private static final Map, Integer> EXCEPTION_ERROR_CODES = + ImmutableMap., Integer>builder() + .put(IllegalArgumentException.class, 400) + .put(ValidationException.class, 400) + .put(NamespaceNotEmptyException.class, 400) + .put(NotAuthorizedException.class, 401) + .put(ForbiddenException.class, 403) + .put(NoSuchNamespaceException.class, 404) + .put(NoSuchTableException.class, 404) + .put(NoSuchIcebergTableException.class, 404) + .put(UnsupportedOperationException.class, 406) + .put(AlreadyExistsException.class, 409) + .put(CommitFailedException.class, 409) + .put(UnprocessableEntityException.class, 422) + .put(CommitStateUnknownException.class, 500) + .build(); + + @Override + public Response toResponse(Exception ex) { + int status = + EXCEPTION_ERROR_CODES.getOrDefault( + ex.getClass(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); + if (status == Status.INTERNAL_SERVER_ERROR.getStatusCode()) { + LOG.warn("Iceberg REST server unexpected exception:", ex); + } else { + LOG.info( + "Iceberg REST server error maybe caused by user request, response http status: {}, exception: {}, exception message: {}", + ex.getClass(), + ex.getMessage()); + } + return IcebergRestUtils.errorResponse(ex, status); + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergObjectMapperProvider.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergObjectMapperProvider.java new file mode 100644 index 00000000000..57efe362777 --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergObjectMapperProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web; + +import com.datastrato.graviton.json.JsonUtils; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import javax.ws.rs.ext.ContextResolver; +import javax.ws.rs.ext.Provider; +import org.apache.iceberg.rest.RESTSerializers; + +@Provider +public class IcebergObjectMapperProvider implements ContextResolver { + + @Override + public ObjectMapper getContext(Class type) { + ObjectMapper mapper = JsonUtils.objectMapper(); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setPropertyNamingStrategy(new PropertyNamingStrategy.KebabCaseStrategy()); + RESTSerializers.registerAll(mapper); + return mapper; + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergRestUtils.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergRestUtils.java new file mode 100644 index 00000000000..fbc2fd6e4fc --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/IcebergRestUtils.java @@ -0,0 +1,37 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import org.apache.iceberg.rest.responses.ErrorResponse; + +public class IcebergRestUtils { + + private IcebergRestUtils() {} + + public static Response ok(T t) { + return Response.status(Response.Status.OK).entity(t).type(MediaType.APPLICATION_JSON).build(); + } + + public static Response noContent() { + return Response.status(Status.NO_CONTENT).build(); + } + + public static Response errorResponse(Exception ex, int httpStatus) { + ErrorResponse errorResponse = + ErrorResponse.builder() + .responseCode(httpStatus) + .withType(ex.getClass().getSimpleName()) + .withMessage(ex.getMessage()) + .withStackTrace(ex) + .build(); + return Response.status(httpStatus) + .entity(errorResponse) + .type(MediaType.APPLICATION_JSON) + .build(); + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/rest/IcebergConfig.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/rest/IcebergConfig.java new file mode 100644 index 00000000000..ea480180765 --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/rest/IcebergConfig.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.rest; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.IcebergRestUtils; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.iceberg.rest.responses.ConfigResponse; + +@Path("/v1/config") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class IcebergConfig { + + @Context private HttpServletRequest httpRequest; + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response getConfig() { + ConfigResponse response = ConfigResponse.builder().build(); + return IcebergRestUtils.ok(response); + } +} diff --git a/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/rest/IcebergNamespaceOperations.java b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/rest/IcebergNamespaceOperations.java new file mode 100644 index 00000000000..0fd3d0e0202 --- /dev/null +++ b/catalog-lakehouse/src/main/java/com/datastrato/graviton/catalog/lakehouse/iceberg/iceberg/web/rest/IcebergNamespaceOperations.java @@ -0,0 +1,96 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.rest; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.ops.IcebergTableOps; +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.IcebergRestUtils; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; +import org.apache.iceberg.rest.responses.CreateNamespaceResponse; +import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Path("/v1/namespaces") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class IcebergNamespaceOperations { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergNamespaceOperations.class); + + private IcebergTableOps icebergTableOps; + + @Context private HttpServletRequest httpRequest; + + @Inject + public IcebergNamespaceOperations(IcebergTableOps icebergTableOps) { + this.icebergTableOps = icebergTableOps; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response listNamespaces(@DefaultValue("") @QueryParam("parent") String parent) { + Namespace parentNamespace = + parent.isEmpty() ? Namespace.empty() : RESTUtil.decodeNamespace(parent); + ListNamespacesResponse response = icebergTableOps.listNamespace(parentNamespace); + return IcebergRestUtils.ok(response); + } + + @GET + @Path("{namespace}") + @Produces(MediaType.APPLICATION_JSON) + public Response loadNamespace(@PathParam("namespace") String namespace) { + GetNamespaceResponse getNamespaceResponse = + icebergTableOps.loadNamespace(RESTUtil.decodeNamespace(namespace)); + return IcebergRestUtils.ok(getNamespaceResponse); + } + + @DELETE + @Path("{namespace}") + @Produces(MediaType.APPLICATION_JSON) + public Response dropNamespace(@PathParam("namespace") String namespace) { + // todo check if table exists in namespace after table ops is added + LOG.info("Drop Iceberg namespace: {}", namespace); + icebergTableOps.dropNamespace(RESTUtil.decodeNamespace(namespace)); + return IcebergRestUtils.noContent(); + } + + @POST + @Produces(MediaType.APPLICATION_JSON) + public Response createNamespace(CreateNamespaceRequest namespaceRequest) { + LOG.info("Create Iceberg namespace: {}", namespaceRequest); + CreateNamespaceResponse response = icebergTableOps.createNamespace(namespaceRequest); + return IcebergRestUtils.ok(response); + } + + @POST + @Path("{namespace}") + @Produces(MediaType.APPLICATION_JSON) + public Response updateNamespace( + @PathParam("namespace") String namespace, UpdateNamespacePropertiesRequest request) { + LOG.info("Update Iceberg namespace: {}, request: {}", namespace, request); + UpdateNamespacePropertiesResponse response = + icebergTableOps.updateNamespaceProperties(RESTUtil.decodeNamespace(namespace), request); + return IcebergRestUtils.ok(response); + } +} diff --git a/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java new file mode 100644 index 00000000000..d823066e48d --- /dev/null +++ b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.graviton.lakehouse.iceberg.utils; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.utils.IcebergCatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestIcebergCatalogUtil { + + @Test + void testLoadCatalog() { + Catalog catalog; + + catalog = IcebergCatalogUtil.loadIcebergCatalog("memory"); + Assertions.assertTrue(catalog instanceof InMemoryCatalog); + + catalog = IcebergCatalogUtil.loadIcebergCatalog("MEMORY"); + Assertions.assertTrue(catalog instanceof InMemoryCatalog); + + Assertions.assertThrowsExactly( + RuntimeException.class, + () -> { + IcebergCatalogUtil.loadIcebergCatalog("other"); + }); + } +} diff --git a/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java new file mode 100644 index 00000000000..c6394699d9f --- /dev/null +++ b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/IcebergRestTestUtil.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.graviton.lakehouse.iceberg.web.rest; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.IcebergExceptionMapper; +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.IcebergObjectMapperProvider; +import org.glassfish.jersey.jackson.JacksonFeature; +import org.glassfish.jersey.server.ResourceConfig; + +public class IcebergRestTestUtil { + + private static String PREFIX = "v1"; + public static String CONFIG_PATH = PREFIX + "/config"; + public static String NAMESPACE_PATH = PREFIX + "/namespaces"; + + public static ResourceConfig getIcebergResourceConfig(Class c) { + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(c); + resourceConfig.register(IcebergObjectMapperProvider.class).register(JacksonFeature.class); + resourceConfig.register(IcebergExceptionMapper.class); + return resourceConfig; + } +} diff --git a/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/TestIcebergConfig.java b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/TestIcebergConfig.java new file mode 100644 index 00000000000..10df18b6360 --- /dev/null +++ b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/TestIcebergConfig.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.graviton.lakehouse.iceberg.web.rest; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.IcebergObjectMapperProvider; +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.rest.IcebergConfig; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.glassfish.jersey.test.JerseyTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestIcebergConfig extends JerseyTest { + + @Override + protected Application configure() { + return IcebergRestTestUtil.getIcebergResourceConfig(IcebergConfig.class); + } + + @Test + public void testConfig() { + Response resp = + target(IcebergRestTestUtil.CONFIG_PATH) + .register(IcebergObjectMapperProvider.class) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept(MediaType.APPLICATION_JSON_TYPE) + .get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ConfigResponse response = resp.readEntity(ConfigResponse.class); + Assertions.assertEquals(response.defaults().size(), 0); + Assertions.assertEquals(response.overrides().size(), 0); + } +} diff --git a/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java new file mode 100644 index 00000000000..83619545082 --- /dev/null +++ b/catalog-lakehouse/src/test/java/com/datastrato/graviton/lakehouse/iceberg/web/rest/TestIcebergNamespaceOperations.java @@ -0,0 +1,258 @@ +/* + * Copyright 2023 Datastrato. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.graviton.lakehouse.iceberg.web.rest; + +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.ops.IcebergTableOps; +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.IcebergObjectMapperProvider; +import com.datastrato.graviton.catalog.lakehouse.iceberg.iceberg.web.rest.IcebergNamespaceOperations; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; +import org.apache.iceberg.rest.responses.CreateNamespaceResponse; +import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestIcebergNamespaceOperations extends JerseyTest { + + private final Map properties = ImmutableMap.of("a", "b"); + private final Map updatedProperties = ImmutableMap.of("b", "c"); + + @Override + protected Application configure() { + ResourceConfig resourceConfig = + IcebergRestTestUtil.getIcebergResourceConfig(IcebergNamespaceOperations.class); + + IcebergTableOps icebergTableOps = new IcebergTableOps(); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bind(icebergTableOps).to(IcebergTableOps.class).ranked(2); + } + }); + + return resourceConfig; + } + + private Builder getNamespaceClientBuilder() { + return getNamespaceClientBuilder(Optional.empty(), Optional.empty()); + } + + private Builder getNamespaceClientBuilder(Optional namespace) { + return getNamespaceClientBuilder(namespace, Optional.empty()); + } + + private Builder getNamespaceClientBuilder( + Optional namespace, Optional> queryParam) { + String path = + Joiner.on("/") + .skipNulls() + .join(IcebergRestTestUtil.NAMESPACE_PATH, namespace.orElseGet(() -> null)); + WebTarget target = target(path); + if (queryParam.isPresent()) { + Map m = queryParam.get(); + for (Entry entry : m.entrySet()) { + target = target.queryParam(entry.getKey(), entry.getValue()); + } + } + + return target + .register(IcebergObjectMapperProvider.class) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept(MediaType.APPLICATION_JSON_TYPE); + } + + private Response doCreateNamespace(String... name) { + CreateNamespaceRequest request = + CreateNamespaceRequest.builder() + .withNamespace(Namespace.of(name)) + .setProperties(properties) + .build(); + return getNamespaceClientBuilder() + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + } + + private Response doListNamespace(Optional parent) { + Optional> queryParam = + parent.isPresent() + ? Optional.of(ImmutableMap.of("parent", parent.get())) + : Optional.empty(); + return getNamespaceClientBuilder(Optional.empty(), queryParam).get(); + } + + private Response doUpdateNamespace(String name) { + UpdateNamespacePropertiesRequest request = + UpdateNamespacePropertiesRequest.builder() + .removeAll(Arrays.asList("a", "a1")) + .updateAll(updatedProperties) + .build(); + return getNamespaceClientBuilder(Optional.of(name)) + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + } + + private Response doLoadNamespace(String name) { + return getNamespaceClientBuilder(Optional.of(name)).get(); + } + + private Response doDropNamespace(String name) { + return getNamespaceClientBuilder(Optional.of(name)).delete(); + } + + private void verifyLoadNamespaceFail(int status, String name) { + Response response = doLoadNamespace(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyLoadNamespaceSucc(String name) { + Response response = doLoadNamespace(name); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + GetNamespaceResponse r = response.readEntity(GetNamespaceResponse.class); + Assertions.assertEquals(name, r.namespace().toString()); + Assertions.assertEquals(properties, r.properties()); + } + + private void verifyDropNamespaceSucc(String name) { + Response response = doDropNamespace(name); + Assertions.assertEquals(Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } + + private void verifyDropNamespaceFail(int status, String name) { + Response response = doDropNamespace(name); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyCreateNamespaceSucc(String... name) { + Response response = doCreateNamespace(name); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + CreateNamespaceResponse namespaceResponse = response.readEntity(CreateNamespaceResponse.class); + Assertions.assertTrue(namespaceResponse.namespace().equals(Namespace.of(name))); + + Assertions.assertEquals(namespaceResponse.properties(), properties); + } + + private void verifyCreateNamespaceFail(int statusCode, String... name) { + Response response = doCreateNamespace(name); + Assertions.assertEquals(statusCode, response.getStatus()); + } + + @Test + void testCreateNamespace() { + verifyCreateNamespaceSucc("create_foo1"); + + // Already Exists Exception + verifyCreateNamespaceFail(409, "create_foo1"); + + // multi level namespaces + verifyCreateNamespaceSucc("create_foo2", "create_foo3"); + + verifyCreateNamespaceFail(400, ""); + } + + @Test + void testLoadNamespace() { + verifyCreateNamespaceSucc("load_foo1"); + verifyLoadNamespaceSucc("load_foo1"); + // load a schema not exists + verifyLoadNamespaceFail(404, "load_foo2"); + } + + @Test + void testDropNamespace() { + verifyCreateNamespaceSucc("drop_foo1"); + verifyDropNamespaceSucc("drop_foo1"); + verifyLoadNamespaceFail(404, "drop_foo1"); + + // drop fail, no such namespace + verifyDropNamespaceFail(404, "drop_foo2"); + // jersery route failed + verifyDropNamespaceFail(500, ""); + } + + private void dropAllExistingNamespace() { + Response response = doListNamespace(Optional.empty()); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + ListNamespacesResponse r = response.readEntity(ListNamespacesResponse.class); + r.namespaces().forEach(n -> doDropNamespace(n.toString())); + } + + private void verifyListNamespaceFail(Optional parent, int status) { + Response response = doListNamespace(parent); + Assertions.assertEquals(status, response.getStatus()); + } + + private void verifyListNamespaceSucc(Optional parent, List schemas) { + Response response = doListNamespace(parent); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + ListNamespacesResponse r = response.readEntity(ListNamespacesResponse.class); + List ns = r.namespaces().stream().map(n -> n.toString()).collect(Collectors.toList()); + Assertions.assertEquals(schemas, ns); + } + + @Test + void testListNamespace() { + dropAllExistingNamespace(); + verifyListNamespaceSucc(Optional.empty(), Arrays.asList()); + + doCreateNamespace("list_foo1"); + doCreateNamespace("list_foo2"); + doCreateNamespace("list_foo3", "a"); + doCreateNamespace("list_foo3", "b"); + + verifyListNamespaceSucc(Optional.empty(), Arrays.asList("list_foo1", "list_foo2", "list_foo3")); + verifyListNamespaceSucc(Optional.of("list_foo3"), Arrays.asList("list_foo3.a", "list_foo3.b")); + + verifyListNamespaceFail(Optional.of("list_fooxx"), 404); + } + + private void verifyUpdateNamespaceSucc(String name) { + Response response = doUpdateNamespace(name); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + + UpdateNamespacePropertiesResponse r = + response.readEntity(UpdateNamespacePropertiesResponse.class); + Assertions.assertEquals(r.removed(), Arrays.asList("a")); + Assertions.assertEquals(r.missing(), Arrays.asList("a1")); + Assertions.assertEquals(r.updated(), Arrays.asList("b")); + } + + private void verifyUpdateNamespaceFail(int status, String name) { + Response response = doUpdateNamespace(name); + Assertions.assertEquals(status, response.getStatus()); + } + + @Test + void testUpdateNamespace() { + verifyCreateNamespaceSucc("update_foo1"); + verifyUpdateNamespaceSucc("update_foo1"); + + verifyUpdateNamespaceFail(404, "update_foo2"); + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3107a9d02d7..be229663d9f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,6 +24,7 @@ commons-lang3 = "3.12.0" commons-io = "2.11.0" caffeine = "2.9.3" rocksdbjni = "7.7.3" +iceberg = '1.3.1' protobuf-plugin = "0.9.2" spotless-plugin = '6.11.0' @@ -76,11 +77,13 @@ commons-lang3 = { group = "org.apache.commons", name = "commons-lang3", version. commons-io = { group = "commons-io", name = "commons-io", version.ref = "commons-io" } caffeine = { group = "com.github.ben-manes.caffeine", name = "caffeine", version.ref = "caffeine" } rocksdbjni = { group = "org.rocksdb", name = "rocksdbjni", version.ref = "rocksdbjni" } +iceberg-core = { group = "org.apache.iceberg", name = "iceberg-core", version.ref = "iceberg" } [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"] jetty = ["jetty-server", "jetty-servlet"] jersey = ["jersey-server", "jersey-container-servlet-core", "jersey-container-jetty-http", "jersey-media-json-jackson", "jersey-hk2"] +iceberg = ["iceberg-core"] [plugins] protobuf = { id = "com.google.protobuf", version.ref = "protobuf-plugin" } @@ -88,4 +91,4 @@ spotless = { id = "com.diffplug.spotless", version.ref = "spotless-plugin" } gradle-extensions = { id = "com.github.vlsi.gradle-extensions", version.ref = "gradle-extensions-plugin" } publish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "publish-plugin" } rat = { id = "org.nosphere.apache.rat", version.ref = "rat-plugin" } -shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow-plugin" } \ No newline at end of file +shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow-plugin" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 746e40a7236..4fffdbad18e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,4 +4,4 @@ */ rootProject.name = "graviton" -include("api", "client-java", "common", "core", "meta", "server", "catalog-hive", "integration-test") +include("api", "client-java", "common", "core", "meta", "server", "catalog-hive", "integration-test", "catalog-lakehouse")