From e10a8dc51b7d7246b66a0cc8e277272ee05e2435 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 26 Jul 2024 10:55:29 +0800 Subject: [PATCH 01/13] add get fileset context api --- .../file/BaseFilesetDataOperationCtx.java | 90 +++++++++++ .../apache/gravitino/file/FilesetCatalog.java | 11 ++ .../apache/gravitino/file/FilesetContext.java | 38 +++++ .../file/FilesetDataOperationCtx.java | 41 +++++ .../hadoop/HadoopCatalogOperations.java | 34 ++++ .../catalog/hadoop/HadoopFilesetContext.java | 48 ++++++ .../hadoop/SecureHadoopCatalogOperations.java | 8 + .../hadoop/TestHadoopCatalogOperations.java | 52 +++++++ .../gravitino/client/FilesetCatalog.java | 35 +++++ .../gravitino/client/TestFilesetCatalog.java | 46 ++++++ .../gravitino/dto/file/FilesetContextDTO.java | 57 +++++++ .../requests/GetFilesetContextRequest.java | 58 +++++++ .../dto/responses/FilesetContextResponse.java | 63 ++++++++ .../gravitino/dto/util/DTOConverters.java | 15 ++ .../dto/file/TestFilesetContextDTO.java | 72 +++++++++ .../TestGetFilesetContextRequest.java | 42 +++++ .../dto/responses/TestResponses.java | 29 ++++ .../gravitino/dto/util/TestDTOConverters.java | 73 +++++++++ .../catalog/FilesetNormalizeDispatcher.java | 9 ++ .../catalog/FilesetOperationDispatcher.java | 18 +++ .../connector/BaseFilesetContext.java | 116 ++++++++++++++ .../listener/FilesetEventDispatcher.java | 19 +++ .../api/event/GetFilesetContextEvent.java | 48 ++++++ .../event/GetFilesetContextFailureEvent.java | 52 +++++++ .../apache/gravitino/TestFilesetContext.java | 50 ++++++ .../TestFilesetOperationDispatcher.java | 46 ++++++ .../connector/TestCatalogOperations.java | 43 ++++- docs/open-api/filesets.yaml | 105 +++++++++++++ .../integration/test/client/FilesetIT.java | 147 ++++++++++++++++++ .../server/web/rest/FilesetOperations.java | 39 +++++ .../web/rest/TestFilesetOperations.java | 54 +++++++ 31 files changed, 1557 insertions(+), 1 deletion(-) create mode 100644 api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java create mode 100644 api/src/main/java/org/apache/gravitino/file/FilesetContext.java create mode 100644 api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java create mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java create mode 100644 common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java create mode 100644 common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java create mode 100644 core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java create mode 100644 core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java create mode 100644 core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java create mode 100644 core/src/test/java/org/apache/gravitino/TestFilesetContext.java create mode 100644 integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java diff --git a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java new file mode 100644 index 00000000000..59cbab3d18d --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.file; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; + +/** Base implementation of {@link FilesetDataOperationCtx}. */ +public class BaseFilesetDataOperationCtx implements FilesetDataOperationCtx { + private String subPath; + private String operation; + private String clientType; + + @Override + public String subPath() { + return subPath; + } + + @Override + public String operation() { + return operation; + } + + @Override + public String clientType() { + return clientType; + } + + public static class Builder { + private BaseFilesetDataOperationCtx context; + /** Creates a new instance of {@link BaseFilesetDataOperationCtx.Builder}. */ + private Builder() { + context = new BaseFilesetDataOperationCtx(); + } + + public BaseFilesetDataOperationCtx.Builder withSubPath(String subPath) { + context.subPath = subPath; + return this; + } + + public BaseFilesetDataOperationCtx.Builder withOperation(String operation) { + context.operation = operation; + return this; + } + + public BaseFilesetDataOperationCtx.Builder withClientType(String clientType) { + context.clientType = clientType; + return this; + } + + private void validate() { + Preconditions.checkArgument(context.subPath != null, "subPath cannot be null"); + Preconditions.checkArgument( + StringUtils.isNotBlank(context.operation), "operation is required"); + Preconditions.checkArgument( + StringUtils.isNotBlank(context.clientType), "clientType is required"); + } + + public BaseFilesetDataOperationCtx build() { + validate(); + return context; + } + } + + /** + * Creates a new instance of {@link Builder}. + * + * @return The new instance. + */ + public static Builder builder() { + return new Builder(); + } +} diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java b/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java index bdd4d02ff2e..6959eff74d1 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java +++ b/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java @@ -120,4 +120,15 @@ Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) * @return true If the fileset is dropped, false the fileset did not exist. */ boolean dropFileset(NameIdentifier ident); + + /** + * Get a fileset context. + * + * @param ident A fileset identifier. + * @param ctx The data operation context. + * @return The fileset context. + * @throws NoSuchFilesetException If the fileset does not exist. + */ + FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + throws NoSuchFilesetException; } diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetContext.java b/api/src/main/java/org/apache/gravitino/file/FilesetContext.java new file mode 100644 index 00000000000..4a62fa9d282 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/file/FilesetContext.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.file; + +import org.apache.gravitino.annotation.Evolving; + +/** + * An interface representing a fileset context with an existing fileset {@link Fileset}. This + * interface defines some contextual information related to Fileset that can be passed. + * + *

{@link FilesetContext} defines the basic properties of a fileset context object. A catalog + * implementation with {@link FilesetCatalog} should implement this interface. + */ +@Evolving +public interface FilesetContext { + /** @return The fileset object. */ + Fileset fileset(); + + /** @return The actual storage path after processing. */ + String actualPath(); +} diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java new file mode 100644 index 00000000000..8e2b1997019 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.file; + +import org.apache.gravitino.annotation.Evolving; + +/** + * An interface representing a fileset data operation context. This interface defines some + * information need to report to the server. + * + *

{@link FilesetDataOperationCtx} defines the basic properties of a fileset data operation + * context object. + */ +@Evolving +public interface FilesetDataOperationCtx { + /** @return The sub path which is operated by the data operation . */ + String subPath(); + + /** @return The data operation type. */ + String operation(); + + /** @return The client type of the data operation. */ + String clientType(); +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index eed8da9a144..decb63bbaa4 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -40,6 +40,7 @@ import org.apache.gravitino.Schema; import org.apache.gravitino.SchemaChange; import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.catalog.EntityCombinedFileset; import org.apache.gravitino.connector.CatalogInfo; import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; @@ -55,6 +56,8 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.FilesetEntity; import org.apache.gravitino.meta.SchemaEntity; @@ -354,6 +357,37 @@ public boolean dropFileset(NameIdentifier ident) { } } + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + throws NoSuchFilesetException { + // TODO we need move some check logics in the Hadoop / Python GVFS to here. + String subPath = ctx.subPath(); + Preconditions.checkArgument(subPath != null, "subPath must not be null"); + + Fileset fileset = loadFileset(ident); + + String storageLocation = fileset.storageLocation(); + String actualPath; + // subPath cannot be null, so we only need check if it is blank + if (StringUtils.isBlank(subPath)) { + actualPath = storageLocation; + } else { + actualPath = + subPath.startsWith("/") + ? String.format("%s%s", storageLocation, subPath) + : String.format("%s/%s", storageLocation, subPath); + } + return HadoopFilesetContext.builder() + .withFileset( + EntityCombinedFileset.of(fileset) + .withHiddenPropertiesSet( + fileset.properties().keySet().stream() + .filter(propertiesMetadata.filesetPropertiesMetadata()::isHiddenProperty) + .collect(Collectors.toSet()))) + .withActualPath(actualPath) + .build(); + } + @Override public NameIdentifier[] listSchemas(Namespace namespace) throws NoSuchCatalogException { try { diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java new file mode 100644 index 00000000000..0fea8136b4e --- /dev/null +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.catalog.hadoop; + +import org.apache.gravitino.connector.BaseFilesetContext; + +public class HadoopFilesetContext extends BaseFilesetContext { + public static class Builder + extends BaseFilesetContext.BaseFilesetContextBuilder< + HadoopFilesetContext.Builder, HadoopFilesetContext> { + /** Creates a new instance of {@link HadoopFilesetContext.Builder}. */ + private Builder() {} + + @Override + protected HadoopFilesetContext internalBuild() { + HadoopFilesetContext context = new HadoopFilesetContext(); + context.fileset = fileset; + context.actualPath = actualPath; + return context; + } + } + + /** + * Creates a new instance of {@link HadoopFilesetContext.Builder}. + * + * @return The new instance. + */ + public static HadoopFilesetContext.Builder builder() { + return new HadoopFilesetContext.Builder(); + } +} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java index 1952cbd2528..35719281403 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java @@ -48,6 +48,8 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.meta.FilesetEntity; import org.apache.gravitino.meta.SchemaEntity; import org.apache.gravitino.utils.PrincipalUtils; @@ -218,6 +220,12 @@ public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { return hadoopCatalogOperations.loadFileset(ident); } + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + throws NoSuchFilesetException { + return hadoopCatalogOperations.getFilesetContext(ident, ctx); + } + @Override public void close() throws IOException { hadoopCatalogOperations.close(); diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 3b9ca0d7f8b..7fa3d7c20a5 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -66,8 +66,10 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NonEmptySchemaException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; +import org.apache.gravitino.file.BaseFilesetDataOperationCtx; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.service.CatalogMetaService; @@ -762,6 +764,56 @@ public void testTestConnection() { ImmutableMap.of())); } + @Test + public void testGetFilesetContext() throws IOException { + String schemaName = "schema1024"; + String comment = "comment1024"; + String schemaPath = TEST_ROOT_PATH + "/" + schemaName; + createSchema(schemaName, comment, null, schemaPath); + + String catalogName = "c1"; + String name = "fileset1024"; + String storageLocation = TEST_ROOT_PATH + "/" + catalogName + "/" + schemaName + "/" + name; + Fileset fileset = + createFileset(name, schemaName, comment, Fileset.Type.MANAGED, null, storageLocation); + + try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { + ops.initialize(Maps.newHashMap(), randomCatalogInfo(), HADOOP_PROPERTIES_METADATA); + NameIdentifier filesetIdent = NameIdentifier.of("m1", "c1", schemaName, name); + BaseFilesetDataOperationCtx dataOperationCtx1 = + BaseFilesetDataOperationCtx.builder() + .withSubPath("/test/test.parquet") + .withOperation("open") + .withClientType("test") + .build(); + FilesetContext context1 = ops.getFilesetContext(filesetIdent, dataOperationCtx1); + Assertions.assertEquals(name, context1.fileset().name()); + Assertions.assertEquals(Fileset.Type.MANAGED, context1.fileset().type()); + Assertions.assertEquals("comment1024", context1.fileset().comment()); + Assertions.assertEquals(fileset.storageLocation(), context1.fileset().storageLocation()); + Assertions.assertEquals( + String.format("%s%s", context1.fileset().storageLocation(), dataOperationCtx1.subPath()), + context1.actualPath()); + Assertions.assertFalse(context1.fileset().properties().containsKey(StringIdentifier.ID_KEY)); + + BaseFilesetDataOperationCtx dataOperationCtx2 = + BaseFilesetDataOperationCtx.builder() + .withSubPath("test/test.parquet") + .withOperation("open") + .withClientType("test") + .build(); + FilesetContext context2 = ops.getFilesetContext(filesetIdent, dataOperationCtx2); + Assertions.assertEquals(name, context2.fileset().name()); + Assertions.assertEquals(Fileset.Type.MANAGED, context2.fileset().type()); + Assertions.assertEquals("comment1024", context2.fileset().comment()); + Assertions.assertEquals(fileset.storageLocation(), context2.fileset().storageLocation()); + Assertions.assertFalse(context2.fileset().properties().containsKey(StringIdentifier.ID_KEY)); + Assertions.assertEquals( + String.format("%s/%s", context2.fileset().storageLocation(), dataOperationCtx2.subPath()), + context2.actualPath()); + } + } + private static Stream locationArguments() { return Stream.of( // Honor the catalog location diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java index 3f0d49530ee..0ee4b09f242 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java @@ -34,14 +34,18 @@ import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; +import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.FilesetContextResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.rest.RESTUtils; /** @@ -226,6 +230,37 @@ public boolean dropFileset(NameIdentifier ident) { return resp.dropped(); } + /** + * Get a fileset context from the catalog. + * + * @param ident A fileset identifier. + * @param ctx The data operation context. + * @return The fileset context. + */ + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + throws NoSuchFilesetException { + checkFilesetNameIdentifier(ident); + + GetFilesetContextRequest req = + GetFilesetContextRequest.builder() + .subPath(ctx.subPath()) + .operation(ctx.operation()) + .clientType(ctx.clientType()) + .build(); + + FilesetContextResponse resp = + restClient.post( + formatFilesetRequestPath(ident.namespace()) + "/" + ident.name() + "/" + "context", + req, + FilesetContextResponse.class, + Collections.emptyMap(), + ErrorHandlers.filesetErrorHandler()); + resp.validate(); + + return resp.getContext(); + } + @VisibleForTesting static String formatFilesetRequestPath(Namespace ns) { Namespace schemaNs = Namespace.of(ns.level(0), ns.level(1)); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java index eee24ee3f21..547d1af4bd3 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java @@ -34,22 +34,27 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.dto.AuditDTO; import org.apache.gravitino.dto.CatalogDTO; +import org.apache.gravitino.dto.file.FilesetContextDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.requests.CatalogCreateRequest; import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; +import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.CatalogResponse; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.FilesetContextResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; +import org.apache.gravitino.file.BaseFilesetDataOperationCtx; import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.file.FilesetContext; import org.apache.hc.core5.http.Method; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -400,6 +405,43 @@ public void testAlterFileset() throws JsonProcessingException { "internal error"); } + @Test + public void testGetFilesetContext() throws JsonProcessingException { + NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + String filesetPath = + withSlash( + FilesetCatalog.formatFilesetRequestPath(fileset.namespace()) + "/fileset1/context"); + + FilesetDTO mockFileset = + mockFilesetDTO( + fileset.name(), + Fileset.Type.MANAGED, + "mock comment", + "mock location", + ImmutableMap.of("k1", "v1")); + String mockActualPath = "mock location/test"; + FilesetContextDTO mockContext = mockFilesetContextDTO(mockFileset, mockActualPath); + FilesetContextResponse resp = new FilesetContextResponse(mockContext); + GetFilesetContextRequest req = + GetFilesetContextRequest.builder() + .subPath("/test") + .operation("create") + .clientType("test") + .build(); + buildMockResource(Method.POST, filesetPath, req, resp, SC_OK); + BaseFilesetDataOperationCtx ctx = + BaseFilesetDataOperationCtx.builder() + .withSubPath("/test") + .withOperation("create") + .withClientType("test") + .build(); + FilesetContext filesetContext = catalog.asFilesetCatalog().getFilesetContext(fileset, ctx); + Assertions.assertNotNull(filesetContext); + assertFileset(mockFileset, filesetContext.fileset()); + + Assertions.assertEquals(mockActualPath, filesetContext.actualPath()); + } + private FilesetDTO mockFilesetDTO( String name, Fileset.Type type, @@ -416,6 +458,10 @@ private FilesetDTO mockFilesetDTO( .build(); } + private FilesetContextDTO mockFilesetContextDTO(FilesetDTO fileset, String actualPath) { + return FilesetContextDTO.builder().fileset(fileset).actualPath(actualPath).build(); + } + private void assertFileset(FilesetDTO expected, Fileset actual) { Assertions.assertEquals(expected.name(), actual.name()); Assertions.assertEquals(expected.comment(), actual.comment()); diff --git a/common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java b/common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java new file mode 100644 index 00000000000..3334ca7b841 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.dto.file; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.gravitino.file.FilesetContext; + +/** Represents a Fileset context DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class FilesetContextDTO implements FilesetContext { + @JsonProperty("fileset") + private FilesetDTO fileset; + + @JsonProperty("actualPath") + private String actualPath; + + public FilesetDTO fileset() { + return fileset; + } + + public String actualPath() { + return actualPath; + } + + @Builder(builderMethodName = "builder") + private static FilesetContextDTO internalBuilder(FilesetDTO fileset, String actualPath) { + Preconditions.checkNotNull(fileset, "fileset cannot be null"); + Preconditions.checkNotNull(actualPath, "actual path cannot be null"); + + return new FilesetContextDTO(fileset, actualPath); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java new file mode 100644 index 00000000000..579a9255d17 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Request to get a fileset context. */ +@Getter +@EqualsAndHashCode +@NoArgsConstructor(force = true) +@AllArgsConstructor +@Builder +@ToString +public class GetFilesetContextRequest implements RESTRequest { + @JsonProperty("subPath") + private String subPath; + + @JsonProperty("operation") + private String operation; + + @JsonProperty("clientType") + private String clientType; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument(subPath != null, "\"subPath\" field cannot be null"); + Preconditions.checkArgument( + StringUtils.isNotBlank(operation), "\"operation\" field is required and cannot be empty"); + Preconditions.checkArgument( + StringUtils.isNotBlank(clientType), "\"clientType\" field is required and cannot be empty"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java new file mode 100644 index 00000000000..30e980ac38a --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.file.FilesetContextDTO; + +/** Response for fileset context. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class FilesetContextResponse extends BaseResponse { + @JsonProperty("filesetContext") + private final FilesetContextDTO context; + + /** Constructor for FilesetContextResponse. */ + public FilesetContextResponse() { + super(0); + this.context = null; + } + + /** + * Constructor for FilesetContextResponse. + * + * @param context the fileset context DTO. + */ + public FilesetContextResponse(FilesetContextDTO context) { + super(0); + this.context = context; + } + + /** + * Validates the response. + * + * @throws IllegalArgumentException if the response is invalid. + */ + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + Preconditions.checkArgument(context != null, "context must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java index d83460af182..06894f22eae 100644 --- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java @@ -45,6 +45,7 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; +import org.apache.gravitino.dto.file.FilesetContextDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.messaging.TopicDTO; import org.apache.gravitino.dto.rel.ColumnDTO; @@ -75,6 +76,7 @@ import org.apache.gravitino.dto.tag.MetadataObjectDTO; import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.messaging.Topic; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; @@ -585,6 +587,19 @@ public static FilesetDTO toDTO(Fileset fileset) { .build(); } + /** + * Converts to a FilesetContextDTO. + * + * @param context The fileset context to be converted. + * @return The fileset context DTO. + */ + public static FilesetContextDTO toDTO(FilesetContext context) { + return FilesetContextDTO.builder() + .fileset(toDTO(context.fileset())) + .actualPath(context.actualPath()) + .build(); + } + /** * Converts a Topic to a TopicDTO. * diff --git a/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java b/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java new file mode 100644 index 00000000000..cf53cd10744 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.dto.file; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import java.time.Instant; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestFilesetContextDTO { + @Test + void testJsonSerDe() throws JsonProcessingException { + FilesetDTO filesetDTO = + FilesetDTO.builder() + .name("test") + .type(Fileset.Type.MANAGED) + .audit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) + .storageLocation("hdfs://host/test") + .build(); + + FilesetContextDTO dto = + FilesetContextDTO.builder() + .fileset(filesetDTO) + .actualPath("hdfs://host/test/1.txt") + .build(); + String value = JsonUtils.objectMapper().writeValueAsString(dto); + + String expectedValue = + String.format( + "{\n" + + " \"fileset\": {\n" + + " \"name\": \"test\",\n" + + " \"comment\": null,\n" + + " \"type\": \"managed\",\n" + + " \"storageLocation\": \"hdfs://host/test\",\n" + + " \"properties\": null,\n" + + " \"audit\": {\n" + + " \"creator\": \"creator\",\n" + + " \"createTime\": \"%s\",\n" + + " \"lastModifier\": null,\n" + + " \"lastModifiedTime\": null\n" + + " }\n" + + " },\n" + + " \"actualPath\": \"hdfs://host/test/1.txt\"\n" + + "}", + filesetDTO.auditInfo().createTime()); + JsonNode expected = JsonUtils.objectMapper().readTree(expectedValue); + JsonNode actual = JsonUtils.objectMapper().readTree(value); + Assertions.assertEquals(expected, actual); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java new file mode 100644 index 00000000000..c41df904d5f --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestGetFilesetContextRequest { + @Test + public void testGetFilesetContextRequest() throws JsonProcessingException { + GetFilesetContextRequest request = + GetFilesetContextRequest.builder() + .subPath("/test/1.txt") + .operation("create") + .clientType("test") + .build(); + String jsonString = JsonUtils.objectMapper().writeValueAsString(request); + String expected = + "{\"subPath\":\"/test/1.txt\",\"operation\":\"create\",\"clientType\":\"test\"}"; + Assertions.assertEquals( + JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(jsonString)); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java index 242c0f95359..9c403e43496 100644 --- a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java @@ -41,11 +41,14 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; +import org.apache.gravitino.dto.file.FilesetContextDTO; +import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.TableDTO; import org.apache.gravitino.dto.rel.partitioning.Partitioning; import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.file.Fileset; import org.apache.gravitino.json.JsonUtils; import org.apache.gravitino.rel.types.Types; import org.junit.jupiter.api.Test; @@ -377,4 +380,30 @@ void testTagResponse() throws JsonProcessingException { Exception e = assertThrows(IllegalArgumentException.class, response1::validate); assertEquals("\"tag\" must not be null", e.getMessage()); } + + @Test + void testFilesetContextResponse() { + AuditDTO audit = + AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build(); + FilesetDTO filesetDTO = + FilesetDTO.builder() + .name("test") + .type(Fileset.Type.MANAGED) + .audit(audit) + .storageLocation("hdfs://host/test") + .build(); + FilesetContextDTO dto = + FilesetContextDTO.builder() + .fileset(filesetDTO) + .actualPath("hdfs://host/test/zz.parquet") + .build(); + FilesetContextResponse response = new FilesetContextResponse(dto); + response.validate(); // No exception thrown + } + + @Test + void testFilesetContextResponseException() { + FilesetContextResponse response = new FilesetContextResponse(); + assertThrows(IllegalArgumentException.class, () -> response.validate()); + } } diff --git a/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java b/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java index 46a2ffa11b2..5eb4347d0f4 100644 --- a/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java +++ b/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java @@ -19,9 +19,12 @@ package org.apache.gravitino.dto.util; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Map; +import org.apache.gravitino.Audit; +import org.apache.gravitino.dto.file.FilesetContextDTO; import org.apache.gravitino.dto.rel.expressions.LiteralDTO; import org.apache.gravitino.dto.rel.partitioning.ListPartitioningDTO; import org.apache.gravitino.dto.rel.partitioning.RangePartitioningDTO; @@ -29,6 +32,8 @@ import org.apache.gravitino.dto.rel.partitions.ListPartitionDTO; import org.apache.gravitino.dto.rel.partitions.PartitionDTO; import org.apache.gravitino.dto.rel.partitions.RangePartitionDTO; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.rel.expressions.literals.Literal; import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.expressions.transforms.Transform; @@ -246,4 +251,72 @@ void testPartitionDTOConvert() { Types.StringType.get().simpleString(), listPartitionAssignments[0].lists()[0][0].value()); Assertions.assertEquals(properties, listPartitionAssignments[0].properties()); } + + @Test + void testFilesetContextConvert() { + FilesetContext filesetContext = + new FilesetContext() { + @Override + public Fileset fileset() { + Fileset mockFileset = + new Fileset() { + @Override + public String name() { + return "test"; + } + + @Override + public Type type() { + return Type.MANAGED; + } + + @Override + public String storageLocation() { + return "hdfs://host/test"; + } + + @Override + public Audit auditInfo() { + Audit mockAudit = + new Audit() { + @Override + public String creator() { + return null; + } + + @Override + public Instant createTime() { + return null; + } + + @Override + public String lastModifier() { + return null; + } + + @Override + public Instant lastModifiedTime() { + return null; + } + }; + return mockAudit; + } + }; + return mockFileset; + } + + @Override + public String actualPath() { + return "hdfs://host/test/1.txt"; + } + }; + + FilesetContextDTO filesetContextDTO = DTOConverters.toDTO(filesetContext); + + // then + Assertions.assertEquals("test", filesetContextDTO.fileset().name()); + Assertions.assertEquals(Fileset.Type.MANAGED, filesetContextDTO.fileset().type()); + Assertions.assertEquals("hdfs://host/test", filesetContextDTO.fileset().storageLocation()); + Assertions.assertEquals("hdfs://host/test/1.txt", filesetContextDTO.actualPath()); + } } diff --git a/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java index d79630392a5..debed164155 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java @@ -32,6 +32,8 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; public class FilesetNormalizeDispatcher implements FilesetDispatcher { private final CatalogManager catalogManager; @@ -94,6 +96,13 @@ public boolean dropFileset(NameIdentifier ident) { return dispatcher.dropFileset(normalizeCaseSensitive(ident)); } + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) { + // The constraints of the name spec may be more strict than underlying catalog, + // and for compatibility reasons, we only apply case-sensitive capabilities here. + return dispatcher.getFilesetContext(normalizeCaseSensitive(ident), ctx); + } + private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { Capability capabilities = getCapability(ident, catalogManager); return applyCapabilities(ident, Capability.Scope.FILESET, capabilities); diff --git a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java index e28369f96bb..1c6e7131d94 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java @@ -33,6 +33,8 @@ import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.storage.IdGenerator; public class FilesetOperationDispatcher extends OperationDispatcher implements FilesetDispatcher { @@ -195,4 +197,20 @@ public boolean dropFileset(NameIdentifier ident) { c -> c.doWithFilesetOps(f -> f.dropFileset(ident)), NonEmptyEntityException.class); } + + /** + * Get a fileset context from the catalog. + * + * @param ident A fileset identifier. + * @param ctx A fileset data operation context. + * @return true If the fileset is dropped, false the fileset did not exist. + */ + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + throws NoSuchFilesetException { + return doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithFilesetOps(f -> f.getFilesetContext(ident, ctx)), + NonEmptyEntityException.class); + } } diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java b/core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java new file mode 100644 index 00000000000..14e0b86b905 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.connector; + +import org.apache.gravitino.annotation.Evolving; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.file.FilesetCatalog; +import org.apache.gravitino.file.FilesetContext; + +/** + * An abstract class representing a base fileset context for {@link FilesetCatalog}. Developers + * should extend this class to implement a custom fileset context for their fileset catalog. + */ +@Evolving +public abstract class BaseFilesetContext implements FilesetContext { + protected Fileset fileset; + protected String actualPath; + + @Override + public Fileset fileset() { + return fileset; + } + + @Override + public String actualPath() { + return actualPath; + } + + interface Builder< + SELF extends BaseFilesetContext.Builder, T extends BaseFilesetContext> { + + SELF withFileset(Fileset fileset); + + SELF withActualPath(String actualPath); + + T build(); + } + + /** + * An abstract class implementing the builder interface for {@link BaseFilesetContext}. This class + * should be extended by the concrete fileset context builders. + * + * @param The type of the builder. + * @param The type of the fileset being built. + */ + public abstract static class BaseFilesetContextBuilder< + SELF extends BaseFilesetContext.Builder, T extends BaseFilesetContext> + implements BaseFilesetContext.Builder { + protected Fileset fileset; + protected String actualPath; + + /** + * Sets the fileset of the fileset context. + * + * @param fileset The fileset object. + * @return The builder instance. + */ + @Override + public SELF withFileset(Fileset fileset) { + this.fileset = fileset; + return self(); + } + + /** + * Sets the actual path of the fileset context. + * + * @param actualPath The actual path for the fileset context. + * @return The builder instance. + */ + @Override + public SELF withActualPath(String actualPath) { + this.actualPath = actualPath; + return self(); + } + + /** + * Builds the instance of the fileset with the provided attributes. + * + * @return The built fileset instance. + */ + @Override + public T build() { + T t = internalBuild(); + return t; + } + + private SELF self() { + return (SELF) this; + } + + /** + * Builds the concrete instance of the fileset with the provided attributes. + * + * @return The built fileset instance. + */ + @Evolving + protected abstract T internalBuild(); + } +} diff --git a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java index 4daac06bab3..cb890033f0f 100644 --- a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java @@ -28,12 +28,16 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.listener.api.event.AlterFilesetEvent; import org.apache.gravitino.listener.api.event.AlterFilesetFailureEvent; import org.apache.gravitino.listener.api.event.CreateFilesetEvent; import org.apache.gravitino.listener.api.event.CreateFilesetFailureEvent; import org.apache.gravitino.listener.api.event.DropFilesetEvent; import org.apache.gravitino.listener.api.event.DropFilesetFailureEvent; +import org.apache.gravitino.listener.api.event.GetFilesetContextEvent; +import org.apache.gravitino.listener.api.event.GetFilesetContextFailureEvent; import org.apache.gravitino.listener.api.event.ListFilesetEvent; import org.apache.gravitino.listener.api.event.ListFilesetFailureEvent; import org.apache.gravitino.listener.api.event.LoadFilesetEvent; @@ -138,4 +142,19 @@ public boolean dropFileset(NameIdentifier ident) { throw e; } } + + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + throws NoSuchFilesetException { + try { + FilesetContext context = dispatcher.getFilesetContext(ident, ctx); + eventBus.dispatchEvent( + new GetFilesetContextEvent(PrincipalUtils.getCurrentUserName(), ident, ctx)); + return context; + } catch (Exception e) { + eventBus.dispatchEvent( + new GetFilesetContextFailureEvent(PrincipalUtils.getCurrentUserName(), ident, ctx, e)); + throw e; + } + } } diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java new file mode 100644 index 00000000000..184c4aaca65 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.listener.api.event; + +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.annotation.DeveloperApi; +import org.apache.gravitino.file.FilesetDataOperationCtx; + +/** Represents an event that occurs when getting a fileset context. */ +@DeveloperApi +public final class GetFilesetContextEvent extends FilesetEvent { + private final FilesetDataOperationCtx ctx; + + /** + * Constructs a new {@code GetFilesetContextEvent}, recording the attempt to get a fileset + * context. + * + * @param user The user who initiated the get fileset context operation. + * @param identifier The identifier of the fileset context that was attempted to be got. + * @param ctx The data operation context to get the fileset context. + */ + public GetFilesetContextEvent( + String user, NameIdentifier identifier, FilesetDataOperationCtx ctx) { + super(user, identifier); + this.ctx = ctx; + } + + public FilesetDataOperationCtx dataOperationContext() { + return ctx; + } +} diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java new file mode 100644 index 00000000000..27d44c7c159 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.listener.api.event; + +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.annotation.DeveloperApi; +import org.apache.gravitino.file.FilesetDataOperationCtx; + +/** + * Represents an event that is generated when an attempt to get a fileset context from the system + * fails. + */ +@DeveloperApi +public final class GetFilesetContextFailureEvent extends FilesetFailureEvent { + private final FilesetDataOperationCtx ctx; + /** + * Constructs a new {@code GetFilesetContextFailureEvent}. + * + * @param user The user who initiated the get fileset context operation. + * @param identifier The identifier of the fileset context that was attempted to be got. + * @param ctx The data operation context to get the fileset context. + * @param exception The exception that was thrown during the get fileset context operation. This + * exception is key to diagnosing the failure, providing insights into what went wrong during + * the operation. + */ + public GetFilesetContextFailureEvent( + String user, NameIdentifier identifier, FilesetDataOperationCtx ctx, Exception exception) { + super(user, identifier, exception); + this.ctx = ctx; + } + + public FilesetDataOperationCtx dataOperationContext() { + return ctx; + } +} diff --git a/core/src/test/java/org/apache/gravitino/TestFilesetContext.java b/core/src/test/java/org/apache/gravitino/TestFilesetContext.java new file mode 100644 index 00000000000..a960d8823a0 --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/TestFilesetContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino; + +import lombok.EqualsAndHashCode; +import org.apache.gravitino.connector.BaseFilesetContext; + +@EqualsAndHashCode(callSuper = true) +public class TestFilesetContext extends BaseFilesetContext { + public static class Builder + extends BaseFilesetContext.BaseFilesetContextBuilder< + TestFilesetContext.Builder, TestFilesetContext> { + /** Creates a new instance of {@link TestFilesetContext.Builder}. */ + private Builder() {} + + @Override + protected TestFilesetContext internalBuild() { + TestFilesetContext context = new TestFilesetContext(); + context.fileset = fileset; + context.actualPath = actualPath; + return context; + } + } + + /** + * Creates a new instance of {@link TestFilesetContext.Builder}. + * + * @return The new instance. + */ + public static TestFilesetContext.Builder builder() { + return new TestFilesetContext.Builder(); + } +} diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java index 551e588a5d7..cb96a5104a9 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -21,12 +21,16 @@ import static org.apache.gravitino.StringIdentifier.ID_KEY; import com.google.common.collect.ImmutableMap; +import java.io.File; import java.io.IOException; import java.util.Map; +import java.util.UUID; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.file.BaseFilesetDataOperationCtx; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -169,4 +173,46 @@ public void testCreateAndDropFileset() { Assertions.assertTrue(dropped); Assertions.assertFalse(filesetOperationDispatcher.dropFileset(filesetIdent1)); } + + @Test + public void testCreateAndGetFilesetContext() { + String tmpDir = "/tmp/test_get_fileset_context_" + UUID.randomUUID(); + try { + Namespace filesetNs = Namespace.of(metalake, catalog, "schema91"); + Map props = ImmutableMap.of("k1", "v1", "location", "schema91"); + schemaOperationDispatcher.createSchema( + NameIdentifier.of(filesetNs.levels()), "comment", props); + + NameIdentifier filesetIdent1 = NameIdentifier.of(filesetNs, "fileset1024"); + Fileset fileset1 = + filesetOperationDispatcher.createFileset( + filesetIdent1, "comment", Fileset.Type.MANAGED, tmpDir, props); + Assertions.assertEquals("fileset1024", fileset1.name()); + Assertions.assertEquals("comment", fileset1.comment()); + testProperties(props, fileset1.properties()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); + Assertions.assertNotNull(fileset1.storageLocation()); + + BaseFilesetDataOperationCtx ctx = + BaseFilesetDataOperationCtx.builder() + .withSubPath("/test/x.parquet") + .withOperation("create") + .withClientType("test") + .build(); + FilesetContext context1 = filesetOperationDispatcher.getFilesetContext(filesetIdent1, ctx); + Assertions.assertEquals(fileset1.name(), context1.fileset().name()); + Assertions.assertEquals(fileset1.comment(), context1.fileset().comment()); + testProperties(props, context1.fileset().properties()); + Assertions.assertEquals(fileset1.type(), context1.fileset().type()); + Assertions.assertEquals(fileset1.storageLocation(), context1.fileset().storageLocation()); + + Assertions.assertEquals( + fileset1.storageLocation() + "/test/x.parquet", context1.actualPath()); + } finally { + File path = new File(tmpDir); + if (path.exists()) { + path.delete(); + } + } + } } diff --git a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java index 7588af17c59..0668cf68e93 100644 --- a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java +++ b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java @@ -18,20 +18,25 @@ */ package org.apache.gravitino.connector; +import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.Schema; import org.apache.gravitino.SchemaChange; import org.apache.gravitino.TestFileset; +import org.apache.gravitino.TestFilesetContext; import org.apache.gravitino.TestSchema; import org.apache.gravitino.TestTable; import org.apache.gravitino.TestTopic; +import org.apache.gravitino.catalog.EntityCombinedFileset; import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchCatalogException; @@ -46,6 +51,8 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.messaging.DataLayout; import org.apache.gravitino.messaging.Topic; import org.apache.gravitino.messaging.TopicCatalog; @@ -63,6 +70,8 @@ public class TestCatalogOperations implements CatalogOperations, TableCatalog, FilesetCatalog, TopicCatalog, SupportsSchemas { + private HasPropertyMetadata propertiesMetadata; + private final Map tables; private final Map schemas; @@ -85,7 +94,9 @@ public TestCatalogOperations(Map config) { @Override public void initialize( Map config, CatalogInfo info, HasPropertyMetadata propertyMetadata) - throws RuntimeException {} + throws RuntimeException { + this.propertiesMetadata = propertyMetadata; + } @Override public void close() throws IOException {} @@ -430,6 +441,36 @@ public boolean dropFileset(NameIdentifier ident) { } } + @Override + public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) { + String subPath = ctx.subPath(); + Preconditions.checkArgument(subPath != null, "subPath must not be null"); + + Fileset fileset = loadFileset(ident); + + String storageLocation = fileset.storageLocation(); + String actualPath; + // subPath cannot be null, so we only need check if it is blank + if (StringUtils.isBlank(subPath)) { + actualPath = storageLocation; + } else { + actualPath = + subPath.startsWith("/") + ? String.format("%s%s", storageLocation, subPath) + : String.format("%s/%s", storageLocation, subPath); + } + + return TestFilesetContext.builder() + .withFileset( + EntityCombinedFileset.of(fileset) + .withHiddenPropertiesSet( + fileset.properties().keySet().stream() + .filter(propertiesMetadata.filesetPropertiesMetadata()::isHiddenProperty) + .collect(Collectors.toSet()))) + .withActualPath(actualPath) + .build(); + } + @Override public NameIdentifier[] listTopics(Namespace namespace) throws NoSuchSchemaException { return topics.keySet().stream() diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 0bf654cd84e..0511a306312 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -160,6 +160,47 @@ paths: "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}/context: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/fileset" + + post: + tags: + - filesetContext + summary: Get fileset context + operationId: getFilesetContext + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/GetFilesetContextRequest" + examples: + GetFilesetContextRequest: + $ref: "#/components/examples/GetFilesetContextRequest" + description: Returns the specified fileset context object + responses: + "200": + $ref: "#/components/responses/FilesetContextResponse" + "404": + description: Not Found - The target fileset context does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchFilesetException: + $ref: "#/components/examples/NoSuchFilesetException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" components: @@ -191,6 +232,17 @@ components: default: {} additionalProperties: type: string + FilesetContext: + type: object + required: + - fileset + - actualPath + properties: + fileset: + $ref: "#/components/schemas/Fileset" + actualPath: + type: string + description: The actual paths which data operation accessed FilesetCreateRequest: type: object required: @@ -339,6 +391,23 @@ components: "@type": "removeComment" } + GetFilesetContextRequest: + type: object + required: + - subPath + - operation + - clientType + properties: + subPath: + type: string + description: The sub path that the data operation to access. Can not be null. + operation: + type: string + description: The data operation to get the fileset context. Can not be empty. + clientType: + type: string + description: The IP of the machine that initiated the data operation. Can not be empty. + responses: FilesetResponse: description: The response of fileset object @@ -359,6 +428,24 @@ components: FilesetResponse: $ref: "#/components/examples/FilesetResponse" + FilesetContextResponse: + description: The response of fileset context object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + context: + $ref: "#/components/schemas/FilesetContext" + examples: + FilesetContextResponse: + $ref: "#/components/examples/FilesetContextResponse" examples: FilesetCreateRequest: @@ -388,6 +475,24 @@ components: } } + FilesetContextResponse: + value: { + "code": 0, + "context": { + "fileset": { + "name": "fileset1", + "type": "managed", + "comment": "This is a comment", + "storageLocation": "hdfs://host/user/s_fileset/schema/fileset1", + "properties": { + "key1": "value1", + "key2": "value2" + } + }, + "actualPath": "hdfs://host/user/fileset/schema/fileset1/test.parquet" + } + } + FilesetAlreadyExistsException: value: { "code": 1004, diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java new file mode 100644 index 00000000000..6f541302f5f --- /dev/null +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.integration.test.client; + +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.file.BaseFilesetDataOperationCtx; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperationCtx; +import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.util.AbstractIT; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag("gravitino-docker-test") +public class FilesetIT extends AbstractIT { + private static final Logger LOG = LoggerFactory.getLogger(FilesetIT.class); + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + + private static final String metalakeName = GravitinoITUtils.genRandomName("fileset_it_metalake"); + + private static GravitinoMetalake metalake; + + private static String hmsUri; + + @BeforeAll + public static void startUp() { + containerSuite.startHiveContainer(); + hmsUri = + String.format( + "thrift://%s:%d", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT); + + Assertions.assertFalse(client.metalakeExists(metalakeName)); + metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); + Assertions.assertTrue(client.metalakeExists(metalakeName)); + } + + @AfterAll + public static void tearDown() { + client.dropMetalake(metalakeName); + + if (client != null) { + client.close(); + client = null; + } + + try { + closer.close(); + } catch (Exception e) { + LOG.error("Exception in closing CloseableGroup", e); + } + } + + @Test + public void testGetFilesetContext() { + String catalogName = GravitinoITUtils.genRandomName("catalog"); + Assertions.assertFalse(metalake.catalogExists(catalogName)); + + Map properties = Maps.newHashMap(); + properties.put("metastore.uris", hmsUri); + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + + String schemaName = GravitinoITUtils.genRandomName("schema"); + Assertions.assertFalse(catalog.asSchemas().schemaExists(schemaName)); + catalog.asSchemas().createSchema(schemaName, "schema comment", Maps.newHashMap()); + Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); + + String filesetName = GravitinoITUtils.genRandomName("fileset"); + NameIdentifier filesetIdent = + NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName); + Assertions.assertFalse(catalog.asFilesetCatalog().filesetExists(filesetIdent)); + Fileset expectedFileset = + catalog + .asFilesetCatalog() + .createFileset( + filesetIdent, + "fileset comment", + Fileset.Type.MANAGED, + generateLocation(catalogName, schemaName, filesetName), + Maps.newHashMap()); + Assertions.assertTrue(catalog.asFilesetCatalog().filesetExists(filesetIdent)); + + FilesetDataOperationCtx ctx = + BaseFilesetDataOperationCtx.builder() + .withSubPath("/test.par") + .withOperation("create") + .withClientType("test") + .build(); + FilesetContext context = catalog.asFilesetCatalog().getFilesetContext(filesetIdent, ctx); + + Fileset actualFileset = context.fileset(); + Assertions.assertEquals(expectedFileset.name(), actualFileset.name()); + Assertions.assertEquals(expectedFileset.comment(), actualFileset.comment()); + Assertions.assertEquals(expectedFileset.type(), actualFileset.type()); + Assertions.assertEquals(expectedFileset.storageLocation(), actualFileset.storageLocation()); + Assertions.assertFalse(actualFileset.properties().containsKey(StringIdentifier.ID_KEY)); + + Assertions.assertEquals(expectedFileset.storageLocation() + "/test.par", context.actualPath()); + } + + private static String generateLocation( + String catalogName, String schemaName, String filesetName) { + return String.format( + "hdfs://%s:%d/user/hadoop/%s/%s/%s", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT, + catalogName, + schemaName, + filesetName); + } +} diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java index 0d6f1fde34b..c26ab675008 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java @@ -38,12 +38,16 @@ import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; +import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.FilesetContextResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.file.BaseFilesetDataOperationCtx; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.lock.LockType; import org.apache.gravitino.lock.TreeLockUtils; import org.apache.gravitino.metrics.MetricNames; @@ -245,4 +249,39 @@ public Response dropFileset( return ExceptionHandlers.handleFilesetException(OperationType.DROP, fileset, schema, e); } } + + // we use POST type here, because we need to use request body to pass some parameters + @POST + @Path("{fileset}/context") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-fileset-context." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-fileset-context", absolute = true) + public Response getFilesetContext( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("fileset") String fileset, + GetFilesetContextRequest request) { + LOG.info( + "Received get fileset context request: {}.{}.{}.{}", metalake, catalog, schema, fileset); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifierUtil.ofFileset(metalake, catalog, schema, fileset); + BaseFilesetDataOperationCtx ctx = + BaseFilesetDataOperationCtx.builder() + .withSubPath(request.getSubPath()) + .withOperation(request.getOperation()) + .withClientType(request.getClientType()) + .build(); + FilesetContext context = + TreeLockUtils.doWithTreeLock( + ident, LockType.READ, () -> dispatcher.getFilesetContext(ident, ctx)); + return Utils.ok(new FilesetContextResponse(DTOConverters.toDTO(context))); + }); + } catch (Exception e) { + return ExceptionHandlers.handleFilesetException(OperationType.GET, fileset, schema, e); + } + } } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java index 1492b8aa69a..51d1ba1dba0 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java @@ -47,16 +47,19 @@ import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; +import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; import org.apache.gravitino.dto.responses.ErrorConstants; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.FilesetContextResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; +import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.rest.RESTUtils; import org.glassfish.jersey.internal.inject.AbstractBinder; @@ -432,6 +435,29 @@ public void testDropFileset() { Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp.getType()); } + @Test + public void testGetFilesetContext() { + Fileset fileset = + mockFileset( + "fileset1", + Fileset.Type.MANAGED, + "mock comment", + "mock location", + ImmutableMap.of("k1", "v1")); + String actualPath = "mock location/path1"; + + FilesetContext context = mockFilesetContext(fileset, actualPath); + + GetFilesetContextRequest req = + GetFilesetContextRequest.builder() + .subPath("path1") + .operation("create") + .clientType("test") + .build(); + + assertGetFilesetContext(req, context); + } + private void assertUpdateFileset(FilesetUpdatesRequest req, Fileset updatedFileset) { when(dispatcher.alterFileset(any(), any(FilesetChange.class))).thenReturn(updatedFileset); @@ -452,6 +478,27 @@ private void assertUpdateFileset(FilesetUpdatesRequest req, Fileset updatedFiles Assertions.assertEquals(updatedFileset.properties(), filesetDTO.properties()); } + private void assertGetFilesetContext(GetFilesetContextRequest req, FilesetContext context) { + when(dispatcher.getFilesetContext(any(), any())).thenReturn(context); + Response resp = + target(filesetPath(metalake, catalog, schema) + "fileset1/context") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + + FilesetContextResponse contextResponse = resp.readEntity(FilesetContextResponse.class); + Assertions.assertEquals(0, contextResponse.getCode()); + + FilesetDTO filesetDTO = contextResponse.getContext().fileset(); + Assertions.assertEquals(context.fileset().name(), filesetDTO.name()); + Assertions.assertEquals(context.fileset().comment(), filesetDTO.comment()); + Assertions.assertEquals(context.fileset().type(), filesetDTO.type()); + Assertions.assertEquals(context.fileset().properties(), filesetDTO.properties()); + + Assertions.assertEquals(context.actualPath(), contextResponse.getContext().actualPath()); + } + private static String filesetPath(String metalake, String catalog, String schema) { return new StringBuilder() .append("/metalakes/") @@ -484,4 +531,11 @@ private static Fileset mockFileset( return mockFileset; } + + private static FilesetContext mockFilesetContext(Fileset fileset, String actualPath) { + FilesetContext context = mock(FilesetContext.class); + when(context.fileset()).thenReturn(fileset); + when(context.actualPath()).thenReturn(actualPath); + return context; + } } From 5c3b517797f5c6603937230f3a4d09210848c7f0 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 30 Jul 2024 10:03:36 +0800 Subject: [PATCH 02/13] add some doc for api --- .../file/BaseFilesetDataOperationCtx.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java index 59cbab3d18d..d0f8306be35 100644 --- a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java +++ b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java @@ -43,6 +43,7 @@ public String clientType() { return clientType; } + /** Builder for {@link BaseFilesetDataOperationCtx}. */ public static class Builder { private BaseFilesetDataOperationCtx context; /** Creates a new instance of {@link BaseFilesetDataOperationCtx.Builder}. */ @@ -50,16 +51,34 @@ private Builder() { context = new BaseFilesetDataOperationCtx(); } + /** + * Set the sub path of this data operation. + * + * @param subPath The subPath of this data operation. + * @return The builder. + */ public BaseFilesetDataOperationCtx.Builder withSubPath(String subPath) { context.subPath = subPath; return this; } + /** + * Set the type of this data operation. + * + * @param operation The type of this data operation. + * @return The builder. + */ public BaseFilesetDataOperationCtx.Builder withOperation(String operation) { context.operation = operation; return this; } + /** + * Set the client type of this data operation. + * + * @param clientType The client type of this data operation. + * @return The builder. + */ public BaseFilesetDataOperationCtx.Builder withClientType(String clientType) { context.clientType = clientType; return this; @@ -73,6 +92,11 @@ private void validate() { StringUtils.isNotBlank(context.clientType), "clientType is required"); } + /** + * Build the {@link BaseFilesetDataOperationCtx}. + * + * @return The created BaseFilesetDataOperationCtx. + */ public BaseFilesetDataOperationCtx build() { validate(); return context; From 67aa7234252a3fe102ee361f0ea78ecbfc09e21a Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 30 Jul 2024 10:28:00 +0800 Subject: [PATCH 03/13] make operation and client type to enums --- .../file/BaseFilesetDataOperationCtx.java | 19 +++++----- .../org/apache/gravitino/file/ClientType.java | 27 ++++++++++++++ .../gravitino/file/FilesetDataOperation.java | 36 +++++++++++++++++++ .../file/FilesetDataOperationCtx.java | 4 +-- .../hadoop/TestHadoopCatalogOperations.java | 10 +++--- .../gravitino/client/FilesetCatalog.java | 4 +-- .../gravitino/client/TestFilesetCatalog.java | 16 ++++++--- .../requests/GetFilesetContextRequest.java | 11 +++--- .../TestGetFilesetContextRequest.java | 8 +++-- .../TestFilesetOperationDispatcher.java | 6 ++-- .../integration/test/client/FilesetIT.java | 6 ++-- .../web/rest/TestFilesetOperations.java | 6 ++-- 12 files changed, 115 insertions(+), 38 deletions(-) create mode 100644 api/src/main/java/org/apache/gravitino/file/ClientType.java create mode 100644 api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java diff --git a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java index d0f8306be35..a3cb5b58e7b 100644 --- a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java +++ b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java @@ -20,13 +20,12 @@ package org.apache.gravitino.file; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.StringUtils; /** Base implementation of {@link FilesetDataOperationCtx}. */ public class BaseFilesetDataOperationCtx implements FilesetDataOperationCtx { private String subPath; - private String operation; - private String clientType; + private FilesetDataOperation operation; + private ClientType clientType; @Override public String subPath() { @@ -34,12 +33,12 @@ public String subPath() { } @Override - public String operation() { + public FilesetDataOperation operation() { return operation; } @Override - public String clientType() { + public ClientType clientType() { return clientType; } @@ -68,7 +67,7 @@ public BaseFilesetDataOperationCtx.Builder withSubPath(String subPath) { * @param operation The type of this data operation. * @return The builder. */ - public BaseFilesetDataOperationCtx.Builder withOperation(String operation) { + public BaseFilesetDataOperationCtx.Builder withOperation(FilesetDataOperation operation) { context.operation = operation; return this; } @@ -79,17 +78,15 @@ public BaseFilesetDataOperationCtx.Builder withOperation(String operation) { * @param clientType The client type of this data operation. * @return The builder. */ - public BaseFilesetDataOperationCtx.Builder withClientType(String clientType) { + public BaseFilesetDataOperationCtx.Builder withClientType(ClientType clientType) { context.clientType = clientType; return this; } private void validate() { Preconditions.checkArgument(context.subPath != null, "subPath cannot be null"); - Preconditions.checkArgument( - StringUtils.isNotBlank(context.operation), "operation is required"); - Preconditions.checkArgument( - StringUtils.isNotBlank(context.clientType), "clientType is required"); + Preconditions.checkArgument(context.operation != null, "operation is required"); + Preconditions.checkArgument(context.clientType != null, "clientType is required"); } /** diff --git a/api/src/main/java/org/apache/gravitino/file/ClientType.java b/api/src/main/java/org/apache/gravitino/file/ClientType.java new file mode 100644 index 00000000000..da2d5587bcb --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/file/ClientType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.file; + +/** An enum class containing fileset data operations client type that supported. */ +public enum ClientType { + HADOOP_GVFS, + PYTHON_GVFS, + UNKNOWN; +} diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java new file mode 100644 index 00000000000..1ac5a3c9f57 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.file; + +/** An enum class containing fileset data operations that supported. */ +public enum FilesetDataOperation { + CREATE, + OPEN, + APPEND, + RENAME, + DELETE, + GET_FILE_STATUS, + LIST_STATUS, + MKDIRS, + GET_DEFAULT_REPLICATION, + GET_DEFAULT_BLOCK_SIZE, + SET_WORKING_DIR, + UNKNOWN; +} diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java index 8e2b1997019..af54a393b6a 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java +++ b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java @@ -34,8 +34,8 @@ public interface FilesetDataOperationCtx { String subPath(); /** @return The data operation type. */ - String operation(); + FilesetDataOperation operation(); /** @return The client type of the data operation. */ - String clientType(); + ClientType clientType(); } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 7fa3d7c20a5..766de69e7a4 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -67,9 +67,11 @@ import org.apache.gravitino.exceptions.NonEmptySchemaException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; import org.apache.gravitino.file.BaseFilesetDataOperationCtx; +import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.service.CatalogMetaService; @@ -783,8 +785,8 @@ public void testGetFilesetContext() throws IOException { BaseFilesetDataOperationCtx dataOperationCtx1 = BaseFilesetDataOperationCtx.builder() .withSubPath("/test/test.parquet") - .withOperation("open") - .withClientType("test") + .withOperation(FilesetDataOperation.OPEN) + .withClientType(ClientType.HADOOP_GVFS) .build(); FilesetContext context1 = ops.getFilesetContext(filesetIdent, dataOperationCtx1); Assertions.assertEquals(name, context1.fileset().name()); @@ -799,8 +801,8 @@ public void testGetFilesetContext() throws IOException { BaseFilesetDataOperationCtx dataOperationCtx2 = BaseFilesetDataOperationCtx.builder() .withSubPath("test/test.parquet") - .withOperation("open") - .withClientType("test") + .withOperation(FilesetDataOperation.OPEN) + .withClientType(ClientType.HADOOP_GVFS) .build(); FilesetContext context2 = ops.getFilesetContext(filesetIdent, dataOperationCtx2); Assertions.assertEquals(name, context2.fileset().name()); diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java index 0ee4b09f242..041d6cc9e41 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java @@ -241,7 +241,7 @@ public boolean dropFileset(NameIdentifier ident) { public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) throws NoSuchFilesetException { checkFilesetNameIdentifier(ident); - + Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); GetFilesetContextRequest req = GetFilesetContextRequest.builder() .subPath(ctx.subPath()) @@ -251,7 +251,7 @@ public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperati FilesetContextResponse resp = restClient.post( - formatFilesetRequestPath(ident.namespace()) + "/" + ident.name() + "/" + "context", + formatFilesetRequestPath(fullNamespace) + "/" + ident.name() + "/" + "context", req, FilesetContextResponse.class, Collections.emptyMap(), diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java index 547d1af4bd3..1866692dc89 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java @@ -53,8 +53,10 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.file.BaseFilesetDataOperationCtx; +import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperation; import org.apache.hc.core5.http.Method; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -425,17 +427,21 @@ public void testGetFilesetContext() throws JsonProcessingException { GetFilesetContextRequest req = GetFilesetContextRequest.builder() .subPath("/test") - .operation("create") - .clientType("test") + .operation(FilesetDataOperation.OPEN) + .clientType(ClientType.HADOOP_GVFS) .build(); buildMockResource(Method.POST, filesetPath, req, resp, SC_OK); BaseFilesetDataOperationCtx ctx = BaseFilesetDataOperationCtx.builder() .withSubPath("/test") - .withOperation("create") - .withClientType("test") + .withOperation(FilesetDataOperation.OPEN) + .withClientType(ClientType.HADOOP_GVFS) .build(); - FilesetContext filesetContext = catalog.asFilesetCatalog().getFilesetContext(fileset, ctx); + FilesetContext filesetContext = + catalog + .asFilesetCatalog() + .getFilesetContext( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), ctx); Assertions.assertNotNull(filesetContext); assertFileset(mockFileset, filesetContext.fileset()); diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java index 579a9255d17..f8faba08497 100644 --- a/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java +++ b/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java @@ -27,7 +27,8 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.ToString; -import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.file.ClientType; +import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.rest.RESTRequest; /** Request to get a fileset context. */ @@ -42,17 +43,17 @@ public class GetFilesetContextRequest implements RESTRequest { private String subPath; @JsonProperty("operation") - private String operation; + private FilesetDataOperation operation; @JsonProperty("clientType") - private String clientType; + private ClientType clientType; @Override public void validate() throws IllegalArgumentException { Preconditions.checkArgument(subPath != null, "\"subPath\" field cannot be null"); Preconditions.checkArgument( - StringUtils.isNotBlank(operation), "\"operation\" field is required and cannot be empty"); + operation != null, "\"operation\" field is required and cannot be null"); Preconditions.checkArgument( - StringUtils.isNotBlank(clientType), "\"clientType\" field is required and cannot be empty"); + clientType != null, "\"clientType\" field is required and cannot be null"); } } diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java index c41df904d5f..49af4f82e4f 100644 --- a/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java @@ -20,6 +20,8 @@ package org.apache.gravitino.dto.requests; import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.gravitino.file.ClientType; +import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.json.JsonUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,12 +32,12 @@ public void testGetFilesetContextRequest() throws JsonProcessingException { GetFilesetContextRequest request = GetFilesetContextRequest.builder() .subPath("/test/1.txt") - .operation("create") - .clientType("test") + .operation(FilesetDataOperation.CREATE) + .clientType(ClientType.HADOOP_GVFS) .build(); String jsonString = JsonUtils.objectMapper().writeValueAsString(request); String expected = - "{\"subPath\":\"/test/1.txt\",\"operation\":\"create\",\"clientType\":\"test\"}"; + "{\"subPath\":\"/test/1.txt\",\"operation\":\"create\",\"clientType\":\"hadoop_gvfs\"}"; Assertions.assertEquals( JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(jsonString)); } diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java index cb96a5104a9..6238da4506d 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -28,9 +28,11 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.file.BaseFilesetDataOperationCtx; +import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperation; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -196,8 +198,8 @@ public void testCreateAndGetFilesetContext() { BaseFilesetDataOperationCtx ctx = BaseFilesetDataOperationCtx.builder() .withSubPath("/test/x.parquet") - .withOperation("create") - .withClientType("test") + .withOperation(FilesetDataOperation.CREATE) + .withClientType(ClientType.HADOOP_GVFS) .build(); FilesetContext context1 = filesetOperationDispatcher.getFilesetContext(filesetIdent1, ctx); Assertions.assertEquals(fileset1.name(), context1.fileset().name()); diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java index 6f541302f5f..5af6844dc87 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java @@ -27,8 +27,10 @@ import org.apache.gravitino.StringIdentifier; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.file.BaseFilesetDataOperationCtx; +import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; @@ -119,8 +121,8 @@ public void testGetFilesetContext() { FilesetDataOperationCtx ctx = BaseFilesetDataOperationCtx.builder() .withSubPath("/test.par") - .withOperation("create") - .withClientType("test") + .withOperation(FilesetDataOperation.CREATE) + .withClientType(ClientType.HADOOP_GVFS) .build(); FilesetContext context = catalog.asFilesetCatalog().getFilesetContext(filesetIdent, ctx); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java index 51d1ba1dba0..f4ccd69c653 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java @@ -57,9 +57,11 @@ import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.file.FilesetContext; +import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.rest.RESTUtils; import org.glassfish.jersey.internal.inject.AbstractBinder; @@ -451,8 +453,8 @@ public void testGetFilesetContext() { GetFilesetContextRequest req = GetFilesetContextRequest.builder() .subPath("path1") - .operation("create") - .clientType("test") + .operation(FilesetDataOperation.OPEN) + .clientType(ClientType.HADOOP_GVFS) .build(); assertGetFilesetContext(req, context); From 4f4c81af31ba8ee156f70a3b6d8c40b39798daa4 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Tue, 30 Jul 2024 11:42:15 +0800 Subject: [PATCH 04/13] fix api docs --- .../java/org/apache/gravitino/file/ClientType.java | 9 +++++++++ .../apache/gravitino/file/FilesetDataOperation.java | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/api/src/main/java/org/apache/gravitino/file/ClientType.java b/api/src/main/java/org/apache/gravitino/file/ClientType.java index da2d5587bcb..37359b6d181 100644 --- a/api/src/main/java/org/apache/gravitino/file/ClientType.java +++ b/api/src/main/java/org/apache/gravitino/file/ClientType.java @@ -21,7 +21,16 @@ /** An enum class containing fileset data operations client type that supported. */ public enum ClientType { + /** + * The client type is `org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem` which in + * the filesystem-hadoop3 module. + */ HADOOP_GVFS, + /** + * The client type is `gravitino.filesystem.gvfs.GravitinoVirtualFileSystem` which in the + * client-python module. + */ PYTHON_GVFS, + /** The client type is unknown. */ UNKNOWN; } diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java index 1ac5a3c9f57..265815d027c 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java +++ b/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java @@ -21,16 +21,28 @@ /** An enum class containing fileset data operations that supported. */ public enum FilesetDataOperation { + /** This data operation means that create a new file. */ CREATE, + /** This data operation means that open a file. */ OPEN, + /** This data operation means that append some content into a file. */ APPEND, + /** This data operation means that rename a file or a directory. */ RENAME, + /** This data operation means that delete a file or a directory. */ DELETE, + /** This data operation means that get a file status from a file or a directory. */ GET_FILE_STATUS, + /** This data operation means that list file statuses under a directory. */ LIST_STATUS, + /** This data operation means that create a directory. */ MKDIRS, + /** This data operation means that get the default replication of a file system. */ GET_DEFAULT_REPLICATION, + /** This data operation means that get the default block size of a file system. */ GET_DEFAULT_BLOCK_SIZE, + /** This data operation means that set current working directory. */ SET_WORKING_DIR, + /** This data operation means that it is an unknown data operation. */ UNKNOWN; } From 05fc5bea79910d7fea214fe62ad727c344d055bd Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Wed, 31 Jul 2024 19:53:31 +0800 Subject: [PATCH 05/13] fix the schema name conflicts in the UTs --- .../gravitino/catalog/TestFilesetOperationDispatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java index 6238da4506d..a6ceca3c987 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -180,8 +180,8 @@ public void testCreateAndDropFileset() { public void testCreateAndGetFilesetContext() { String tmpDir = "/tmp/test_get_fileset_context_" + UUID.randomUUID(); try { - Namespace filesetNs = Namespace.of(metalake, catalog, "schema91"); - Map props = ImmutableMap.of("k1", "v1", "location", "schema91"); + Namespace filesetNs = Namespace.of(metalake, catalog, "schema1024"); + Map props = ImmutableMap.of("k1", "v1", "location", "schema1024"); schemaOperationDispatcher.createSchema( NameIdentifier.of(filesetNs.levels()), "comment", props); From 910b7dae24a531190f3ef23baf530411bf4c4016 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 5 Aug 2024 19:33:51 +0800 Subject: [PATCH 06/13] fix its --- .../integration/test/client/FilesetIT.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java index 5af6844dc87..8201980c88a 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java @@ -37,6 +37,7 @@ import org.apache.gravitino.integration.test.util.AbstractIT; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -86,6 +87,19 @@ public static void tearDown() { } } + @AfterEach + public void cleanUp() { + String[] catalog = metalake.listCatalogs(); + for (String catalogName : catalog) { + Catalog filesetCatalog = metalake.loadCatalog(catalogName); + String[] schemas = filesetCatalog.asSchemas().listSchemas(); + for (String schemaName : schemas) { + filesetCatalog.asSchemas().dropSchema(schemaName, true); + } + metalake.dropCatalog(catalogName); + } + } + @Test public void testGetFilesetContext() { String catalogName = GravitinoITUtils.genRandomName("catalog"); @@ -104,8 +118,7 @@ public void testGetFilesetContext() { Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); String filesetName = GravitinoITUtils.genRandomName("fileset"); - NameIdentifier filesetIdent = - NameIdentifier.of(metalakeName, catalogName, schemaName, filesetName); + NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); Assertions.assertFalse(catalog.asFilesetCatalog().filesetExists(filesetIdent)); Fileset expectedFileset = catalog From e45fd5c7e1a54fb65cc2038012dad5dd2a83de7f Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 9 Aug 2024 17:37:53 +0800 Subject: [PATCH 07/13] refactor the code --- .../file/BaseFilesetDataOperationCtx.java | 111 -------------- .../apache/gravitino/file/FilesetCatalog.java | 10 +- .../hadoop/HadoopCatalogOperations.java | 22 +-- .../catalog/hadoop/HadoopFilesetContext.java | 48 ------- .../hadoop/SecureHadoopCatalogOperations.java | 6 +- .../hadoop/TestHadoopCatalogOperations.java | 42 ++---- .../gravitino/client/FilesetCatalog.java | 40 +++--- .../gravitino/client/TestFilesetCatalog.java | 134 +++++++++++------ .../audit/FilesetAuditConstants.java | 22 +-- .../gravitino/context/CallerContext.java | 108 ++++++++++++++ .../gravitino/dto/file/FilesetContextDTO.java | 57 -------- .../requests/GetFilesetContextRequest.java | 59 -------- ...esponse.java => FileLocationResponse.java} | 27 ++-- .../gravitino/dto/util/DTOConverters.java | 15 -- .../enums}/FilesetDataOperation.java | 11 +- .../gravitino/enums/InternalClientType.java | 15 +- .../gravitino/context/TestCallerContext.java | 34 ++--- .../dto/file/TestFilesetContextDTO.java | 72 ---------- .../TestGetFilesetContextRequest.java | 44 ------ .../dto/responses/TestResponses.java | 26 +--- .../gravitino/dto/util/TestDTOConverters.java | 73 ---------- .../catalog/FilesetNormalizeDispatcher.java | 6 +- .../catalog/FilesetOperationDispatcher.java | 14 +- .../connector/BaseFilesetContext.java | 116 --------------- .../listener/FilesetEventDispatcher.java | 20 +-- .../api/event/GetFileLocationEvent.java | 58 ++++++++ ....java => GetFileLocationFailureEvent.java} | 31 ++-- .../api/event/GetFilesetContextEvent.java | 48 ------- .../apache/gravitino/TestFilesetContext.java | 50 ------- .../TestFilesetOperationDispatcher.java | 27 +--- .../connector/TestCatalogOperations.java | 30 +--- .../listener/api/event/TestFilesetEvent.java | 136 +++++++++++++----- .../integration/test/client/FilesetIT.java | 62 +++++--- .../apache/gravitino/server/web/Utils.java | 27 ++++ .../server/web/rest/FilesetOperations.java | 47 +++--- .../web/rest/TestFilesetOperations.java | 62 ++------ 36 files changed, 612 insertions(+), 1098 deletions(-) delete mode 100644 api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java delete mode 100644 catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java rename api/src/main/java/org/apache/gravitino/file/FilesetContext.java => common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java (56%) create mode 100644 common/src/main/java/org/apache/gravitino/context/CallerContext.java delete mode 100644 common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java delete mode 100644 common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java rename common/src/main/java/org/apache/gravitino/dto/responses/{FilesetContextResponse.java => FileLocationResponse.java} (68%) rename {api/src/main/java/org/apache/gravitino/file => common/src/main/java/org/apache/gravitino/enums}/FilesetDataOperation.java (86%) rename api/src/main/java/org/apache/gravitino/file/ClientType.java => common/src/main/java/org/apache/gravitino/enums/InternalClientType.java (73%) rename api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java => common/src/test/java/org/apache/gravitino/context/TestCallerContext.java (52%) delete mode 100644 common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java delete mode 100644 common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java delete mode 100644 core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java create mode 100644 core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java rename core/src/main/java/org/apache/gravitino/listener/api/event/{GetFilesetContextFailureEvent.java => GetFileLocationFailureEvent.java} (56%) delete mode 100644 core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java delete mode 100644 core/src/test/java/org/apache/gravitino/TestFilesetContext.java diff --git a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java b/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java deleted file mode 100644 index a3cb5b58e7b..00000000000 --- a/api/src/main/java/org/apache/gravitino/file/BaseFilesetDataOperationCtx.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.file; - -import com.google.common.base.Preconditions; - -/** Base implementation of {@link FilesetDataOperationCtx}. */ -public class BaseFilesetDataOperationCtx implements FilesetDataOperationCtx { - private String subPath; - private FilesetDataOperation operation; - private ClientType clientType; - - @Override - public String subPath() { - return subPath; - } - - @Override - public FilesetDataOperation operation() { - return operation; - } - - @Override - public ClientType clientType() { - return clientType; - } - - /** Builder for {@link BaseFilesetDataOperationCtx}. */ - public static class Builder { - private BaseFilesetDataOperationCtx context; - /** Creates a new instance of {@link BaseFilesetDataOperationCtx.Builder}. */ - private Builder() { - context = new BaseFilesetDataOperationCtx(); - } - - /** - * Set the sub path of this data operation. - * - * @param subPath The subPath of this data operation. - * @return The builder. - */ - public BaseFilesetDataOperationCtx.Builder withSubPath(String subPath) { - context.subPath = subPath; - return this; - } - - /** - * Set the type of this data operation. - * - * @param operation The type of this data operation. - * @return The builder. - */ - public BaseFilesetDataOperationCtx.Builder withOperation(FilesetDataOperation operation) { - context.operation = operation; - return this; - } - - /** - * Set the client type of this data operation. - * - * @param clientType The client type of this data operation. - * @return The builder. - */ - public BaseFilesetDataOperationCtx.Builder withClientType(ClientType clientType) { - context.clientType = clientType; - return this; - } - - private void validate() { - Preconditions.checkArgument(context.subPath != null, "subPath cannot be null"); - Preconditions.checkArgument(context.operation != null, "operation is required"); - Preconditions.checkArgument(context.clientType != null, "clientType is required"); - } - - /** - * Build the {@link BaseFilesetDataOperationCtx}. - * - * @return The created BaseFilesetDataOperationCtx. - */ - public BaseFilesetDataOperationCtx build() { - validate(); - return context; - } - } - - /** - * Creates a new instance of {@link Builder}. - * - * @return The new instance. - */ - public static Builder builder() { - return new Builder(); - } -} diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java b/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java index 6959eff74d1..a4ba788eefe 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java +++ b/api/src/main/java/org/apache/gravitino/file/FilesetCatalog.java @@ -122,13 +122,13 @@ Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) boolean dropFileset(NameIdentifier ident); /** - * Get a fileset context. + * Get the actual location of a file or directory based on the storage location of Fileset and the + * sub path. * * @param ident A fileset identifier. - * @param ctx The data operation context. - * @return The fileset context. + * @param subPath The sub path to the file or directory. + * @return The actual location of the file or directory. * @throws NoSuchFilesetException If the fileset does not exist. */ - FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) - throws NoSuchFilesetException; + String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException; } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index decb63bbaa4..b2612e2af61 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -40,7 +40,6 @@ import org.apache.gravitino.Schema; import org.apache.gravitino.SchemaChange; import org.apache.gravitino.StringIdentifier; -import org.apache.gravitino.catalog.EntityCombinedFileset; import org.apache.gravitino.connector.CatalogInfo; import org.apache.gravitino.connector.CatalogOperations; import org.apache.gravitino.connector.HasPropertyMetadata; @@ -56,8 +55,6 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.FilesetEntity; import org.apache.gravitino.meta.SchemaEntity; @@ -358,34 +355,25 @@ public boolean dropFileset(NameIdentifier ident) { } @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { // TODO we need move some check logics in the Hadoop / Python GVFS to here. - String subPath = ctx.subPath(); Preconditions.checkArgument(subPath != null, "subPath must not be null"); Fileset fileset = loadFileset(ident); String storageLocation = fileset.storageLocation(); - String actualPath; + String fileLocation; // subPath cannot be null, so we only need check if it is blank if (StringUtils.isBlank(subPath)) { - actualPath = storageLocation; + fileLocation = storageLocation; } else { - actualPath = + fileLocation = subPath.startsWith("/") ? String.format("%s%s", storageLocation, subPath) : String.format("%s/%s", storageLocation, subPath); } - return HadoopFilesetContext.builder() - .withFileset( - EntityCombinedFileset.of(fileset) - .withHiddenPropertiesSet( - fileset.properties().keySet().stream() - .filter(propertiesMetadata.filesetPropertiesMetadata()::isHiddenProperty) - .collect(Collectors.toSet()))) - .withActualPath(actualPath) - .build(); + return fileLocation; } @Override diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java deleted file mode 100644 index 0fea8136b4e..00000000000 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopFilesetContext.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.catalog.hadoop; - -import org.apache.gravitino.connector.BaseFilesetContext; - -public class HadoopFilesetContext extends BaseFilesetContext { - public static class Builder - extends BaseFilesetContext.BaseFilesetContextBuilder< - HadoopFilesetContext.Builder, HadoopFilesetContext> { - /** Creates a new instance of {@link HadoopFilesetContext.Builder}. */ - private Builder() {} - - @Override - protected HadoopFilesetContext internalBuild() { - HadoopFilesetContext context = new HadoopFilesetContext(); - context.fileset = fileset; - context.actualPath = actualPath; - return context; - } - } - - /** - * Creates a new instance of {@link HadoopFilesetContext.Builder}. - * - * @return The new instance. - */ - public static HadoopFilesetContext.Builder builder() { - return new HadoopFilesetContext.Builder(); - } -} diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java index 35719281403..79777f75d64 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/SecureHadoopCatalogOperations.java @@ -48,8 +48,6 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.meta.FilesetEntity; import org.apache.gravitino.meta.SchemaEntity; import org.apache.gravitino.utils.PrincipalUtils; @@ -221,9 +219,9 @@ public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { } @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { - return hadoopCatalogOperations.getFilesetContext(ident, ctx); + return hadoopCatalogOperations.getFileLocation(ident, subPath); } @Override diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 766de69e7a4..ae9f1915db9 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -66,12 +66,8 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NonEmptySchemaException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; -import org.apache.gravitino.file.BaseFilesetDataOperationCtx; -import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.service.CatalogMetaService; @@ -767,7 +763,7 @@ public void testTestConnection() { } @Test - public void testGetFilesetContext() throws IOException { + public void testGetFileLocation() throws IOException { String schemaName = "schema1024"; String comment = "comment1024"; String schemaPath = TEST_ROOT_PATH + "/" + schemaName; @@ -782,37 +778,15 @@ public void testGetFilesetContext() throws IOException { try (SecureHadoopCatalogOperations ops = new SecureHadoopCatalogOperations(store)) { ops.initialize(Maps.newHashMap(), randomCatalogInfo(), HADOOP_PROPERTIES_METADATA); NameIdentifier filesetIdent = NameIdentifier.of("m1", "c1", schemaName, name); - BaseFilesetDataOperationCtx dataOperationCtx1 = - BaseFilesetDataOperationCtx.builder() - .withSubPath("/test/test.parquet") - .withOperation(FilesetDataOperation.OPEN) - .withClientType(ClientType.HADOOP_GVFS) - .build(); - FilesetContext context1 = ops.getFilesetContext(filesetIdent, dataOperationCtx1); - Assertions.assertEquals(name, context1.fileset().name()); - Assertions.assertEquals(Fileset.Type.MANAGED, context1.fileset().type()); - Assertions.assertEquals("comment1024", context1.fileset().comment()); - Assertions.assertEquals(fileset.storageLocation(), context1.fileset().storageLocation()); + String subPath1 = "/test/test.parquet"; + String fileLocation1 = ops.getFileLocation(filesetIdent, subPath1); Assertions.assertEquals( - String.format("%s%s", context1.fileset().storageLocation(), dataOperationCtx1.subPath()), - context1.actualPath()); - Assertions.assertFalse(context1.fileset().properties().containsKey(StringIdentifier.ID_KEY)); - - BaseFilesetDataOperationCtx dataOperationCtx2 = - BaseFilesetDataOperationCtx.builder() - .withSubPath("test/test.parquet") - .withOperation(FilesetDataOperation.OPEN) - .withClientType(ClientType.HADOOP_GVFS) - .build(); - FilesetContext context2 = ops.getFilesetContext(filesetIdent, dataOperationCtx2); - Assertions.assertEquals(name, context2.fileset().name()); - Assertions.assertEquals(Fileset.Type.MANAGED, context2.fileset().type()); - Assertions.assertEquals("comment1024", context2.fileset().comment()); - Assertions.assertEquals(fileset.storageLocation(), context2.fileset().storageLocation()); - Assertions.assertFalse(context2.fileset().properties().containsKey(StringIdentifier.ID_KEY)); + String.format("%s%s", fileset.storageLocation(), subPath1), fileLocation1); + + String subPath2 = "test/test.parquet"; + String fileLocation2 = ops.getFileLocation(filesetIdent, subPath2); Assertions.assertEquals( - String.format("%s/%s", context2.fileset().storageLocation(), dataOperationCtx2.subPath()), - context2.actualPath()); + String.format("%s/%s", fileset.storageLocation(), subPath2), fileLocation2); } } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java index 041d6cc9e41..09c6cc29149 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/FilesetCatalog.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -29,23 +30,21 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.context.CallerContext; import org.apache.gravitino.dto.AuditDTO; import org.apache.gravitino.dto.CatalogDTO; import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; -import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; -import org.apache.gravitino.dto.responses.FilesetContextResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.rest.RESTUtils; /** @@ -231,34 +230,33 @@ public boolean dropFileset(NameIdentifier ident) { } /** - * Get a fileset context from the catalog. + * Get the actual path of a file or directory based on the storage location of Fileset and the sub + * path. * * @param ident A fileset identifier. - * @param ctx The data operation context. - * @return The fileset context. + * @param subPath The sub path to the file or directory. + * @return The actual location of the file or directory. + * @throws NoSuchFilesetException If the fileset does not exist. */ @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { checkFilesetNameIdentifier(ident); Namespace fullNamespace = getFilesetFullNamespace(ident.namespace()); - GetFilesetContextRequest req = - GetFilesetContextRequest.builder() - .subPath(ctx.subPath()) - .operation(ctx.operation()) - .clientType(ctx.clientType()) - .build(); - FilesetContextResponse resp = - restClient.post( - formatFilesetRequestPath(fullNamespace) + "/" + ident.name() + "/" + "context", - req, - FilesetContextResponse.class, - Collections.emptyMap(), + CallerContext callerContext = CallerContext.CallerContextHolder.get(); + Map queryParams = Maps.newHashMap(); + queryParams.put("subPath", subPath); + FileLocationResponse resp = + restClient.get( + formatFilesetRequestPath(fullNamespace) + "/" + ident.name() + "/" + "fileLocation", + queryParams, + FileLocationResponse.class, + callerContext != null ? callerContext.context() : Collections.emptyMap(), ErrorHandlers.filesetErrorHandler()); resp.validate(); - return resp.getContext(); + return resp.getFileLocation(); } @VisibleForTesting diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java index 1866692dc89..874e150bb19 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/TestFilesetCatalog.java @@ -22,45 +22,54 @@ import static org.apache.hc.core5.http.HttpStatus.SC_NOT_FOUND; import static org.apache.hc.core5.http.HttpStatus.SC_OK; import static org.apache.hc.core5.http.HttpStatus.SC_SERVER_ERROR; +import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.jsonwebtoken.lang.Maps; import java.nio.file.NoSuchFileException; import java.time.Instant; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.audit.FilesetAuditConstants; +import org.apache.gravitino.context.CallerContext; import org.apache.gravitino.dto.AuditDTO; import org.apache.gravitino.dto.CatalogDTO; -import org.apache.gravitino.dto.file.FilesetContextDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.requests.CatalogCreateRequest; import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; -import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.CatalogResponse; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; import org.apache.gravitino.dto.responses.ErrorResponse; -import org.apache.gravitino.dto.responses.FilesetContextResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; +import org.apache.gravitino.enums.FilesetDataOperation; +import org.apache.gravitino.enums.InternalClientType; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NotFoundException; -import org.apache.gravitino.file.BaseFilesetDataOperationCtx; -import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperation; import org.apache.hc.core5.http.Method; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; +import org.mockserver.matchers.Times; +import org.mockserver.model.HttpRequest; +import org.mockserver.model.HttpResponse; +import org.mockserver.model.Parameter; public class TestFilesetCatalog extends TestBase { @@ -408,44 +417,87 @@ public void testAlterFileset() throws JsonProcessingException { } @Test - public void testGetFilesetContext() throws JsonProcessingException { + public void testGetFileLocation() throws JsonProcessingException { NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + String mockSubPath = "mock_location/test"; String filesetPath = withSlash( - FilesetCatalog.formatFilesetRequestPath(fileset.namespace()) + "/fileset1/context"); - - FilesetDTO mockFileset = - mockFilesetDTO( - fileset.name(), - Fileset.Type.MANAGED, - "mock comment", - "mock location", - ImmutableMap.of("k1", "v1")); - String mockActualPath = "mock location/test"; - FilesetContextDTO mockContext = mockFilesetContextDTO(mockFileset, mockActualPath); - FilesetContextResponse resp = new FilesetContextResponse(mockContext); - GetFilesetContextRequest req = - GetFilesetContextRequest.builder() - .subPath("/test") - .operation(FilesetDataOperation.OPEN) - .clientType(ClientType.HADOOP_GVFS) - .build(); - buildMockResource(Method.POST, filesetPath, req, resp, SC_OK); - BaseFilesetDataOperationCtx ctx = - BaseFilesetDataOperationCtx.builder() - .withSubPath("/test") - .withOperation(FilesetDataOperation.OPEN) - .withClientType(ClientType.HADOOP_GVFS) - .build(); - FilesetContext filesetContext = + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/fileset1/fileLocation"); + Map queryParams = Maps.of("subPath", mockSubPath).build(); + String mockFileLocation = + String.format("file:/fileset/%s/%s/%s/%s", catalogName, "schema1", "fileset1", mockSubPath); + FileLocationResponse resp = new FileLocationResponse(mockFileLocation); + buildMockResource(Method.GET, filesetPath, queryParams, null, resp, SC_OK); + + String actualFileLocation = catalog .asFilesetCatalog() - .getFilesetContext( - NameIdentifier.of(fileset.namespace().level(2), fileset.name()), ctx); - Assertions.assertNotNull(filesetContext); - assertFileset(mockFileset, filesetContext.fileset()); + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath); + Assertions.assertTrue(StringUtils.isNotBlank(actualFileLocation)); + Assertions.assertEquals(mockFileLocation, actualFileLocation); + } - Assertions.assertEquals(mockActualPath, filesetContext.actualPath()); + @Test + public void testCallerContextToHeader() throws JsonProcessingException { + NameIdentifier fileset = NameIdentifier.of(metalakeName, catalogName, "schema1", "fileset1"); + String mockSubPath = "mock_location/test"; + String filesetPath = + withSlash( + FilesetCatalog.formatFilesetRequestPath( + Namespace.of(metalakeName, catalogName, "schema1")) + + "/fileset1/fileLocation"); + Map queryParams = Maps.of("subPath", mockSubPath).build(); + String mockFileLocation = + String.format("file:/fileset/%s/%s/%s/%s", catalogName, "schema1", "fileset1", mockSubPath); + FileLocationResponse resp = new FileLocationResponse(mockFileLocation); + String respJson = MAPPER.writeValueAsString(resp); + + List parameters = + queryParams.entrySet().stream() + .map(kv -> new Parameter(kv.getKey(), kv.getValue())) + .collect(Collectors.toList()); + HttpRequest mockRequest = + HttpRequest.request(filesetPath) + .withMethod(Method.GET.name()) + .withQueryStringParameters(parameters); + HttpResponse mockResponse = HttpResponse.response().withStatusCode(SC_OK).withBody(respJson); + + // set the thread local context + Map context = new HashMap<>(); + context.put( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, + InternalClientType.HADOOP_GVFS.name()); + context.put( + FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION, + FilesetDataOperation.GET_FILE_STATUS.name()); + CallerContext callerContext = CallerContext.builder().withContext(context).build(); + CallerContext.CallerContextHolder.set(callerContext); + + // Using Times.exactly(1) will only match once for the request, so we could set difference + // responses for the same request and path. + AtomicReference internalClientType = new AtomicReference<>(null); + AtomicReference dataOperation = new AtomicReference<>(null); + mockServer + .when(mockRequest, Times.exactly(1)) + .respond( + httpRequest -> { + internalClientType.set( + httpRequest.getFirstHeader( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); + dataOperation.set( + httpRequest.getFirstHeader( + FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); + return mockResponse; + }); + catalog + .asFilesetCatalog() + .getFileLocation( + NameIdentifier.of(fileset.namespace().level(2), fileset.name()), mockSubPath); + assertEquals(FilesetDataOperation.GET_FILE_STATUS.name(), dataOperation.get()); + assertEquals(InternalClientType.HADOOP_GVFS.name(), internalClientType.get()); } private FilesetDTO mockFilesetDTO( @@ -464,10 +516,6 @@ private FilesetDTO mockFilesetDTO( .build(); } - private FilesetContextDTO mockFilesetContextDTO(FilesetDTO fileset, String actualPath) { - return FilesetContextDTO.builder().fileset(fileset).actualPath(actualPath).build(); - } - private void assertFileset(FilesetDTO expected, Fileset actual) { Assertions.assertEquals(expected.name(), actual.name()); Assertions.assertEquals(expected.comment(), actual.comment()); diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetContext.java b/common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java similarity index 56% rename from api/src/main/java/org/apache/gravitino/file/FilesetContext.java rename to common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java index 4a62fa9d282..01184405df0 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetContext.java +++ b/common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java @@ -17,22 +17,14 @@ * under the License. */ -package org.apache.gravitino.file; +package org.apache.gravitino.audit; -import org.apache.gravitino.annotation.Evolving; +public class FilesetAuditConstants { + private FilesetAuditConstants() {} -/** - * An interface representing a fileset context with an existing fileset {@link Fileset}. This - * interface defines some contextual information related to Fileset that can be passed. - * - *

{@link FilesetContext} defines the basic properties of a fileset context object. A catalog - * implementation with {@link FilesetCatalog} should implement this interface. - */ -@Evolving -public interface FilesetContext { - /** @return The fileset object. */ - Fileset fileset(); + /** The HTTP header used to pass the internal client type. */ + public static final String HTTP_HEADER_INTERNAL_CLIENT_TYPE = "InternalClientType"; - /** @return The actual storage path after processing. */ - String actualPath(); + /** The HTTP header used to pass the fileset data operation. */ + public static final String HTTP_HEADER_FILESET_DATA_OPERATION = "FilesetDataOperation"; } diff --git a/common/src/main/java/org/apache/gravitino/context/CallerContext.java b/common/src/main/java/org/apache/gravitino/context/CallerContext.java new file mode 100644 index 00000000000..85c0dfbe016 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/context/CallerContext.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.context; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import java.util.Map; + +/** + * A class defining the caller context for auditing coarse granularity operations. + * + *

Reference: + * + *

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/\ ipc/CallerContext.java + */ +public class CallerContext { + private Map context; + + private CallerContext() {} + + public Map context() { + return context; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof CallerContext)) return false; + CallerContext context1 = (CallerContext) o; + return Objects.equal(context, context1.context); + } + + @Override + public int hashCode() { + return Objects.hashCode(context); + } + + public static class Builder { + private final CallerContext callerContext; + + private Builder() { + callerContext = new CallerContext(); + } + + /** + * Sets the context for CallerContext + * + * @param context The context to set. + * @return This Builder instance for method chaining. + */ + public CallerContext.Builder withContext(Map context) { + callerContext.context = context; + return this; + } + + private void validate() { + Preconditions.checkArgument(callerContext.context != null, "context cannot be null"); + } + + public CallerContext build() { + validate(); + return callerContext; + } + } + + /** + * Create a new builder for the CallerContext. + * + * @return A new builder for the CallerContext. + */ + public static CallerContext.Builder builder() { + return new CallerContext.Builder(); + } + + public static class CallerContextHolder { + + private static final ThreadLocal CALLER_CONTEXT = new ThreadLocal<>(); + + public static CallerContext get() { + return CALLER_CONTEXT.get(); + } + + public static void set(CallerContext context) { + CALLER_CONTEXT.set(context); + } + + public static void remove() { + CALLER_CONTEXT.remove(); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java b/common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java deleted file mode 100644 index 3334ca7b841..00000000000 --- a/common/src/main/java/org/apache/gravitino/dto/file/FilesetContextDTO.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.dto.file; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; -import org.apache.gravitino.file.FilesetContext; - -/** Represents a Fileset context DTO (Data Transfer Object). */ -@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) -@AllArgsConstructor(access = AccessLevel.PRIVATE) -@EqualsAndHashCode -public class FilesetContextDTO implements FilesetContext { - @JsonProperty("fileset") - private FilesetDTO fileset; - - @JsonProperty("actualPath") - private String actualPath; - - public FilesetDTO fileset() { - return fileset; - } - - public String actualPath() { - return actualPath; - } - - @Builder(builderMethodName = "builder") - private static FilesetContextDTO internalBuilder(FilesetDTO fileset, String actualPath) { - Preconditions.checkNotNull(fileset, "fileset cannot be null"); - Preconditions.checkNotNull(actualPath, "actual path cannot be null"); - - return new FilesetContextDTO(fileset, actualPath); - } -} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java deleted file mode 100644 index f8faba08497..00000000000 --- a/common/src/main/java/org/apache/gravitino/dto/requests/GetFilesetContextRequest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.dto.requests; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.ToString; -import org.apache.gravitino.file.ClientType; -import org.apache.gravitino.file.FilesetDataOperation; -import org.apache.gravitino.rest.RESTRequest; - -/** Request to get a fileset context. */ -@Getter -@EqualsAndHashCode -@NoArgsConstructor(force = true) -@AllArgsConstructor -@Builder -@ToString -public class GetFilesetContextRequest implements RESTRequest { - @JsonProperty("subPath") - private String subPath; - - @JsonProperty("operation") - private FilesetDataOperation operation; - - @JsonProperty("clientType") - private ClientType clientType; - - @Override - public void validate() throws IllegalArgumentException { - Preconditions.checkArgument(subPath != null, "\"subPath\" field cannot be null"); - Preconditions.checkArgument( - operation != null, "\"operation\" field is required and cannot be null"); - Preconditions.checkArgument( - clientType != null, "\"clientType\" field is required and cannot be null"); - } -} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/FileLocationResponse.java similarity index 68% rename from common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java rename to common/src/main/java/org/apache/gravitino/dto/responses/FileLocationResponse.java index 30e980ac38a..42da22ab5b8 100644 --- a/common/src/main/java/org/apache/gravitino/dto/responses/FilesetContextResponse.java +++ b/common/src/main/java/org/apache/gravitino/dto/responses/FileLocationResponse.java @@ -24,30 +24,30 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.apache.gravitino.dto.file.FilesetContextDTO; +import org.apache.commons.lang3.StringUtils; -/** Response for fileset context. */ +/** Response for the actual file location. */ @Getter @ToString @EqualsAndHashCode(callSuper = true) -public class FilesetContextResponse extends BaseResponse { - @JsonProperty("filesetContext") - private final FilesetContextDTO context; +public class FileLocationResponse extends BaseResponse { + @JsonProperty("fileLocation") + private final String fileLocation; - /** Constructor for FilesetContextResponse. */ - public FilesetContextResponse() { + /** Constructor for FileLocationResponse. */ + public FileLocationResponse() { super(0); - this.context = null; + this.fileLocation = null; } /** - * Constructor for FilesetContextResponse. + * Constructor for FileLocationResponse. * - * @param context the fileset context DTO. + * @param fileLocation the actual file location. */ - public FilesetContextResponse(FilesetContextDTO context) { + public FileLocationResponse(String fileLocation) { super(0); - this.context = context; + this.fileLocation = fileLocation; } /** @@ -58,6 +58,7 @@ public FilesetContextResponse(FilesetContextDTO context) { @Override public void validate() throws IllegalArgumentException { super.validate(); - Preconditions.checkArgument(context != null, "context must not be null"); + Preconditions.checkArgument( + StringUtils.isNotBlank(fileLocation), "fileLocation must not be null"); } } diff --git a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java index 06894f22eae..d83460af182 100644 --- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java @@ -45,7 +45,6 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; -import org.apache.gravitino.dto.file.FilesetContextDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.messaging.TopicDTO; import org.apache.gravitino.dto.rel.ColumnDTO; @@ -76,7 +75,6 @@ import org.apache.gravitino.dto.tag.MetadataObjectDTO; import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.messaging.Topic; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; @@ -587,19 +585,6 @@ public static FilesetDTO toDTO(Fileset fileset) { .build(); } - /** - * Converts to a FilesetContextDTO. - * - * @param context The fileset context to be converted. - * @return The fileset context DTO. - */ - public static FilesetContextDTO toDTO(FilesetContext context) { - return FilesetContextDTO.builder() - .fileset(toDTO(context.fileset())) - .actualPath(context.actualPath()) - .build(); - } - /** * Converts a Topic to a TopicDTO. * diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java b/common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java similarity index 86% rename from api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java rename to common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java index 265815d027c..85e3f7d768f 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperation.java +++ b/common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.gravitino.file; +package org.apache.gravitino.enums; /** An enum class containing fileset data operations that supported. */ public enum FilesetDataOperation { @@ -45,4 +45,13 @@ public enum FilesetDataOperation { SET_WORKING_DIR, /** This data operation means that it is an unknown data operation. */ UNKNOWN; + + public static boolean checkValid(String operation) { + try { + FilesetDataOperation.valueOf(operation); + return true; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown fileset data operation: " + operation, e); + } + } } diff --git a/api/src/main/java/org/apache/gravitino/file/ClientType.java b/common/src/main/java/org/apache/gravitino/enums/InternalClientType.java similarity index 73% rename from api/src/main/java/org/apache/gravitino/file/ClientType.java rename to common/src/main/java/org/apache/gravitino/enums/InternalClientType.java index 37359b6d181..31e00ae1f80 100644 --- a/api/src/main/java/org/apache/gravitino/file/ClientType.java +++ b/common/src/main/java/org/apache/gravitino/enums/InternalClientType.java @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.gravitino.file; +package org.apache.gravitino.enums; -/** An enum class containing fileset data operations client type that supported. */ -public enum ClientType { +/** An enum class containing internal client type that supported. */ +public enum InternalClientType { /** * The client type is `org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem` which in * the filesystem-hadoop3 module. @@ -33,4 +33,13 @@ public enum ClientType { PYTHON_GVFS, /** The client type is unknown. */ UNKNOWN; + + public static boolean checkValid(String clientType) { + try { + InternalClientType.valueOf(clientType); + return true; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown internal client type: " + clientType, e); + } + } } diff --git a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java b/common/src/test/java/org/apache/gravitino/context/TestCallerContext.java similarity index 52% rename from api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java rename to common/src/test/java/org/apache/gravitino/context/TestCallerContext.java index af54a393b6a..5d281069a00 100644 --- a/api/src/main/java/org/apache/gravitino/file/FilesetDataOperationCtx.java +++ b/common/src/test/java/org/apache/gravitino/context/TestCallerContext.java @@ -17,25 +17,25 @@ * under the License. */ -package org.apache.gravitino.file; +package org.apache.gravitino.context; -import org.apache.gravitino.annotation.Evolving; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -/** - * An interface representing a fileset data operation context. This interface defines some - * information need to report to the server. - * - *

{@link FilesetDataOperationCtx} defines the basic properties of a fileset data operation - * context object. - */ -@Evolving -public interface FilesetDataOperationCtx { - /** @return The sub path which is operated by the data operation . */ - String subPath(); +public class TestCallerContext { + @Test + public void testCallerContext() { + Map contextMap = new HashMap<>(); + contextMap.put("test", "test"); + contextMap.put("test2", "test2"); + CallerContext callerContext = CallerContext.builder().withContext(contextMap).build(); + CallerContext.CallerContextHolder.set(callerContext); - /** @return The data operation type. */ - FilesetDataOperation operation(); + CallerContext actualCallerContext = CallerContext.CallerContextHolder.get(); - /** @return The client type of the data operation. */ - ClientType clientType(); + Assertions.assertEquals(callerContext, actualCallerContext); + Assertions.assertEquals(callerContext.context(), actualCallerContext.context()); + } } diff --git a/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java b/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java deleted file mode 100644 index cf53cd10744..00000000000 --- a/common/src/test/java/org/apache/gravitino/dto/file/TestFilesetContextDTO.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.dto.file; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import java.time.Instant; -import org.apache.gravitino.dto.AuditDTO; -import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.json.JsonUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TestFilesetContextDTO { - @Test - void testJsonSerDe() throws JsonProcessingException { - FilesetDTO filesetDTO = - FilesetDTO.builder() - .name("test") - .type(Fileset.Type.MANAGED) - .audit(AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build()) - .storageLocation("hdfs://host/test") - .build(); - - FilesetContextDTO dto = - FilesetContextDTO.builder() - .fileset(filesetDTO) - .actualPath("hdfs://host/test/1.txt") - .build(); - String value = JsonUtils.objectMapper().writeValueAsString(dto); - - String expectedValue = - String.format( - "{\n" - + " \"fileset\": {\n" - + " \"name\": \"test\",\n" - + " \"comment\": null,\n" - + " \"type\": \"managed\",\n" - + " \"storageLocation\": \"hdfs://host/test\",\n" - + " \"properties\": null,\n" - + " \"audit\": {\n" - + " \"creator\": \"creator\",\n" - + " \"createTime\": \"%s\",\n" - + " \"lastModifier\": null,\n" - + " \"lastModifiedTime\": null\n" - + " }\n" - + " },\n" - + " \"actualPath\": \"hdfs://host/test/1.txt\"\n" - + "}", - filesetDTO.auditInfo().createTime()); - JsonNode expected = JsonUtils.objectMapper().readTree(expectedValue); - JsonNode actual = JsonUtils.objectMapper().readTree(value); - Assertions.assertEquals(expected, actual); - } -} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java deleted file mode 100644 index 49af4f82e4f..00000000000 --- a/common/src/test/java/org/apache/gravitino/dto/requests/TestGetFilesetContextRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.dto.requests; - -import com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.gravitino.file.ClientType; -import org.apache.gravitino.file.FilesetDataOperation; -import org.apache.gravitino.json.JsonUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TestGetFilesetContextRequest { - @Test - public void testGetFilesetContextRequest() throws JsonProcessingException { - GetFilesetContextRequest request = - GetFilesetContextRequest.builder() - .subPath("/test/1.txt") - .operation(FilesetDataOperation.CREATE) - .clientType(ClientType.HADOOP_GVFS) - .build(); - String jsonString = JsonUtils.objectMapper().writeValueAsString(request); - String expected = - "{\"subPath\":\"/test/1.txt\",\"operation\":\"create\",\"clientType\":\"hadoop_gvfs\"}"; - Assertions.assertEquals( - JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(jsonString)); - } -} diff --git a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java index 9c403e43496..5f947820222 100644 --- a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java @@ -41,14 +41,11 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; -import org.apache.gravitino.dto.file.FilesetContextDTO; -import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.TableDTO; import org.apache.gravitino.dto.rel.partitioning.Partitioning; import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.dto.util.DTOConverters; -import org.apache.gravitino.file.Fileset; import org.apache.gravitino.json.JsonUtils; import org.apache.gravitino.rel.types.Types; import org.junit.jupiter.api.Test; @@ -382,28 +379,15 @@ void testTagResponse() throws JsonProcessingException { } @Test - void testFilesetContextResponse() { - AuditDTO audit = - AuditDTO.builder().withCreator("creator").withCreateTime(Instant.now()).build(); - FilesetDTO filesetDTO = - FilesetDTO.builder() - .name("test") - .type(Fileset.Type.MANAGED) - .audit(audit) - .storageLocation("hdfs://host/test") - .build(); - FilesetContextDTO dto = - FilesetContextDTO.builder() - .fileset(filesetDTO) - .actualPath("hdfs://host/test/zz.parquet") - .build(); - FilesetContextResponse response = new FilesetContextResponse(dto); + void testFileLocationResponse() { + String actualFileLocation = "file:/test/1"; + FileLocationResponse response = new FileLocationResponse(actualFileLocation); response.validate(); // No exception thrown } @Test - void testFilesetContextResponseException() { - FilesetContextResponse response = new FilesetContextResponse(); + void testFileLocationResponseException() { + FileLocationResponse response = new FileLocationResponse(); assertThrows(IllegalArgumentException.class, () -> response.validate()); } } diff --git a/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java b/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java index 5eb4347d0f4..46a2ffa11b2 100644 --- a/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java +++ b/common/src/test/java/org/apache/gravitino/dto/util/TestDTOConverters.java @@ -19,12 +19,9 @@ package org.apache.gravitino.dto.util; -import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Map; -import org.apache.gravitino.Audit; -import org.apache.gravitino.dto.file.FilesetContextDTO; import org.apache.gravitino.dto.rel.expressions.LiteralDTO; import org.apache.gravitino.dto.rel.partitioning.ListPartitioningDTO; import org.apache.gravitino.dto.rel.partitioning.RangePartitioningDTO; @@ -32,8 +29,6 @@ import org.apache.gravitino.dto.rel.partitions.ListPartitionDTO; import org.apache.gravitino.dto.rel.partitions.PartitionDTO; import org.apache.gravitino.dto.rel.partitions.RangePartitionDTO; -import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.rel.expressions.literals.Literal; import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.expressions.transforms.Transform; @@ -251,72 +246,4 @@ void testPartitionDTOConvert() { Types.StringType.get().simpleString(), listPartitionAssignments[0].lists()[0][0].value()); Assertions.assertEquals(properties, listPartitionAssignments[0].properties()); } - - @Test - void testFilesetContextConvert() { - FilesetContext filesetContext = - new FilesetContext() { - @Override - public Fileset fileset() { - Fileset mockFileset = - new Fileset() { - @Override - public String name() { - return "test"; - } - - @Override - public Type type() { - return Type.MANAGED; - } - - @Override - public String storageLocation() { - return "hdfs://host/test"; - } - - @Override - public Audit auditInfo() { - Audit mockAudit = - new Audit() { - @Override - public String creator() { - return null; - } - - @Override - public Instant createTime() { - return null; - } - - @Override - public String lastModifier() { - return null; - } - - @Override - public Instant lastModifiedTime() { - return null; - } - }; - return mockAudit; - } - }; - return mockFileset; - } - - @Override - public String actualPath() { - return "hdfs://host/test/1.txt"; - } - }; - - FilesetContextDTO filesetContextDTO = DTOConverters.toDTO(filesetContext); - - // then - Assertions.assertEquals("test", filesetContextDTO.fileset().name()); - Assertions.assertEquals(Fileset.Type.MANAGED, filesetContextDTO.fileset().type()); - Assertions.assertEquals("hdfs://host/test", filesetContextDTO.fileset().storageLocation()); - Assertions.assertEquals("hdfs://host/test/1.txt", filesetContextDTO.actualPath()); - } } diff --git a/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java index debed164155..e1c20de702f 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/FilesetNormalizeDispatcher.java @@ -32,8 +32,6 @@ import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; public class FilesetNormalizeDispatcher implements FilesetDispatcher { private final CatalogManager catalogManager; @@ -97,10 +95,10 @@ public boolean dropFileset(NameIdentifier ident) { } @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) { + public String getFileLocation(NameIdentifier ident, String subPath) { // The constraints of the name spec may be more strict than underlying catalog, // and for compatibility reasons, we only apply case-sensitive capabilities here. - return dispatcher.getFilesetContext(normalizeCaseSensitive(ident), ctx); + return dispatcher.getFileLocation(normalizeCaseSensitive(ident), subPath); } private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) { diff --git a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java index 1c6e7131d94..98c6311bd7c 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/FilesetOperationDispatcher.java @@ -33,8 +33,6 @@ import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.storage.IdGenerator; public class FilesetOperationDispatcher extends OperationDispatcher implements FilesetDispatcher { @@ -199,18 +197,20 @@ public boolean dropFileset(NameIdentifier ident) { } /** - * Get a fileset context from the catalog. + * Get the actual location of a file or directory based on the storage location of Fileset and the + * sub path. * * @param ident A fileset identifier. - * @param ctx A fileset data operation context. - * @return true If the fileset is dropped, false the fileset did not exist. + * @param subPath The sub path to the file or directory. + * @return The actual location of the file or directory. + * @throws NoSuchFilesetException If the fileset does not exist. */ @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { return doWithCatalog( getCatalogIdentifier(ident), - c -> c.doWithFilesetOps(f -> f.getFilesetContext(ident, ctx)), + c -> c.doWithFilesetOps(f -> f.getFileLocation(ident, subPath)), NonEmptyEntityException.class); } } diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java b/core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java deleted file mode 100644 index 14e0b86b905..00000000000 --- a/core/src/main/java/org/apache/gravitino/connector/BaseFilesetContext.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.connector; - -import org.apache.gravitino.annotation.Evolving; -import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.file.FilesetCatalog; -import org.apache.gravitino.file.FilesetContext; - -/** - * An abstract class representing a base fileset context for {@link FilesetCatalog}. Developers - * should extend this class to implement a custom fileset context for their fileset catalog. - */ -@Evolving -public abstract class BaseFilesetContext implements FilesetContext { - protected Fileset fileset; - protected String actualPath; - - @Override - public Fileset fileset() { - return fileset; - } - - @Override - public String actualPath() { - return actualPath; - } - - interface Builder< - SELF extends BaseFilesetContext.Builder, T extends BaseFilesetContext> { - - SELF withFileset(Fileset fileset); - - SELF withActualPath(String actualPath); - - T build(); - } - - /** - * An abstract class implementing the builder interface for {@link BaseFilesetContext}. This class - * should be extended by the concrete fileset context builders. - * - * @param The type of the builder. - * @param The type of the fileset being built. - */ - public abstract static class BaseFilesetContextBuilder< - SELF extends BaseFilesetContext.Builder, T extends BaseFilesetContext> - implements BaseFilesetContext.Builder { - protected Fileset fileset; - protected String actualPath; - - /** - * Sets the fileset of the fileset context. - * - * @param fileset The fileset object. - * @return The builder instance. - */ - @Override - public SELF withFileset(Fileset fileset) { - this.fileset = fileset; - return self(); - } - - /** - * Sets the actual path of the fileset context. - * - * @param actualPath The actual path for the fileset context. - * @return The builder instance. - */ - @Override - public SELF withActualPath(String actualPath) { - this.actualPath = actualPath; - return self(); - } - - /** - * Builds the instance of the fileset with the provided attributes. - * - * @return The built fileset instance. - */ - @Override - public T build() { - T t = internalBuild(); - return t; - } - - private SELF self() { - return (SELF) this; - } - - /** - * Builds the concrete instance of the fileset with the provided attributes. - * - * @return The built fileset instance. - */ - @Evolving - protected abstract T internalBuild(); - } -} diff --git a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java index cb890033f0f..a7e1a770332 100644 --- a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java @@ -23,21 +23,20 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.context.CallerContext; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.listener.api.event.AlterFilesetEvent; import org.apache.gravitino.listener.api.event.AlterFilesetFailureEvent; import org.apache.gravitino.listener.api.event.CreateFilesetEvent; import org.apache.gravitino.listener.api.event.CreateFilesetFailureEvent; import org.apache.gravitino.listener.api.event.DropFilesetEvent; import org.apache.gravitino.listener.api.event.DropFilesetFailureEvent; -import org.apache.gravitino.listener.api.event.GetFilesetContextEvent; -import org.apache.gravitino.listener.api.event.GetFilesetContextFailureEvent; +import org.apache.gravitino.listener.api.event.GetFileLocationEvent; +import org.apache.gravitino.listener.api.event.GetFileLocationFailureEvent; import org.apache.gravitino.listener.api.event.ListFilesetEvent; import org.apache.gravitino.listener.api.event.ListFilesetFailureEvent; import org.apache.gravitino.listener.api.event.LoadFilesetEvent; @@ -144,16 +143,19 @@ public boolean dropFileset(NameIdentifier ident) { } @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) + public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { try { - FilesetContext context = dispatcher.getFilesetContext(ident, ctx); + String actualFileLocation = dispatcher.getFileLocation(ident, subPath); + // get the audit info from the thread local context + CallerContext context = CallerContext.CallerContextHolder.get(); eventBus.dispatchEvent( - new GetFilesetContextEvent(PrincipalUtils.getCurrentUserName(), ident, ctx)); - return context; + new GetFileLocationEvent( + PrincipalUtils.getCurrentUserName(), ident, actualFileLocation, context)); + return actualFileLocation; } catch (Exception e) { eventBus.dispatchEvent( - new GetFilesetContextFailureEvent(PrincipalUtils.getCurrentUserName(), ident, ctx, e)); + new GetFileLocationFailureEvent(PrincipalUtils.getCurrentUserName(), ident, subPath, e)); throw e; } } diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java new file mode 100644 index 00000000000..b1597766d1f --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.listener.api.event; + +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.annotation.DeveloperApi; +import org.apache.gravitino.context.CallerContext; + +/** Represents an event that occurs when getting a actual file location. */ +@DeveloperApi +public final class GetFileLocationEvent extends FilesetEvent { + private final String actualFileLocation; + + private final CallerContext callerContext; + + /** + * Constructs a new {@code GetFileLocationEvent}, recording the attempt to get a file location. + * + * @param user The user who initiated the get file location. + * @param identifier The identifier of the file location that was attempted to be got. + * @param callerContext The audit caller context, this param can be null. + * @param actualFileLocation The actual file location which want to get. + */ + public GetFileLocationEvent( + String user, + NameIdentifier identifier, + String actualFileLocation, + CallerContext callerContext) { + super(user, identifier); + this.actualFileLocation = actualFileLocation; + this.callerContext = callerContext; + } + + public String actualFileLocation() { + return actualFileLocation; + } + + public CallerContext callerContext() { + return callerContext; + } +} diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java similarity index 56% rename from core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java rename to core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java index 27d44c7c159..a79b88ba858 100644 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextFailureEvent.java +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java @@ -21,32 +21,31 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.annotation.DeveloperApi; -import org.apache.gravitino.file.FilesetDataOperationCtx; /** - * Represents an event that is generated when an attempt to get a fileset context from the system + * Represents an event that is generated when an attempt to get a file location from the system * fails. */ @DeveloperApi -public final class GetFilesetContextFailureEvent extends FilesetFailureEvent { - private final FilesetDataOperationCtx ctx; +public final class GetFileLocationFailureEvent extends FilesetFailureEvent { + private final String subPath; /** - * Constructs a new {@code GetFilesetContextFailureEvent}. + * Constructs a new {@code GetFileLocationFailureEvent}. * - * @param user The user who initiated the get fileset context operation. - * @param identifier The identifier of the fileset context that was attempted to be got. - * @param ctx The data operation context to get the fileset context. - * @param exception The exception that was thrown during the get fileset context operation. This - * exception is key to diagnosing the failure, providing insights into what went wrong during - * the operation. + * @param user The user who initiated the get a file location. + * @param identifier The identifier of the file location that was attempted to be got. + * @param subPath The sub path of the actual file location which want to get. + * @param exception The exception that was thrown during the get a file location. This exception + * is key to diagnosing the failure, providing insights into what went wrong during the + * operation. */ - public GetFilesetContextFailureEvent( - String user, NameIdentifier identifier, FilesetDataOperationCtx ctx, Exception exception) { + public GetFileLocationFailureEvent( + String user, NameIdentifier identifier, String subPath, Exception exception) { super(user, identifier, exception); - this.ctx = ctx; + this.subPath = subPath; } - public FilesetDataOperationCtx dataOperationContext() { - return ctx; + public String subPath() { + return subPath; } } diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java deleted file mode 100644 index 184c4aaca65..00000000000 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFilesetContextEvent.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.listener.api.event; - -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.annotation.DeveloperApi; -import org.apache.gravitino.file.FilesetDataOperationCtx; - -/** Represents an event that occurs when getting a fileset context. */ -@DeveloperApi -public final class GetFilesetContextEvent extends FilesetEvent { - private final FilesetDataOperationCtx ctx; - - /** - * Constructs a new {@code GetFilesetContextEvent}, recording the attempt to get a fileset - * context. - * - * @param user The user who initiated the get fileset context operation. - * @param identifier The identifier of the fileset context that was attempted to be got. - * @param ctx The data operation context to get the fileset context. - */ - public GetFilesetContextEvent( - String user, NameIdentifier identifier, FilesetDataOperationCtx ctx) { - super(user, identifier); - this.ctx = ctx; - } - - public FilesetDataOperationCtx dataOperationContext() { - return ctx; - } -} diff --git a/core/src/test/java/org/apache/gravitino/TestFilesetContext.java b/core/src/test/java/org/apache/gravitino/TestFilesetContext.java deleted file mode 100644 index a960d8823a0..00000000000 --- a/core/src/test/java/org/apache/gravitino/TestFilesetContext.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino; - -import lombok.EqualsAndHashCode; -import org.apache.gravitino.connector.BaseFilesetContext; - -@EqualsAndHashCode(callSuper = true) -public class TestFilesetContext extends BaseFilesetContext { - public static class Builder - extends BaseFilesetContext.BaseFilesetContextBuilder< - TestFilesetContext.Builder, TestFilesetContext> { - /** Creates a new instance of {@link TestFilesetContext.Builder}. */ - private Builder() {} - - @Override - protected TestFilesetContext internalBuild() { - TestFilesetContext context = new TestFilesetContext(); - context.fileset = fileset; - context.actualPath = actualPath; - return context; - } - } - - /** - * Creates a new instance of {@link TestFilesetContext.Builder}. - * - * @return The new instance. - */ - public static TestFilesetContext.Builder builder() { - return new TestFilesetContext.Builder(); - } -} diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java index a6ceca3c987..2c1dce93874 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestFilesetOperationDispatcher.java @@ -27,12 +27,8 @@ import java.util.UUID; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; -import org.apache.gravitino.file.BaseFilesetDataOperationCtx; -import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperation; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -177,8 +173,8 @@ public void testCreateAndDropFileset() { } @Test - public void testCreateAndGetFilesetContext() { - String tmpDir = "/tmp/test_get_fileset_context_" + UUID.randomUUID(); + public void testCreateAndGetFileLocation() { + String tmpDir = "/tmp/test_get_file_location_" + UUID.randomUUID(); try { Namespace filesetNs = Namespace.of(metalake, catalog, "schema1024"); Map props = ImmutableMap.of("k1", "v1", "location", "schema1024"); @@ -194,22 +190,9 @@ public void testCreateAndGetFilesetContext() { testProperties(props, fileset1.properties()); Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); Assertions.assertNotNull(fileset1.storageLocation()); - - BaseFilesetDataOperationCtx ctx = - BaseFilesetDataOperationCtx.builder() - .withSubPath("/test/x.parquet") - .withOperation(FilesetDataOperation.CREATE) - .withClientType(ClientType.HADOOP_GVFS) - .build(); - FilesetContext context1 = filesetOperationDispatcher.getFilesetContext(filesetIdent1, ctx); - Assertions.assertEquals(fileset1.name(), context1.fileset().name()); - Assertions.assertEquals(fileset1.comment(), context1.fileset().comment()); - testProperties(props, context1.fileset().properties()); - Assertions.assertEquals(fileset1.type(), context1.fileset().type()); - Assertions.assertEquals(fileset1.storageLocation(), context1.fileset().storageLocation()); - - Assertions.assertEquals( - fileset1.storageLocation() + "/test/x.parquet", context1.actualPath()); + String subPath = "/test/x.parquet"; + String fileLocation = filesetOperationDispatcher.getFileLocation(filesetIdent1, subPath); + Assertions.assertEquals(fileset1.storageLocation() + subPath, fileLocation); } finally { File path = new File(tmpDir); if (path.exists()) { diff --git a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java index 0668cf68e93..ddd67e40d88 100644 --- a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java +++ b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java @@ -24,7 +24,6 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; @@ -32,11 +31,9 @@ import org.apache.gravitino.Schema; import org.apache.gravitino.SchemaChange; import org.apache.gravitino.TestFileset; -import org.apache.gravitino.TestFilesetContext; import org.apache.gravitino.TestSchema; import org.apache.gravitino.TestTable; import org.apache.gravitino.TestTopic; -import org.apache.gravitino.catalog.EntityCombinedFileset; import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchCatalogException; @@ -51,8 +48,6 @@ import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetCatalog; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.messaging.DataLayout; import org.apache.gravitino.messaging.Topic; import org.apache.gravitino.messaging.TopicCatalog; @@ -70,8 +65,6 @@ public class TestCatalogOperations implements CatalogOperations, TableCatalog, FilesetCatalog, TopicCatalog, SupportsSchemas { - private HasPropertyMetadata propertiesMetadata; - private final Map tables; private final Map schemas; @@ -94,9 +87,7 @@ public TestCatalogOperations(Map config) { @Override public void initialize( Map config, CatalogInfo info, HasPropertyMetadata propertyMetadata) - throws RuntimeException { - this.propertiesMetadata = propertyMetadata; - } + throws RuntimeException {} @Override public void close() throws IOException {} @@ -442,33 +433,24 @@ public boolean dropFileset(NameIdentifier ident) { } @Override - public FilesetContext getFilesetContext(NameIdentifier ident, FilesetDataOperationCtx ctx) { - String subPath = ctx.subPath(); + public String getFileLocation(NameIdentifier ident, String subPath) { Preconditions.checkArgument(subPath != null, "subPath must not be null"); Fileset fileset = loadFileset(ident); String storageLocation = fileset.storageLocation(); - String actualPath; + String fileLocation; // subPath cannot be null, so we only need check if it is blank if (StringUtils.isBlank(subPath)) { - actualPath = storageLocation; + fileLocation = storageLocation; } else { - actualPath = + fileLocation = subPath.startsWith("/") ? String.format("%s%s", storageLocation, subPath) : String.format("%s/%s", storageLocation, subPath); } - return TestFilesetContext.builder() - .withFileset( - EntityCombinedFileset.of(fileset) - .withHiddenPropertiesSet( - fileset.properties().keySet().stream() - .filter(propertiesMetadata.filesetPropertiesMetadata()::isHiddenProperty) - .collect(Collectors.toSet()))) - .withActualPath(actualPath) - .build(); + return fileLocation; } @Override diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java index cd040e2431e..fb23b95ef3a 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java @@ -19,16 +19,22 @@ package org.apache.gravitino.listener.api.event; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Map; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.context.CallerContext; +import org.apache.gravitino.enums.FilesetDataOperation; +import org.apache.gravitino.enums.InternalClientType; import org.apache.gravitino.exceptions.GravitinoRuntimeException; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; @@ -70,8 +76,8 @@ void testCreateFilesetEvent() { fileset.storageLocation(), fileset.properties()); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(CreateFilesetEvent.class, event.getClass()); + assertEquals(identifier, event.identifier()); + assertEquals(CreateFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((CreateFilesetEvent) event).createdFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); } @@ -81,8 +87,8 @@ void testLoadFilesetEvent() { NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); dispatcher.loadFileset(identifier); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(LoadFilesetEvent.class, event.getClass()); + assertEquals(identifier, event.identifier()); + assertEquals(LoadFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((LoadFilesetEvent) event).loadedFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); } @@ -93,12 +99,12 @@ void testAlterFilesetEvent() { FilesetChange change = FilesetChange.setProperty("a", "b"); dispatcher.alterFileset(identifier, change); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(AlterFilesetEvent.class, event.getClass()); + assertEquals(identifier, event.identifier()); + assertEquals(AlterFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((AlterFilesetEvent) event).updatedFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); - Assertions.assertEquals(1, ((AlterFilesetEvent) event).filesetChanges().length); - Assertions.assertEquals(change, ((AlterFilesetEvent) event).filesetChanges()[0]); + assertEquals(1, ((AlterFilesetEvent) event).filesetChanges().length); + assertEquals(change, ((AlterFilesetEvent) event).filesetChanges()[0]); } @Test @@ -106,8 +112,8 @@ void testDropFilesetEvent() { NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); dispatcher.dropFileset(identifier); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(DropFilesetEvent.class, event.getClass()); + assertEquals(identifier, event.identifier()); + assertEquals(DropFilesetEvent.class, event.getClass()); Assertions.assertTrue(((DropFilesetEvent) event).isExists()); } @@ -116,9 +122,51 @@ void testListFilesetEvent() { Namespace namespace = Namespace.of("metalake", "catalog"); dispatcher.listFilesets(namespace); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(namespace.toString(), event.identifier().toString()); - Assertions.assertEquals(ListFilesetEvent.class, event.getClass()); - Assertions.assertEquals(namespace, ((ListFilesetEvent) event).namespace()); + assertEquals(namespace.toString(), event.identifier().toString()); + assertEquals(ListFilesetEvent.class, event.getClass()); + assertEquals(namespace, ((ListFilesetEvent) event).namespace()); + } + + @Test + void testGetFileLocationEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); + dispatcher.createFileset( + identifier, + fileset.comment(), + fileset.type(), + fileset.storageLocation(), + fileset.properties()); + Event event = dummyEventListener.popEvent(); + assertEquals(identifier, event.identifier()); + assertEquals(CreateFilesetEvent.class, event.getClass()); + FilesetInfo filesetInfo = ((CreateFilesetEvent) event).createdFilesetInfo(); + checkFilesetInfo(filesetInfo, fileset); + + Map contextMap = Maps.newHashMap(); + contextMap.put( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, + InternalClientType.HADOOP_GVFS.name()); + contextMap.put( + FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION, + FilesetDataOperation.GET_FILE_STATUS.name()); + CallerContext callerContext = CallerContext.builder().withContext(contextMap).build(); + CallerContext.CallerContextHolder.set(callerContext); + String fileLocation = dispatcher.getFileLocation(identifier, "test"); + Event event1 = dummyEventListener.popEvent(); + assertEquals(identifier, event1.identifier()); + assertEquals(GetFileLocationEvent.class, event1.getClass()); + String actualFileLocation = ((GetFileLocationEvent) event1).actualFileLocation(); + assertEquals(actualFileLocation, fileLocation); + CallerContext actualCallerContext = ((GetFileLocationEvent) event1).callerContext(); + assertEquals(2, actualCallerContext.context().size()); + assertEquals( + InternalClientType.HADOOP_GVFS.name(), + actualCallerContext.context().get(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); + assertEquals( + FilesetDataOperation.GET_FILE_STATUS.name(), + actualCallerContext + .context() + .get(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); } @Test @@ -134,9 +182,9 @@ void testCreateSchemaFailureEvent() { fileset.storageLocation(), fileset.properties())); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(CreateFilesetFailureEvent.class, event.getClass()); - Assertions.assertEquals( + assertEquals(identifier, event.identifier()); + assertEquals(CreateFilesetFailureEvent.class, event.getClass()); + assertEquals( GravitinoRuntimeException.class, ((CreateFilesetFailureEvent) event).exception().getClass()); checkFilesetInfo(((CreateFilesetFailureEvent) event).createFilesetRequest(), fileset); @@ -148,9 +196,9 @@ void testLoadFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.loadFileset(identifier)); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(LoadFilesetFailureEvent.class, event.getClass()); - Assertions.assertEquals( + assertEquals(identifier, event.identifier()); + assertEquals(LoadFilesetFailureEvent.class, event.getClass()); + assertEquals( GravitinoRuntimeException.class, ((LoadFilesetFailureEvent) event).exception().getClass()); } @@ -161,12 +209,12 @@ void testAlterFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.alterFileset(identifier, change)); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(AlterFilesetFailureEvent.class, event.getClass()); - Assertions.assertEquals( + assertEquals(identifier, event.identifier()); + assertEquals(AlterFilesetFailureEvent.class, event.getClass()); + assertEquals( GravitinoRuntimeException.class, ((AlterFilesetFailureEvent) event).exception().getClass()); - Assertions.assertEquals(1, ((AlterFilesetFailureEvent) event).filesetChanges().length); - Assertions.assertEquals(change, ((AlterFilesetFailureEvent) event).filesetChanges()[0]); + assertEquals(1, ((AlterFilesetFailureEvent) event).filesetChanges().length); + assertEquals(change, ((AlterFilesetFailureEvent) event).filesetChanges()[0]); } @Test @@ -175,9 +223,9 @@ void testDropFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.dropFileset(identifier)); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(identifier, event.identifier()); - Assertions.assertEquals(DropFilesetFailureEvent.class, event.getClass()); - Assertions.assertEquals( + assertEquals(identifier, event.identifier()); + assertEquals(DropFilesetFailureEvent.class, event.getClass()); + assertEquals( GravitinoRuntimeException.class, ((DropFilesetFailureEvent) event).exception().getClass()); } @@ -187,19 +235,33 @@ void testListFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.listFilesets(namespace)); Event event = dummyEventListener.popEvent(); - Assertions.assertEquals(namespace.toString(), event.identifier().toString()); - Assertions.assertEquals(ListFilesetFailureEvent.class, event.getClass()); - Assertions.assertEquals( + assertEquals(namespace.toString(), event.identifier().toString()); + assertEquals(ListFilesetFailureEvent.class, event.getClass()); + assertEquals( GravitinoRuntimeException.class, ((ListFilesetFailureEvent) event).exception().getClass()); - Assertions.assertEquals(namespace, ((ListFilesetFailureEvent) event).namespace()); + assertEquals(namespace, ((ListFilesetFailureEvent) event).namespace()); + } + + @Test + void testGetFileLocationFailureEvent() { + NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", "fileset"); + Assertions.assertThrowsExactly( + GravitinoRuntimeException.class, + () -> failureDispatcher.getFileLocation(identifier, "/test")); + Event event = dummyEventListener.popEvent(); + assertEquals(identifier, event.identifier()); + assertEquals(GetFileLocationFailureEvent.class, event.getClass()); + assertEquals( + GravitinoRuntimeException.class, + ((GetFileLocationFailureEvent) event).exception().getClass()); } private void checkFilesetInfo(FilesetInfo filesetInfo, Fileset fileset) { - Assertions.assertEquals(fileset.name(), filesetInfo.name()); - Assertions.assertEquals(fileset.type(), filesetInfo.type()); - Assertions.assertEquals(fileset.storageLocation(), filesetInfo.storageLocation()); - Assertions.assertEquals(fileset.properties(), filesetInfo.properties()); - Assertions.assertEquals(fileset.comment(), filesetInfo.comment()); + assertEquals(fileset.name(), filesetInfo.name()); + assertEquals(fileset.type(), filesetInfo.type()); + assertEquals(fileset.storageLocation(), filesetInfo.storageLocation()); + assertEquals(fileset.properties(), filesetInfo.properties()); + assertEquals(fileset.comment(), filesetInfo.comment()); } private Fileset mockFileset() { @@ -227,6 +289,8 @@ private FilesetDispatcher mockFilesetDispatcher() { when(dispatcher.listFilesets(any(Namespace.class))).thenReturn(null); when(dispatcher.alterFileset(any(NameIdentifier.class), any(FilesetChange.class))) .thenReturn(fileset); + when(dispatcher.getFileLocation(any(NameIdentifier.class), any())) + .thenReturn("file:/test/xxx.parquet"); return dispatcher; } diff --git a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java index 8201980c88a..34933bf13e4 100644 --- a/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java +++ b/integration-test/src/test/java/org/apache/gravitino/integration/test/client/FilesetIT.java @@ -19,19 +19,18 @@ package org.apache.gravitino.integration.test.client; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.google.common.collect.Maps; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.client.GravitinoMetalake; -import org.apache.gravitino.file.BaseFilesetDataOperationCtx; -import org.apache.gravitino.file.ClientType; +import org.apache.gravitino.context.CallerContext; import org.apache.gravitino.file.Fileset; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperation; -import org.apache.gravitino.file.FilesetDataOperationCtx; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.util.AbstractIT; @@ -101,7 +100,7 @@ public void cleanUp() { } @Test - public void testGetFilesetContext() { + public void testGetFileLocation() { String catalogName = GravitinoITUtils.genRandomName("catalog"); Assertions.assertFalse(metalake.catalogExists(catalogName)); @@ -131,22 +130,39 @@ public void testGetFilesetContext() { Maps.newHashMap()); Assertions.assertTrue(catalog.asFilesetCatalog().filesetExists(filesetIdent)); - FilesetDataOperationCtx ctx = - BaseFilesetDataOperationCtx.builder() - .withSubPath("/test.par") - .withOperation(FilesetDataOperation.CREATE) - .withClientType(ClientType.HADOOP_GVFS) - .build(); - FilesetContext context = catalog.asFilesetCatalog().getFilesetContext(filesetIdent, ctx); - - Fileset actualFileset = context.fileset(); - Assertions.assertEquals(expectedFileset.name(), actualFileset.name()); - Assertions.assertEquals(expectedFileset.comment(), actualFileset.comment()); - Assertions.assertEquals(expectedFileset.type(), actualFileset.type()); - Assertions.assertEquals(expectedFileset.storageLocation(), actualFileset.storageLocation()); - Assertions.assertFalse(actualFileset.properties().containsKey(StringIdentifier.ID_KEY)); - - Assertions.assertEquals(expectedFileset.storageLocation() + "/test.par", context.actualPath()); + String actualFileLocation = + catalog.asFilesetCatalog().getFileLocation(filesetIdent, "/test.par"); + + Assertions.assertEquals(expectedFileset.storageLocation() + "/test.par", actualFileLocation); + } + + @Test + public void testGetFileLocationWithInvalidAuditHeaders() { + try { + String catalogName = GravitinoITUtils.genRandomName("catalog"); + String schemaName = GravitinoITUtils.genRandomName("schema"); + String filesetName = GravitinoITUtils.genRandomName("fileset"); + NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); + + Map properties = Maps.newHashMap(); + properties.put("metastore.uris", hmsUri); + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + + Map context = new HashMap<>(); + // this is a invalid internal client type. + context.put(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, "test"); + CallerContext callerContext = CallerContext.builder().withContext(context).build(); + CallerContext.CallerContextHolder.set(callerContext); + + assertThrows( + IllegalArgumentException.class, + () -> catalog.asFilesetCatalog().getFileLocation(filesetIdent, "/test.par")); + } finally { + CallerContext.CallerContextHolder.remove(); + } } private static String generateLocation( diff --git a/server/src/main/java/org/apache/gravitino/server/web/Utils.java b/server/src/main/java/org/apache/gravitino/server/web/Utils.java index e2405a6adff..12574412557 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/Utils.java +++ b/server/src/main/java/org/apache/gravitino/server/web/Utils.java @@ -18,14 +18,20 @@ */ package org.apache.gravitino.server.web; +import com.google.common.collect.Maps; import java.security.PrivilegedExceptionAction; +import java.util.Map; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.UserPrincipal; +import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.auth.AuthConstants; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.enums.FilesetDataOperation; +import org.apache.gravitino.enums.InternalClientType; import org.apache.gravitino.utils.PrincipalUtils; public class Utils { @@ -148,4 +154,25 @@ public static Response doAs( } return PrincipalUtils.doAs(principal, action); } + + public static Map filterFilesetAuditHeaders(HttpServletRequest httpRequest) { + Map filteredHeaders = Maps.newHashMap(); + + String internalClientType = + httpRequest.getHeader(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE); + if (StringUtils.isNotBlank(internalClientType) + && InternalClientType.checkValid(internalClientType)) { + filteredHeaders.put( + FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE, internalClientType); + } + + String dataOperation = + httpRequest.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION); + if (StringUtils.isNotBlank( + httpRequest.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)) + && FilesetDataOperation.checkValid(dataOperation)) { + filteredHeaders.put(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION, dataOperation); + } + return filteredHeaders; + } } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java index c26ab675008..470ab076804 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java @@ -20,6 +20,7 @@ import com.codahale.metrics.annotation.ResponseMetered; import com.codahale.metrics.annotation.Timed; +import java.util.Map; import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; @@ -30,24 +31,23 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.context.CallerContext; import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; -import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; -import org.apache.gravitino.dto.responses.FilesetContextResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.dto.util.DTOConverters; -import org.apache.gravitino.file.BaseFilesetDataOperationCtx; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; import org.apache.gravitino.lock.LockType; import org.apache.gravitino.lock.TreeLockUtils; import org.apache.gravitino.metrics.MetricNames; @@ -250,35 +250,40 @@ public Response dropFileset( } } - // we use POST type here, because we need to use request body to pass some parameters - @POST - @Path("{fileset}/context") + @GET + @Path("{fileset}/fileLocation") @Produces("application/vnd.gravitino.v1+json") - @Timed(name = "get-fileset-context." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) - @ResponseMetered(name = "get-fileset-context", absolute = true) - public Response getFilesetContext( + @Timed(name = "get-file-location." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-file-location", absolute = true) + public Response getFileLocation( @PathParam("metalake") String metalake, @PathParam("catalog") String catalog, @PathParam("schema") String schema, @PathParam("fileset") String fileset, - GetFilesetContextRequest request) { + @QueryParam("subPath") String subPath) { LOG.info( - "Received get fileset context request: {}.{}.{}.{}", metalake, catalog, schema, fileset); + "Received get file location request: {}.{}.{}.{}, sub path:{}", + metalake, + catalog, + schema, + fileset, + subPath); try { return Utils.doAs( httpRequest, () -> { NameIdentifier ident = NameIdentifierUtil.ofFileset(metalake, catalog, schema, fileset); - BaseFilesetDataOperationCtx ctx = - BaseFilesetDataOperationCtx.builder() - .withSubPath(request.getSubPath()) - .withOperation(request.getOperation()) - .withClientType(request.getClientType()) - .build(); - FilesetContext context = + Map filteredAuditHeaders = Utils.filterFilesetAuditHeaders(httpRequest); + // set the audit info into the thread local context + if (!filteredAuditHeaders.isEmpty()) { + CallerContext context = + CallerContext.builder().withContext(filteredAuditHeaders).build(); + CallerContext.CallerContextHolder.set(context); + } + String actualFileLocation = TreeLockUtils.doWithTreeLock( - ident, LockType.READ, () -> dispatcher.getFilesetContext(ident, ctx)); - return Utils.ok(new FilesetContextResponse(DTOConverters.toDTO(context))); + ident, LockType.READ, () -> dispatcher.getFileLocation(ident, subPath)); + return Utils.ok(new FileLocationResponse(actualFileLocation)); }); } catch (Exception e) { return ExceptionHandlers.handleFilesetException(OperationType.GET, fileset, schema, e); diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java index f4ccd69c653..8ee4ffd5964 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestFilesetOperations.java @@ -47,21 +47,17 @@ import org.apache.gravitino.dto.requests.FilesetCreateRequest; import org.apache.gravitino.dto.requests.FilesetUpdateRequest; import org.apache.gravitino.dto.requests.FilesetUpdatesRequest; -import org.apache.gravitino.dto.requests.GetFilesetContextRequest; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.EntityListResponse; import org.apache.gravitino.dto.responses.ErrorConstants; import org.apache.gravitino.dto.responses.ErrorResponse; -import org.apache.gravitino.dto.responses.FilesetContextResponse; +import org.apache.gravitino.dto.responses.FileLocationResponse; import org.apache.gravitino.dto.responses.FilesetResponse; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchSchemaException; -import org.apache.gravitino.file.ClientType; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.file.FilesetChange; -import org.apache.gravitino.file.FilesetContext; -import org.apache.gravitino.file.FilesetDataOperation; import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.rest.RESTUtils; import org.glassfish.jersey.internal.inject.AbstractBinder; @@ -438,26 +434,22 @@ public void testDropFileset() { } @Test - public void testGetFilesetContext() { - Fileset fileset = - mockFileset( - "fileset1", - Fileset.Type.MANAGED, - "mock comment", - "mock location", - ImmutableMap.of("k1", "v1")); + public void testGetFileLocation() { String actualPath = "mock location/path1"; - FilesetContext context = mockFilesetContext(fileset, actualPath); + when(dispatcher.getFileLocation(any(), any())).thenReturn(actualPath); + Response resp = + target(filesetPath(metalake, catalog, schema) + "fileset1/fileLocation") + .queryParam("subPath", "test/1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); - GetFilesetContextRequest req = - GetFilesetContextRequest.builder() - .subPath("path1") - .operation(FilesetDataOperation.OPEN) - .clientType(ClientType.HADOOP_GVFS) - .build(); + FileLocationResponse contextResponse = resp.readEntity(FileLocationResponse.class); + Assertions.assertEquals(0, contextResponse.getCode()); - assertGetFilesetContext(req, context); + Assertions.assertEquals(actualPath, contextResponse.getFileLocation()); } private void assertUpdateFileset(FilesetUpdatesRequest req, Fileset updatedFileset) { @@ -480,27 +472,6 @@ private void assertUpdateFileset(FilesetUpdatesRequest req, Fileset updatedFiles Assertions.assertEquals(updatedFileset.properties(), filesetDTO.properties()); } - private void assertGetFilesetContext(GetFilesetContextRequest req, FilesetContext context) { - when(dispatcher.getFilesetContext(any(), any())).thenReturn(context); - Response resp = - target(filesetPath(metalake, catalog, schema) + "fileset1/context") - .request(MediaType.APPLICATION_JSON_TYPE) - .accept("application/vnd.gravitino.v1+json") - .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); - Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); - - FilesetContextResponse contextResponse = resp.readEntity(FilesetContextResponse.class); - Assertions.assertEquals(0, contextResponse.getCode()); - - FilesetDTO filesetDTO = contextResponse.getContext().fileset(); - Assertions.assertEquals(context.fileset().name(), filesetDTO.name()); - Assertions.assertEquals(context.fileset().comment(), filesetDTO.comment()); - Assertions.assertEquals(context.fileset().type(), filesetDTO.type()); - Assertions.assertEquals(context.fileset().properties(), filesetDTO.properties()); - - Assertions.assertEquals(context.actualPath(), contextResponse.getContext().actualPath()); - } - private static String filesetPath(String metalake, String catalog, String schema) { return new StringBuilder() .append("/metalakes/") @@ -533,11 +504,4 @@ private static Fileset mockFileset( return mockFileset; } - - private static FilesetContext mockFilesetContext(Fileset fileset, String actualPath) { - FilesetContext context = mock(FilesetContext.class); - when(context.fileset()).thenReturn(fileset); - when(context.actualPath()).thenReturn(actualPath); - return context; - } } From 23c2609589314a0472cc05ff452e233d16e7c78b Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Fri, 9 Aug 2024 17:51:25 +0800 Subject: [PATCH 08/13] fix docs --- .../hadoop/HadoopCatalogOperations.java | 13 ++- .../audit/FilesetAuditConstants.java | 1 + .../gravitino/context/CallerContext.java | 2 +- .../gravitino/context/TestCallerContext.java | 20 ++-- .../gravitino/hook/FilesetHookDispatcher.java | 6 ++ .../listener/api/event/TestFilesetEvent.java | 92 +++++++++---------- docs/open-api/filesets.yaml | 87 +++++------------- .../server/web/rest/FilesetOperations.java | 3 +- .../gravitino/server/web/TestUtils.java | 38 ++++++++ 9 files changed, 140 insertions(+), 122 deletions(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index b2612e2af61..2ffe2cffb25 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -69,6 +69,7 @@ public class HadoopCatalogOperations implements CatalogOperations, SupportsSchem private static final String SCHEMA_DOES_NOT_EXIST_MSG = "Schema %s does not exist"; private static final String FILESET_DOES_NOT_EXIST_MSG = "Fileset %s does not exist"; + private static final String SLASH = "/"; private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogOperations.class); @@ -359,19 +360,25 @@ public String getFileLocation(NameIdentifier ident, String subPath) throws NoSuchFilesetException { // TODO we need move some check logics in the Hadoop / Python GVFS to here. Preconditions.checkArgument(subPath != null, "subPath must not be null"); + String processedSubPath; + if (!subPath.trim().isEmpty() && !subPath.trim().startsWith(SLASH)) { + processedSubPath = SLASH + subPath.trim(); + } else { + processedSubPath = subPath.trim(); + } Fileset fileset = loadFileset(ident); String storageLocation = fileset.storageLocation(); String fileLocation; // subPath cannot be null, so we only need check if it is blank - if (StringUtils.isBlank(subPath)) { + if (StringUtils.isBlank(processedSubPath)) { fileLocation = storageLocation; } else { fileLocation = subPath.startsWith("/") - ? String.format("%s%s", storageLocation, subPath) - : String.format("%s/%s", storageLocation, subPath); + ? String.format("%s%s", storageLocation, processedSubPath) + : String.format("%s/%s", storageLocation, processedSubPath); } return fileLocation; } diff --git a/common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java b/common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java index 01184405df0..188f7509468 100644 --- a/common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java +++ b/common/src/main/java/org/apache/gravitino/audit/FilesetAuditConstants.java @@ -19,6 +19,7 @@ package org.apache.gravitino.audit; +/** Constants used for fileset data operation audits. */ public class FilesetAuditConstants { private FilesetAuditConstants() {} diff --git a/common/src/main/java/org/apache/gravitino/context/CallerContext.java b/common/src/main/java/org/apache/gravitino/context/CallerContext.java index 85c0dfbe016..3866a9194fb 100644 --- a/common/src/main/java/org/apache/gravitino/context/CallerContext.java +++ b/common/src/main/java/org/apache/gravitino/context/CallerContext.java @@ -28,7 +28,7 @@ * *

Reference: * - *

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/\ ipc/CallerContext.java + *

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java */ public class CallerContext { private Map context; diff --git a/common/src/test/java/org/apache/gravitino/context/TestCallerContext.java b/common/src/test/java/org/apache/gravitino/context/TestCallerContext.java index 5d281069a00..353e90e9125 100644 --- a/common/src/test/java/org/apache/gravitino/context/TestCallerContext.java +++ b/common/src/test/java/org/apache/gravitino/context/TestCallerContext.java @@ -27,15 +27,19 @@ public class TestCallerContext { @Test public void testCallerContext() { - Map contextMap = new HashMap<>(); - contextMap.put("test", "test"); - contextMap.put("test2", "test2"); - CallerContext callerContext = CallerContext.builder().withContext(contextMap).build(); - CallerContext.CallerContextHolder.set(callerContext); + try { + Map contextMap = new HashMap<>(); + contextMap.put("test", "test"); + contextMap.put("test2", "test2"); + CallerContext callerContext = CallerContext.builder().withContext(contextMap).build(); + CallerContext.CallerContextHolder.set(callerContext); - CallerContext actualCallerContext = CallerContext.CallerContextHolder.get(); + CallerContext actualCallerContext = CallerContext.CallerContextHolder.get(); - Assertions.assertEquals(callerContext, actualCallerContext); - Assertions.assertEquals(callerContext.context(), actualCallerContext.context()); + Assertions.assertEquals(callerContext, actualCallerContext); + Assertions.assertEquals(callerContext.context(), actualCallerContext.context()); + } finally { + CallerContext.CallerContextHolder.remove(); + } } } diff --git a/core/src/main/java/org/apache/gravitino/hook/FilesetHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/FilesetHookDispatcher.java index 6e5a82eb26c..e7780135588 100644 --- a/core/src/main/java/org/apache/gravitino/hook/FilesetHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/FilesetHookDispatcher.java @@ -93,4 +93,10 @@ public boolean dropFileset(NameIdentifier ident) { public boolean filesetExists(NameIdentifier ident) { return dispatcher.filesetExists(ident); } + + @Override + public String getFileLocation(NameIdentifier ident, String subPath) + throws NoSuchFilesetException { + return dispatcher.getFileLocation(ident, subPath); + } } diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java index fb23b95ef3a..fd181404a9c 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java @@ -76,8 +76,8 @@ void testCreateFilesetEvent() { fileset.storageLocation(), fileset.properties()); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(CreateFilesetEvent.class, event.getClass()); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((CreateFilesetEvent) event).createdFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); } @@ -87,8 +87,8 @@ void testLoadFilesetEvent() { NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); dispatcher.loadFileset(identifier); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(LoadFilesetEvent.class, event.getClass()); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((LoadFilesetEvent) event).loadedFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); } @@ -99,12 +99,12 @@ void testAlterFilesetEvent() { FilesetChange change = FilesetChange.setProperty("a", "b"); dispatcher.alterFileset(identifier, change); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(AlterFilesetEvent.class, event.getClass()); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((AlterFilesetEvent) event).updatedFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); - assertEquals(1, ((AlterFilesetEvent) event).filesetChanges().length); - assertEquals(change, ((AlterFilesetEvent) event).filesetChanges()[0]); + Assertions.assertEquals(1, ((AlterFilesetEvent) event).filesetChanges().length); + Assertions.assertEquals(change, ((AlterFilesetEvent) event).filesetChanges()[0]); } @Test @@ -112,8 +112,8 @@ void testDropFilesetEvent() { NameIdentifier identifier = NameIdentifier.of("metalake", "catalog", fileset.name()); dispatcher.dropFileset(identifier); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(DropFilesetEvent.class, event.getClass()); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropFilesetEvent.class, event.getClass()); Assertions.assertTrue(((DropFilesetEvent) event).isExists()); } @@ -122,9 +122,9 @@ void testListFilesetEvent() { Namespace namespace = Namespace.of("metalake", "catalog"); dispatcher.listFilesets(namespace); Event event = dummyEventListener.popEvent(); - assertEquals(namespace.toString(), event.identifier().toString()); - assertEquals(ListFilesetEvent.class, event.getClass()); - assertEquals(namespace, ((ListFilesetEvent) event).namespace()); + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListFilesetEvent.class, event.getClass()); + Assertions.assertEquals(namespace, ((ListFilesetEvent) event).namespace()); } @Test @@ -137,8 +137,8 @@ void testGetFileLocationEvent() { fileset.storageLocation(), fileset.properties()); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(CreateFilesetEvent.class, event.getClass()); + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateFilesetEvent.class, event.getClass()); FilesetInfo filesetInfo = ((CreateFilesetEvent) event).createdFilesetInfo(); checkFilesetInfo(filesetInfo, fileset); @@ -153,16 +153,16 @@ void testGetFileLocationEvent() { CallerContext.CallerContextHolder.set(callerContext); String fileLocation = dispatcher.getFileLocation(identifier, "test"); Event event1 = dummyEventListener.popEvent(); - assertEquals(identifier, event1.identifier()); - assertEquals(GetFileLocationEvent.class, event1.getClass()); + Assertions.assertEquals(identifier, event1.identifier()); + Assertions.assertEquals(GetFileLocationEvent.class, event1.getClass()); String actualFileLocation = ((GetFileLocationEvent) event1).actualFileLocation(); - assertEquals(actualFileLocation, fileLocation); + Assertions.assertEquals(actualFileLocation, fileLocation); CallerContext actualCallerContext = ((GetFileLocationEvent) event1).callerContext(); assertEquals(2, actualCallerContext.context().size()); - assertEquals( + Assertions.assertEquals( InternalClientType.HADOOP_GVFS.name(), actualCallerContext.context().get(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); - assertEquals( + Assertions.assertEquals( FilesetDataOperation.GET_FILE_STATUS.name(), actualCallerContext .context() @@ -182,9 +182,9 @@ void testCreateSchemaFailureEvent() { fileset.storageLocation(), fileset.properties())); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(CreateFilesetFailureEvent.class, event.getClass()); - assertEquals( + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(CreateFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( GravitinoRuntimeException.class, ((CreateFilesetFailureEvent) event).exception().getClass()); checkFilesetInfo(((CreateFilesetFailureEvent) event).createFilesetRequest(), fileset); @@ -196,9 +196,9 @@ void testLoadFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.loadFileset(identifier)); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(LoadFilesetFailureEvent.class, event.getClass()); - assertEquals( + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(LoadFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( GravitinoRuntimeException.class, ((LoadFilesetFailureEvent) event).exception().getClass()); } @@ -209,12 +209,12 @@ void testAlterFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.alterFileset(identifier, change)); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(AlterFilesetFailureEvent.class, event.getClass()); - assertEquals( + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(AlterFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( GravitinoRuntimeException.class, ((AlterFilesetFailureEvent) event).exception().getClass()); - assertEquals(1, ((AlterFilesetFailureEvent) event).filesetChanges().length); - assertEquals(change, ((AlterFilesetFailureEvent) event).filesetChanges()[0]); + Assertions.assertEquals(1, ((AlterFilesetFailureEvent) event).filesetChanges().length); + Assertions.assertEquals(change, ((AlterFilesetFailureEvent) event).filesetChanges()[0]); } @Test @@ -223,9 +223,9 @@ void testDropFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.dropFileset(identifier)); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(DropFilesetFailureEvent.class, event.getClass()); - assertEquals( + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(DropFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( GravitinoRuntimeException.class, ((DropFilesetFailureEvent) event).exception().getClass()); } @@ -235,11 +235,11 @@ void testListFilesetFailureEvent() { Assertions.assertThrowsExactly( GravitinoRuntimeException.class, () -> failureDispatcher.listFilesets(namespace)); Event event = dummyEventListener.popEvent(); - assertEquals(namespace.toString(), event.identifier().toString()); - assertEquals(ListFilesetFailureEvent.class, event.getClass()); - assertEquals( + Assertions.assertEquals(namespace.toString(), event.identifier().toString()); + Assertions.assertEquals(ListFilesetFailureEvent.class, event.getClass()); + Assertions.assertEquals( GravitinoRuntimeException.class, ((ListFilesetFailureEvent) event).exception().getClass()); - assertEquals(namespace, ((ListFilesetFailureEvent) event).namespace()); + Assertions.assertEquals(namespace, ((ListFilesetFailureEvent) event).namespace()); } @Test @@ -249,19 +249,19 @@ void testGetFileLocationFailureEvent() { GravitinoRuntimeException.class, () -> failureDispatcher.getFileLocation(identifier, "/test")); Event event = dummyEventListener.popEvent(); - assertEquals(identifier, event.identifier()); - assertEquals(GetFileLocationFailureEvent.class, event.getClass()); - assertEquals( + Assertions.assertEquals(identifier, event.identifier()); + Assertions.assertEquals(GetFileLocationFailureEvent.class, event.getClass()); + Assertions.assertEquals( GravitinoRuntimeException.class, ((GetFileLocationFailureEvent) event).exception().getClass()); } private void checkFilesetInfo(FilesetInfo filesetInfo, Fileset fileset) { - assertEquals(fileset.name(), filesetInfo.name()); - assertEquals(fileset.type(), filesetInfo.type()); - assertEquals(fileset.storageLocation(), filesetInfo.storageLocation()); - assertEquals(fileset.properties(), filesetInfo.properties()); - assertEquals(fileset.comment(), filesetInfo.comment()); + Assertions.assertEquals(fileset.name(), filesetInfo.name()); + Assertions.assertEquals(fileset.type(), filesetInfo.type()); + Assertions.assertEquals(fileset.storageLocation(), filesetInfo.storageLocation()); + Assertions.assertEquals(fileset.properties(), filesetInfo.properties()); + Assertions.assertEquals(fileset.comment(), filesetInfo.comment()); } private Fileset mockFileset() { diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 0511a306312..7b8cd8ec8b4 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -160,32 +160,31 @@ paths: "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" - /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}/context: + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/filesets/{fileset}/fileLocation: parameters: - $ref: "./openapi.yaml#/components/parameters/metalake" - $ref: "./openapi.yaml#/components/parameters/catalog" - $ref: "./openapi.yaml#/components/parameters/schema" - $ref: "./openapi.yaml#/components/parameters/fileset" - post: + get: tags: - - filesetContext - summary: Get fileset context - operationId: getFilesetContext - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/GetFilesetContextRequest" - examples: - GetFilesetContextRequest: - $ref: "#/components/examples/GetFilesetContextRequest" - description: Returns the specified fileset context object + - fileLocation + summary: Get file location + operationId: getFileLocation + description: Returns the specified file location object + parameters: + - name: subPath + in: query + required: true + schema: + type: string + description: The sub path to the file or directory responses: "200": - $ref: "#/components/responses/FilesetContextResponse" + $ref: "#/components/responses/FileLocationResponse" "404": - description: Not Found - The target fileset context does not exist + description: Not Found - The target fileset does not exist content: application/vnd.gravitino.v1+json: schema: @@ -232,17 +231,6 @@ components: default: {} additionalProperties: type: string - FilesetContext: - type: object - required: - - fileset - - actualPath - properties: - fileset: - $ref: "#/components/schemas/Fileset" - actualPath: - type: string - description: The actual paths which data operation accessed FilesetCreateRequest: type: object required: @@ -391,23 +379,6 @@ components: "@type": "removeComment" } - GetFilesetContextRequest: - type: object - required: - - subPath - - operation - - clientType - properties: - subPath: - type: string - description: The sub path that the data operation to access. Can not be null. - operation: - type: string - description: The data operation to get the fileset context. Can not be empty. - clientType: - type: string - description: The IP of the machine that initiated the data operation. Can not be empty. - responses: FilesetResponse: description: The response of fileset object @@ -428,8 +399,8 @@ components: FilesetResponse: $ref: "#/components/examples/FilesetResponse" - FilesetContextResponse: - description: The response of fileset context object + FileLocationResponse: + description: The response of the file location object content: application/vnd.gravitino.v1+json: schema: @@ -441,11 +412,12 @@ components: description: Status code of the response enum: - 0 - context: - $ref: "#/components/schemas/FilesetContext" + fileLocation: + type: string + description: The actual file location examples: - FilesetContextResponse: - $ref: "#/components/examples/FilesetContextResponse" + FileLocationResponse: + $ref: "#/components/examples/FileLocationResponse" examples: FilesetCreateRequest: @@ -475,21 +447,10 @@ components: } } - FilesetContextResponse: + FileLocationResponse: value: { "code": 0, - "context": { - "fileset": { - "name": "fileset1", - "type": "managed", - "comment": "This is a comment", - "storageLocation": "hdfs://host/user/s_fileset/schema/fileset1", - "properties": { - "key1": "value1", - "key2": "value2" - } - }, - "actualPath": "hdfs://host/user/fileset/schema/fileset1/test.parquet" + "fileLocation": "hdfs://host/user/fileset/schema/fileset1/test.parquet" } } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java index 470ab076804..ff58ff32ef4 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/FilesetOperations.java @@ -24,6 +24,7 @@ import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; +import javax.validation.constraints.NotNull; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -260,7 +261,7 @@ public Response getFileLocation( @PathParam("catalog") String catalog, @PathParam("schema") String schema, @PathParam("fileset") String fileset, - @QueryParam("subPath") String subPath) { + @QueryParam("subPath") @NotNull String subPath) { LOG.info( "Received get file location request: {}.{}.{}.{}, sub path:{}", metalake, diff --git a/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java b/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java index adaaacaed09..5b339b12e33 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java +++ b/server/src/test/java/org/apache/gravitino/server/web/TestUtils.java @@ -23,11 +23,17 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.gravitino.audit.FilesetAuditConstants; import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.enums.FilesetDataOperation; +import org.apache.gravitino.enums.InternalClientType; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -167,4 +173,36 @@ public void testUnsupportedOperation() { ErrorResponse errorResponse = (ErrorResponse) response.getEntity(); assertEquals("Unsupported operation", errorResponse.getMessage()); } + + @Test + public void testFilterFilesetAuditHeaders() { + // test invalid internal client type + HttpServletRequest mockRequest1 = Mockito.mock(HttpServletRequest.class); + when(mockRequest1.getHeader(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)) + .thenReturn("test"); + Assertions.assertThrows( + IllegalArgumentException.class, () -> Utils.filterFilesetAuditHeaders(mockRequest1)); + + // test invalid fileset data operation + HttpServletRequest mockRequest2 = Mockito.mock(HttpServletRequest.class); + when(mockRequest2.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)) + .thenReturn("test"); + Assertions.assertThrows( + IllegalArgumentException.class, () -> Utils.filterFilesetAuditHeaders(mockRequest2)); + + // test normal audit headers + HttpServletRequest mockRequest3 = Mockito.mock(HttpServletRequest.class); + when(mockRequest3.getHeader(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)) + .thenReturn(InternalClientType.HADOOP_GVFS.name()); + when(mockRequest3.getHeader(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)) + .thenReturn(FilesetDataOperation.GET_FILE_STATUS.name()); + Map filteredMap = Utils.filterFilesetAuditHeaders(mockRequest3); + Assertions.assertEquals(2, filteredMap.size()); + Assertions.assertEquals( + InternalClientType.HADOOP_GVFS.name(), + filteredMap.get(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); + Assertions.assertEquals( + FilesetDataOperation.GET_FILE_STATUS.name(), + filteredMap.get(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); + } } From 9949b3fbb2c38f775c4aa5e1f69bfa153dc46463 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Mon, 12 Aug 2024 20:14:48 +0800 Subject: [PATCH 09/13] fix doc checks --- .../gravitino/context/CallerContext.java | 24 +++++++++++++++++++ .../gravitino/enums/FilesetDataOperation.java | 6 +++++ .../gravitino/enums/InternalClientType.java | 6 +++++ docs/open-api/filesets.yaml | 1 - 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/gravitino/context/CallerContext.java b/common/src/main/java/org/apache/gravitino/context/CallerContext.java index 3866a9194fb..55778c452af 100644 --- a/common/src/main/java/org/apache/gravitino/context/CallerContext.java +++ b/common/src/main/java/org/apache/gravitino/context/CallerContext.java @@ -35,6 +35,11 @@ public class CallerContext { private CallerContext() {} + /** + * Returns the context map in the caller context. + * + * @return the context map + */ public Map context() { return context; } @@ -52,6 +57,7 @@ public int hashCode() { return Objects.hashCode(context); } + /** Builder to create a caller context. */ public static class Builder { private final CallerContext callerContext; @@ -70,10 +76,16 @@ public CallerContext.Builder withContext(Map context) { return this; } + /** Validate the variables in the CallerContext. */ private void validate() { Preconditions.checkArgument(callerContext.context != null, "context cannot be null"); } + /** + * Build the CallerContext. + * + * @return The CallerContext. + */ public CallerContext build() { validate(); return callerContext; @@ -89,18 +101,30 @@ public static CallerContext.Builder builder() { return new CallerContext.Builder(); } + /** A thread local holder for the CallerContext. */ public static class CallerContextHolder { private static final ThreadLocal CALLER_CONTEXT = new ThreadLocal<>(); + /** + * Get the CallerContext from the thread local. + * + * @return The CallerContext. + */ public static CallerContext get() { return CALLER_CONTEXT.get(); } + /** + * Set the CallerContext in the thread local. + * + * @param context The CallerContext to set. + */ public static void set(CallerContext context) { CALLER_CONTEXT.set(context); } + /** Remove the CallerContext from the thread local. */ public static void remove() { CALLER_CONTEXT.remove(); } diff --git a/common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java b/common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java index 85e3f7d768f..021ed3db6b1 100644 --- a/common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java +++ b/common/src/main/java/org/apache/gravitino/enums/FilesetDataOperation.java @@ -46,6 +46,12 @@ public enum FilesetDataOperation { /** This data operation means that it is an unknown data operation. */ UNKNOWN; + /** + * Check if the operation is valid. + * + * @param operation the operation to check + * @return true if the operation is valid, false otherwise + */ public static boolean checkValid(String operation) { try { FilesetDataOperation.valueOf(operation); diff --git a/common/src/main/java/org/apache/gravitino/enums/InternalClientType.java b/common/src/main/java/org/apache/gravitino/enums/InternalClientType.java index 31e00ae1f80..ddc21806738 100644 --- a/common/src/main/java/org/apache/gravitino/enums/InternalClientType.java +++ b/common/src/main/java/org/apache/gravitino/enums/InternalClientType.java @@ -34,6 +34,12 @@ public enum InternalClientType { /** The client type is unknown. */ UNKNOWN; + /** + * Check if the client type is valid. + * + * @param clientType the client type + * @return true if the client type is valid, false otherwise + */ public static boolean checkValid(String clientType) { try { InternalClientType.valueOf(clientType); diff --git a/docs/open-api/filesets.yaml b/docs/open-api/filesets.yaml index 7b8cd8ec8b4..598b55557c8 100644 --- a/docs/open-api/filesets.yaml +++ b/docs/open-api/filesets.yaml @@ -451,7 +451,6 @@ components: value: { "code": 0, "fileLocation": "hdfs://host/user/fileset/schema/fileset1/test.parquet" - } } FilesetAlreadyExistsException: From e99f7e0b910cb0aa03329f83b032e19f22b13763 Mon Sep 17 00:00:00 2001 From: xiaojiebao Date: Thu, 22 Aug 2024 11:34:25 +0800 Subject: [PATCH 10/13] fix --- .../gravitino/catalog/hadoop/HadoopCatalogOperations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 2ffe2cffb25..6008b04f33c 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -376,7 +376,7 @@ public String getFileLocation(NameIdentifier ident, String subPath) fileLocation = storageLocation; } else { fileLocation = - subPath.startsWith("/") + processedSubPath.startsWith("/") ? String.format("%s%s", storageLocation, processedSubPath) : String.format("%s/%s", storageLocation, processedSubPath); } From 2a8a6fd61a6d14ccfaaeb1d68481753f4c2ae9ce Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Tue, 10 Sep 2024 09:46:26 +0800 Subject: [PATCH 11/13] fix comments --- .../listener/FilesetEventDispatcher.java | 7 +++++-- .../listener/api/event/GetFileLocationEvent.java | 16 ++++++++-------- .../listener/api/event/TestFilesetEvent.java | 10 ++++------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java index f841ead5f7c..2df5fa75a38 100644 --- a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java @@ -19,6 +19,7 @@ package org.apache.gravitino.listener; +import com.google.common.collect.ImmutableMap; import java.util.Map; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; @@ -148,10 +149,12 @@ public String getFileLocation(NameIdentifier ident, String subPath) try { String actualFileLocation = dispatcher.getFileLocation(ident, subPath); // get the audit info from the thread local context - CallerContext context = CallerContext.CallerContextHolder.get(); + CallerContext callerContext = CallerContext.CallerContextHolder.get(); + ImmutableMap.Builder builder = ImmutableMap.builder(); + builder.putAll(callerContext.context()); eventBus.dispatchEvent( new GetFileLocationEvent( - PrincipalUtils.getCurrentUserName(), ident, actualFileLocation, context)); + PrincipalUtils.getCurrentUserName(), ident, actualFileLocation, builder.build())); return actualFileLocation; } catch (Exception e) { eventBus.dispatchEvent( diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java index a1d13939bdb..0b4f05ee996 100644 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java @@ -19,40 +19,40 @@ package org.apache.gravitino.listener.api.event; +import java.util.Map; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.annotation.DeveloperApi; -import org.apache.gravitino.audit.CallerContext; -/** Represents an event that occurs when getting a actual file location. */ +/** Represents an event that occurs when getting an actual file location. */ @DeveloperApi public final class GetFileLocationEvent extends FilesetEvent { private final String actualFileLocation; - private final CallerContext callerContext; + private final Map context; /** * Constructs a new {@code GetFileLocationEvent}, recording the attempt to get a file location. * * @param user The user who initiated the get file location. * @param identifier The identifier of the file location that was attempted to be got. - * @param callerContext The audit caller context, this param can be null. + * @param context The audit context, this param can be null. * @param actualFileLocation The actual file location which want to get. */ public GetFileLocationEvent( String user, NameIdentifier identifier, String actualFileLocation, - CallerContext callerContext) { + Map context) { super(user, identifier); this.actualFileLocation = actualFileLocation; - this.callerContext = callerContext; + this.context = context; } public String actualFileLocation() { return actualFileLocation; } - public CallerContext callerContext() { - return callerContext; + public Map context() { + return context; } } diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java index fb4fc723c56..b94ad35f8e1 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java @@ -157,16 +157,14 @@ void testGetFileLocationEvent() { Assertions.assertEquals(GetFileLocationEvent.class, event1.getClass()); String actualFileLocation = ((GetFileLocationEvent) event1).actualFileLocation(); Assertions.assertEquals(actualFileLocation, fileLocation); - CallerContext actualCallerContext = ((GetFileLocationEvent) event1).callerContext(); - assertEquals(2, actualCallerContext.context().size()); + Map actualContext = ((GetFileLocationEvent) event1).context(); + assertEquals(2, actualContext.size()); Assertions.assertEquals( InternalClientType.HADOOP_GVFS.name(), - actualCallerContext.context().get(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); + actualContext.get(FilesetAuditConstants.HTTP_HEADER_INTERNAL_CLIENT_TYPE)); Assertions.assertEquals( FilesetDataOperation.GET_FILE_STATUS.name(), - actualCallerContext - .context() - .get(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); + actualContext.get(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); } @Test From aff70417e6d98698fd8666fba514ed850b97613d Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Wed, 11 Sep 2024 09:54:28 +0800 Subject: [PATCH 12/13] fix docs --- .../listener/api/event/GetFileLocationEvent.java | 10 ++++++++++ .../api/event/GetFileLocationFailureEvent.java | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java index 0b4f05ee996..bca2c1672d9 100644 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java @@ -48,10 +48,20 @@ public GetFileLocationEvent( this.context = context; } + /** + * Get the actual file location after processing of the get file location operation. + * + * @return The actual file location. + */ public String actualFileLocation() { return actualFileLocation; } + /** + * Get the audit context map of the get file location operation. + * + * @return The audit context map. + */ public Map context() { return context; } diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java index a79b88ba858..0826f08156a 100644 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationFailureEvent.java @@ -29,6 +29,7 @@ @DeveloperApi public final class GetFileLocationFailureEvent extends FilesetFailureEvent { private final String subPath; + /** * Constructs a new {@code GetFileLocationFailureEvent}. * @@ -45,6 +46,11 @@ public GetFileLocationFailureEvent( this.subPath = subPath; } + /** + * Get the audit context map of the get file location operation. + * + * @return The audit context map. + */ public String subPath() { return subPath; } From 105aab20dab368cf8c16efffc2ed8290a1a25da3 Mon Sep 17 00:00:00 2001 From: xloya <982052490@qq.com> Date: Wed, 11 Sep 2024 10:30:43 +0800 Subject: [PATCH 13/13] add sub path in the event --- .../listener/FilesetEventDispatcher.java | 6 +++++- .../listener/api/event/GetFileLocationEvent.java | 16 ++++++++++++++-- .../listener/api/event/TestFilesetEvent.java | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java index 2df5fa75a38..fd8e6c370e6 100644 --- a/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/FilesetEventDispatcher.java @@ -154,7 +154,11 @@ public String getFileLocation(NameIdentifier ident, String subPath) builder.putAll(callerContext.context()); eventBus.dispatchEvent( new GetFileLocationEvent( - PrincipalUtils.getCurrentUserName(), ident, actualFileLocation, builder.build())); + PrincipalUtils.getCurrentUserName(), + ident, + actualFileLocation, + subPath, + builder.build())); return actualFileLocation; } catch (Exception e) { eventBus.dispatchEvent( diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java index bca2c1672d9..8995f7db746 100644 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/GetFileLocationEvent.java @@ -27,7 +27,7 @@ @DeveloperApi public final class GetFileLocationEvent extends FilesetEvent { private final String actualFileLocation; - + private final String subPath; private final Map context; /** @@ -35,16 +35,19 @@ public final class GetFileLocationEvent extends FilesetEvent { * * @param user The user who initiated the get file location. * @param identifier The identifier of the file location that was attempted to be got. - * @param context The audit context, this param can be null. * @param actualFileLocation The actual file location which want to get. + * @param subPath The accessing sub path of the get file location operation. + * @param context The audit context, this param can be null. */ public GetFileLocationEvent( String user, NameIdentifier identifier, String actualFileLocation, + String subPath, Map context) { super(user, identifier); this.actualFileLocation = actualFileLocation; + this.subPath = subPath; this.context = context; } @@ -57,6 +60,15 @@ public String actualFileLocation() { return actualFileLocation; } + /** + * Get the accessing sub path of the get file location operation. + * + * @return The accessing sub path. + */ + public String subPath() { + return subPath; + } + /** * Get the audit context map of the get file location operation. * diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java index b94ad35f8e1..efc073b1979 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestFilesetEvent.java @@ -165,6 +165,7 @@ void testGetFileLocationEvent() { Assertions.assertEquals( FilesetDataOperation.GET_FILE_STATUS.name(), actualContext.get(FilesetAuditConstants.HTTP_HEADER_FILESET_DATA_OPERATION)); + Assertions.assertEquals("test", ((GetFileLocationEvent) event1).subPath()); } @Test