diff --git a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java index da405ff7b39..15d9fcb4308 100644 --- a/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -105,6 +105,11 @@ public void initialize(Map config) throws RuntimeException { @Override public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { try { + NameIdentifier schemaIdent = NameIdentifier.of(namespace.levels()); + if (!store.exists(schemaIdent, Entity.EntityType.SCHEMA)) { + throw new NoSuchSchemaException("Schema " + schemaIdent + " does not exist"); + } + List filesets = store.list(namespace, FilesetEntity.class, Entity.EntityType.FILESET); return filesets.stream() diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java index 3a293d36309..3510d19e298 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java @@ -13,16 +13,23 @@ import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.datastrato.gravitino.StringIdentifier; +import com.datastrato.gravitino.catalog.file.EntityCombinedFileset; import com.datastrato.gravitino.catalog.rel.EntityCombinedSchema; import com.datastrato.gravitino.catalog.rel.EntityCombinedTable; +import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.exceptions.NoSuchFilesetException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; +import com.datastrato.gravitino.exceptions.NonEmptyEntityException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetCatalog; +import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.SchemaEntity; import com.datastrato.gravitino.meta.TableEntity; @@ -33,8 +40,6 @@ import com.datastrato.gravitino.rel.Table; import com.datastrato.gravitino.rel.TableCatalog; import com.datastrato.gravitino.rel.TableChange; -import com.datastrato.gravitino.rel.TableChange.RemoveProperty; -import com.datastrato.gravitino.rel.TableChange.SetProperty; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; @@ -50,10 +55,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +66,7 @@ * A catalog operation dispatcher that dispatches the catalog operations to the underlying catalog * implementation. */ -public class CatalogOperationDispatcher implements TableCatalog, SupportsSchemas { +public class CatalogOperationDispatcher implements TableCatalog, FilesetCatalog, SupportsSchemas { private static final Logger LOG = LoggerFactory.getLogger(CatalogOperationDispatcher.class); @@ -141,11 +146,14 @@ public Schema createSchema(NameIdentifier ident, String comment, Map getHiddenTablePropertyNames( - NameIdentifier catalogIdent, Map properties) { - return doWithCatalog( - catalogIdent, - c -> - c.doWithPropertiesMeta( - p -> { - PropertiesMetadata propertiesMetadata = p.tablePropertiesMetadata(); - return properties.keySet().stream() - .filter(propertiesMetadata::isHiddenProperty) - .collect(Collectors.toSet()); - }), - IllegalArgumentException.class); - } - - private Set getHiddenSchemaPropertyNames( - NameIdentifier catalogIdent, Map properties) { - return doWithCatalog( - catalogIdent, - c -> - c.doWithPropertiesMeta( - p -> { - PropertiesMetadata propertiesMetadata = p.schemaPropertiesMetadata(); - return properties.keySet().stream() - .filter(propertiesMetadata::isHiddenProperty) - .collect(Collectors.toSet()); - }), - IllegalArgumentException.class); + getHiddenPropertyNames( + catalogIdentifier, + HasPropertyMetadata::tablePropertiesMetadata, + table.properties())); } /** @@ -492,11 +500,15 @@ public Table createTable( } catch (Exception e) { LOG.error(FormattedErrorMessages.STORE_OP_FAILURE, "put", ident, e); return EntityCombinedTable.of(table) - .withHiddenPropertiesSet(getHiddenTablePropertyNames(catalogIdent, table.properties())); + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, HasPropertyMetadata::tablePropertiesMetadata, table.properties())); } return EntityCombinedTable.of(table, tableEntity) - .withHiddenPropertiesSet(getHiddenTablePropertyNames(catalogIdent, table.properties())); + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, HasPropertyMetadata::tablePropertiesMetadata, table.properties())); } /** @@ -512,7 +524,7 @@ public Table createTable( @Override public Table alterTable(NameIdentifier ident, TableChange... changes) throws NoSuchTableException, IllegalArgumentException { - validateAlterTableProperties(ident, changes); + validateAlterProperties(ident, HasPropertyMetadata::tablePropertiesMetadata, changes); NameIdentifier catalogIdent = getCatalogIdentifier(ident); Table tempAlteredTable = @@ -537,7 +549,10 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) if (stringId == null) { return EntityCombinedTable.of(alteredTable) .withHiddenPropertiesSet( - getHiddenTablePropertyNames(getCatalogIdentifier(ident), alteredTable.properties())); + getHiddenPropertyNames( + getCatalogIdentifier(ident), + HasPropertyMetadata::tablePropertiesMetadata, + alteredTable.properties())); } TableEntity updatedTableEntity = @@ -574,78 +589,10 @@ public Table alterTable(NameIdentifier ident, TableChange... changes) return EntityCombinedTable.of(alteredTable, updatedTableEntity) .withHiddenPropertiesSet( - getHiddenTablePropertyNames(getCatalogIdentifier(ident), alteredTable.properties())); - } - - private Pair, Map> getTableAlterProperty( - TableChange... tableChanges) { - Map upserts = Maps.newHashMap(); - Map deletes = Maps.newHashMap(); - - Arrays.stream(tableChanges) - .forEach( - tableChange -> { - if (tableChange instanceof SetProperty) { - SetProperty setProperty = (SetProperty) tableChange; - upserts.put(setProperty.getProperty(), setProperty.getValue()); - } else if (tableChange instanceof RemoveProperty) { - RemoveProperty removeProperty = (RemoveProperty) tableChange; - deletes.put(removeProperty.getProperty(), removeProperty.getProperty()); - } - }); - - return Pair.of(upserts, deletes); - } - - private Pair, Map> getSchemaAlterProperty( - SchemaChange... schemaChanges) { - Map upserts = Maps.newHashMap(); - Map deletes = Maps.newHashMap(); - - Arrays.stream(schemaChanges) - .forEach( - schemaChange -> { - if (schemaChange instanceof SchemaChange.SetProperty) { - SchemaChange.SetProperty setProperty = (SchemaChange.SetProperty) schemaChange; - upserts.put(setProperty.getProperty(), setProperty.getValue()); - } else if (schemaChange instanceof SchemaChange.RemoveProperty) { - SchemaChange.RemoveProperty removeProperty = - (SchemaChange.RemoveProperty) schemaChange; - deletes.put(removeProperty.getProperty(), removeProperty.getProperty()); - } - }); - - return Pair.of(upserts, deletes); - } - - private void validateAlterTableProperties(NameIdentifier ident, TableChange... changes) { - doWithCatalog( - getCatalogIdentifier(ident), - c -> - c.doWithPropertiesMeta( - p -> { - Pair, Map> alterProperty = - getTableAlterProperty(changes); - p.tablePropertiesMetadata() - .validatePropertyForAlter(alterProperty.getLeft(), alterProperty.getRight()); - return null; - }), - IllegalArgumentException.class); - } - - private void validateAlterSchemaProperties(NameIdentifier ident, SchemaChange... changes) { - doWithCatalog( - getCatalogIdentifier(ident), - c -> - c.doWithPropertiesMeta( - p -> { - Pair, Map> alterProperty = - getSchemaAlterProperty(changes); - p.schemaPropertiesMetadata() - .validatePropertyForAlter(alterProperty.getLeft(), alterProperty.getRight()); - return null; - }), - IllegalArgumentException.class); + getHiddenPropertyNames( + getCatalogIdentifier(ident), + HasPropertyMetadata::tablePropertiesMetadata, + alteredTable.properties())); } /** @@ -699,6 +646,97 @@ public boolean purgeTable(NameIdentifier ident) throws UnsupportedOperationExcep return true; } + @Override + public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { + return doWithCatalog( + getCatalogIdentifier(NameIdentifier.of(namespace.levels())), + c -> c.doWithFilesetOps(f -> f.listFilesets(namespace)), + NoSuchSchemaException.class); + } + + @Override + public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + Fileset fileset = + doWithCatalog( + catalogIdent, + c -> c.doWithFilesetOps(f -> f.loadFileset(ident)), + NoSuchFilesetException.class); + + // Currently we only support maintaining the Fileset in the Gravitino's store. + return EntityCombinedFileset.of(fileset) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, HasPropertyMetadata::tablePropertiesMetadata, fileset.properties())); + } + + @Override + public Fileset createFileset( + NameIdentifier ident, + String comment, + Fileset.Type type, + String storageLocation, + Map properties) + throws NoSuchSchemaException, FilesetAlreadyExistsException { + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + doWithCatalog( + catalogIdent, + c -> + c.doWithPropertiesMeta( + p -> { + p.filesetPropertiesMetadata().validatePropertyForCreate(properties); + return null; + }), + IllegalArgumentException.class); + long uid = idGenerator.nextId(); + StringIdentifier stringId = StringIdentifier.fromId(uid); + Map updatedProperties = + StringIdentifier.newPropertiesWithId(stringId, properties); + + Fileset createdFileset = + doWithCatalog( + catalogIdent, + c -> + c.doWithFilesetOps( + f -> f.createFileset(ident, comment, type, storageLocation, updatedProperties)), + NoSuchSchemaException.class, + FilesetAlreadyExistsException.class); + return EntityCombinedFileset.of(createdFileset) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, + HasPropertyMetadata::tablePropertiesMetadata, + createdFileset.properties())); + } + + @Override + public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) + throws NoSuchFilesetException, IllegalArgumentException { + validateAlterProperties(ident, HasPropertyMetadata::filesetPropertiesMetadata, changes); + + NameIdentifier catalogIdent = getCatalogIdentifier(ident); + Fileset alteredFileset = + doWithCatalog( + catalogIdent, + c -> c.doWithFilesetOps(f -> f.alterFileset(ident, changes)), + NoSuchFilesetException.class, + IllegalArgumentException.class); + return EntityCombinedFileset.of(alteredFileset) + .withHiddenPropertiesSet( + getHiddenPropertyNames( + catalogIdent, + HasPropertyMetadata::tablePropertiesMetadata, + alteredFileset.properties())); + } + + @Override + public boolean dropFileset(NameIdentifier ident) { + return doWithCatalog( + getCatalogIdentifier(ident), + c -> c.doWithFilesetOps(f -> f.dropFileset(ident)), + NonEmptyEntityException.class); + } + private R doWithCatalog( NameIdentifier ident, ThrowableFunction fn, Class ex) throws E { @@ -736,6 +774,76 @@ private R doWithCatalog( } } + private Set getHiddenPropertyNames( + NameIdentifier catalogIdent, + ThrowableFunction provider, + Map properties) { + return doWithCatalog( + catalogIdent, + c -> + c.doWithPropertiesMeta( + p -> { + PropertiesMetadata propertiesMetadata = provider.apply(p); + return properties.keySet().stream() + .filter(propertiesMetadata::isHiddenProperty) + .collect(Collectors.toSet()); + }), + IllegalArgumentException.class); + } + + private void validateAlterProperties( + NameIdentifier ident, + ThrowableFunction provider, + T... changes) { + doWithCatalog( + getCatalogIdentifier(ident), + c -> + c.doWithPropertiesMeta( + p -> { + Map upserts = getPropertiesForSet(changes); + Map deletes = getPropertiesForDelete(changes); + provider.apply(p).validatePropertyForAlter(upserts, deletes); + return null; + }), + IllegalArgumentException.class); + } + + private Map getPropertiesForSet(T... t) { + Map properties = Maps.newHashMap(); + for (T item : t) { + if (item instanceof TableChange.SetProperty) { + TableChange.SetProperty setProperty = (TableChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } else if (item instanceof SchemaChange.SetProperty) { + SchemaChange.SetProperty setProperty = (SchemaChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } else if (item instanceof FilesetChange.SetProperty) { + FilesetChange.SetProperty setProperty = (FilesetChange.SetProperty) item; + properties.put(setProperty.getProperty(), setProperty.getValue()); + } + } + + return properties; + } + + private Map getPropertiesForDelete(T... t) { + Map properties = Maps.newHashMap(); + for (T item : t) { + if (item instanceof TableChange.RemoveProperty) { + TableChange.RemoveProperty removeProperty = (TableChange.RemoveProperty) item; + properties.put(removeProperty.getProperty(), removeProperty.getProperty()); + } else if (item instanceof SchemaChange.RemoveProperty) { + SchemaChange.RemoveProperty removeProperty = (SchemaChange.RemoveProperty) item; + properties.put(removeProperty.getProperty(), removeProperty.getProperty()); + } else if (item instanceof FilesetChange.RemoveProperty) { + FilesetChange.RemoveProperty removeProperty = (FilesetChange.RemoveProperty) item; + properties.put(removeProperty.getProperty(), removeProperty.getProperty()); + } + } + + return properties; + } + private StringIdentifier getStringIdFromProperties(Map properties) { try { StringIdentifier stringId = StringIdentifier.fromProperties(properties); @@ -795,11 +903,14 @@ NameIdentifier getCatalogIdentifier(NameIdentifier ident) { return NameIdentifier.of(allElems.get(0), allElems.get(1)); } - private boolean isManagedSchema(Schema schema) { - return schema - .properties() - .getOrDefault(BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY, Boolean.FALSE.toString()) - .equals(Boolean.TRUE.toString()); + private boolean isManagedEntity(Map properties) { + return Optional.ofNullable(properties) + .map( + p -> + p.getOrDefault( + BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY, Boolean.FALSE.toString()) + .equals(Boolean.TRUE.toString())) + .orElse(false); } private static final class FormattedErrorMessages { diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/file/EntityCombinedFileset.java b/core/src/main/java/com/datastrato/gravitino/catalog/file/EntityCombinedFileset.java new file mode 100644 index 00000000000..1c84cdef5dc --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/catalog/file/EntityCombinedFileset.java @@ -0,0 +1,83 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.catalog.file; + +import com.datastrato.gravitino.Audit; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.meta.AuditInfo; +import com.datastrato.gravitino.meta.FilesetEntity; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public final class EntityCombinedFileset implements Fileset { + + private final Fileset fileset; + + private final FilesetEntity filesetEntity; + + // Sets of properties that should be hidden from the user. + private Set hiddenProperties; + + private EntityCombinedFileset(Fileset fileset, FilesetEntity filesetEntity) { + this.fileset = fileset; + this.filesetEntity = filesetEntity; + } + + public static EntityCombinedFileset of(Fileset fileset, FilesetEntity filesetEntity) { + return new EntityCombinedFileset(fileset, filesetEntity); + } + + public static EntityCombinedFileset of(Fileset fileset) { + return new EntityCombinedFileset(fileset, null); + } + + public EntityCombinedFileset withHiddenPropertiesSet(Set hiddenProperties) { + this.hiddenProperties = hiddenProperties; + return this; + } + + @Override + public String name() { + return fileset.name(); + } + + @Override + public String comment() { + return fileset.comment(); + } + + @Override + public Type type() { + return fileset.type(); + } + + @Override + public String storageLocation() { + return fileset.storageLocation(); + } + + @Override + public Map properties() { + return fileset.properties().entrySet().stream() + .filter(p -> !hiddenProperties.contains(p.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Override + public Audit auditInfo() { + AuditInfo mergedAudit = + new AuditInfo.Builder() + .withCreator(fileset.auditInfo().creator()) + .withCreateTime(fileset.auditInfo().createTime()) + .withLastModifier(fileset.auditInfo().lastModifier()) + .withLastModifiedTime(fileset.auditInfo().lastModifiedTime()) + .build(); + + return filesetEntity == null + ? fileset.auditInfo() + : mergedAudit.merge(filesetEntity.auditInfo(), true /* overwrite */); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java b/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java index b74f0c5c5f5..1cdf60f26d3 100644 --- a/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java +++ b/core/src/test/java/com/datastrato/gravitino/TestCatalogOperations.java @@ -8,12 +8,17 @@ import com.datastrato.gravitino.catalog.CatalogOperations; import com.datastrato.gravitino.catalog.PropertiesMetadata; import com.datastrato.gravitino.catalog.PropertyEntry; +import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException; import com.datastrato.gravitino.exceptions.NoSuchCatalogException; +import com.datastrato.gravitino.exceptions.NoSuchFilesetException; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.NonEmptySchemaException; import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetCatalog; +import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Schema; @@ -33,12 +38,15 @@ import java.util.HashMap; import java.util.Map; -public class TestCatalogOperations implements CatalogOperations, TableCatalog, SupportsSchemas { +public class TestCatalogOperations + implements CatalogOperations, TableCatalog, FilesetCatalog, SupportsSchemas { private final Map tables; private final Map schemas; + private final Map filesets; + private final BasePropertiesMetadata tablePropertiesMetadata; private final BasePropertiesMetadata schemaPropertiesMetadata; @@ -52,6 +60,7 @@ public class TestCatalogOperations implements CatalogOperations, TableCatalog, S public TestCatalogOperations(Map config) { tables = Maps.newHashMap(); schemas = Maps.newHashMap(); + filesets = Maps.newHashMap(); tablePropertiesMetadata = new TestBasePropertiesMetadata(); schemaPropertiesMetadata = new TestBasePropertiesMetadata(); filesetPropertiesMetadata = new TestBasePropertiesMetadata(); @@ -376,4 +385,103 @@ protected Map> specificPropertyEntries() { public PropertiesMetadata filesetPropertiesMetadata() throws UnsupportedOperationException { return filesetPropertiesMetadata; } + + @Override + public NameIdentifier[] listFilesets(Namespace namespace) throws NoSuchSchemaException { + return filesets.keySet().stream() + .filter(ident -> ident.namespace().equals(namespace)) + .toArray(NameIdentifier[]::new); + } + + @Override + public Fileset loadFileset(NameIdentifier ident) throws NoSuchFilesetException { + if (filesets.containsKey(ident)) { + return filesets.get(ident); + } else { + throw new NoSuchFilesetException("Fileset " + ident + " does not exist"); + } + } + + @Override + public Fileset createFileset( + NameIdentifier ident, + String comment, + Fileset.Type type, + String storageLocation, + Map properties) + throws NoSuchSchemaException, FilesetAlreadyExistsException { + AuditInfo auditInfo = + new AuditInfo.Builder().withCreator("test").withCreateTime(Instant.now()).build(); + TestFileset fileset = + new TestFileset.Builder() + .withName(ident.name()) + .withComment(comment) + .withProperties(properties) + .withAuditInfo(auditInfo) + .withType(type) + .withStorageLocation(storageLocation) + .build(); + + if (tables.containsKey(ident)) { + throw new FilesetAlreadyExistsException("Fileset " + ident + " already exists"); + } else { + filesets.put(ident, fileset); + } + + return fileset; + } + + @Override + public Fileset alterFileset(NameIdentifier ident, FilesetChange... changes) + throws NoSuchFilesetException, IllegalArgumentException { + if (!filesets.containsKey(ident)) { + throw new NoSuchFilesetException("Fileset " + ident + " does not exist"); + } + + AuditInfo updatedAuditInfo = + new AuditInfo.Builder() + .withCreator("test") + .withCreateTime(Instant.now()) + .withLastModifier("test") + .withLastModifiedTime(Instant.now()) + .build(); + + TestFileset fileset = filesets.get(ident); + Map newProps = + fileset.properties() != null ? Maps.newHashMap(fileset.properties()) : Maps.newHashMap(); + + for (FilesetChange change : changes) { + if (change instanceof FilesetChange.SetProperty) { + newProps.put( + ((FilesetChange.SetProperty) change).getProperty(), + ((FilesetChange.SetProperty) change).getValue()); + } else if (change instanceof FilesetChange.RemoveProperty) { + newProps.remove(((FilesetChange.RemoveProperty) change).getProperty()); + } else { + throw new IllegalArgumentException("Unsupported fileset change: " + change); + } + } + + TestFileset updatedFileset = + new TestFileset.Builder() + .withName(ident.name()) + .withComment(fileset.comment()) + .withProperties(newProps) + .withAuditInfo(updatedAuditInfo) + .withType(fileset.type()) + .withStorageLocation(fileset.storageLocation()) + .build(); + filesets.put(ident, updatedFileset); + return updatedFileset; + } + + @Override + public boolean dropFileset(NameIdentifier ident) { + if (filesets.containsKey(ident)) { + filesets.remove(ident); + return true; + } else { + return false; + } + } } diff --git a/core/src/test/java/com/datastrato/gravitino/TestFileset.java b/core/src/test/java/com/datastrato/gravitino/TestFileset.java new file mode 100644 index 00000000000..2ec17364162 --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/TestFileset.java @@ -0,0 +1,27 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino; + +import com.datastrato.gravitino.catalog.file.BaseFileset; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +public class TestFileset extends BaseFileset { + + public static class Builder extends BaseFilesetBuilder { + + @Override + protected TestFileset internalBuild() { + TestFileset fileset = new TestFileset(); + fileset.name = name; + fileset.comment = comment; + fileset.properties = properties; + fileset.auditInfo = auditInfo; + fileset.type = type; + fileset.storageLocation = storageLocation; + return fileset; + } + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java index f13e28ca585..63f36e3a1bf 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java @@ -8,6 +8,7 @@ import static com.datastrato.gravitino.Entity.EntityType.TABLE; import static com.datastrato.gravitino.StringIdentifier.ID_KEY; import static com.datastrato.gravitino.TestBasePropertiesMetadata.COMMENT_KEY; +import static com.datastrato.gravitino.catalog.BasePropertiesMetadata.GRAVITINO_MANAGED_ENTITY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -29,6 +30,8 @@ import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.exceptions.IllegalNamespaceException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; +import com.datastrato.gravitino.file.Fileset; +import com.datastrato.gravitino.file.FilesetChange; import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.meta.BaseMetalake; import com.datastrato.gravitino.meta.SchemaEntity; @@ -535,6 +538,119 @@ public void testCreateAndDropTable() throws IOException { Assertions.assertThrows(RuntimeException.class, () -> dispatcher.dropTable(tableIdent)); } + @Test + public void testCreateAndListFilesets() { + Namespace filesetNs = Namespace.of(metalake, catalog, "schema81"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + dispatcher.createSchema(NameIdentifier.of(filesetNs.levels()), "comment", props); + + NameIdentifier filesetIdent1 = NameIdentifier.of(filesetNs, "fileset1"); + Fileset fileset1 = + dispatcher.createFileset(filesetIdent1, "comment", Fileset.Type.MANAGED, "test", props); + Assertions.assertEquals("fileset1", fileset1.name()); + Assertions.assertEquals("comment", fileset1.comment()); + testProperties(props, fileset1.properties()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); + Assertions.assertEquals("test", fileset1.storageLocation()); + + NameIdentifier[] idents = dispatcher.listFilesets(filesetNs); + Assertions.assertEquals(1, idents.length); + Assertions.assertEquals(filesetIdent1, idents[0]); + + Map illegalProps = ImmutableMap.of("k2", "v2"); + testPropertyException( + () -> + dispatcher.createFileset( + filesetIdent1, "comment", Fileset.Type.MANAGED, "test", illegalProps), + "Properties are required and must be set"); + + Map illegalProps2 = ImmutableMap.of("k1", "v1", ID_KEY, "test"); + testPropertyException( + () -> + dispatcher.createFileset( + filesetIdent1, "comment", Fileset.Type.MANAGED, "test", illegalProps2), + "Properties are reserved and cannot be set", + "gravitino.identifier"); + } + + @Test + public void testCreateAndLoadFileset() { + Namespace filesetNs = Namespace.of(metalake, catalog, "schema91"); + Map props = ImmutableMap.of("k1", "v1", "location", "schema91"); + dispatcher.createSchema(NameIdentifier.of(filesetNs.levels()), "comment", props); + + NameIdentifier filesetIdent1 = NameIdentifier.of(filesetNs, "fileset11"); + Fileset fileset1 = + dispatcher.createFileset(filesetIdent1, "comment", Fileset.Type.MANAGED, null, props); + Assertions.assertEquals("fileset11", fileset1.name()); + Assertions.assertEquals("comment", fileset1.comment()); + testProperties(props, fileset1.properties()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); + Assertions.assertNull(fileset1.storageLocation()); + + Fileset loadedFileset1 = dispatcher.loadFileset(filesetIdent1); + Assertions.assertEquals(fileset1.name(), loadedFileset1.name()); + Assertions.assertEquals(fileset1.comment(), loadedFileset1.comment()); + testProperties(props, loadedFileset1.properties()); + Assertions.assertEquals(fileset1.type(), loadedFileset1.type()); + Assertions.assertEquals(fileset1.storageLocation(), loadedFileset1.storageLocation()); + } + + @Test + public void testCreateAndAlterFileset() { + Namespace filesetNs = Namespace.of(metalake, catalog, "schema101"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + dispatcher.createSchema(NameIdentifier.of(filesetNs.levels()), "comment", props); + + NameIdentifier filesetIdent1 = NameIdentifier.of(filesetNs, "fileset21"); + Fileset fileset1 = + dispatcher.createFileset( + filesetIdent1, "comment", Fileset.Type.MANAGED, "fileset21", props); + Assertions.assertEquals("fileset21", fileset1.name()); + Assertions.assertEquals("comment", fileset1.comment()); + testProperties(props, fileset1.properties()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); + Assertions.assertEquals("fileset21", fileset1.storageLocation()); + + FilesetChange[] changes = + new FilesetChange[] { + FilesetChange.setProperty("k3", "v3"), FilesetChange.removeProperty("k1") + }; + + Fileset alteredFileset = dispatcher.alterFileset(filesetIdent1, changes); + Assertions.assertEquals(fileset1.name(), alteredFileset.name()); + Assertions.assertEquals(fileset1.comment(), alteredFileset.comment()); + Map expectedProps = ImmutableMap.of("k2", "v2", "k3", "v3"); + testProperties(expectedProps, alteredFileset.properties()); + + // Test immutable fileset properties + FilesetChange[] illegalChange = + new FilesetChange[] {FilesetChange.setProperty(GRAVITINO_MANAGED_ENTITY, "test")}; + testPropertyException( + () -> dispatcher.alterFileset(filesetIdent1, illegalChange), + "Property gravitino.managed.entity is immutable or reserved, cannot be set"); + } + + @Test + public void testCreateAndDropFileset() { + Namespace filesetNs = Namespace.of(metalake, catalog, "schema111"); + Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); + dispatcher.createSchema(NameIdentifier.of(filesetNs.levels()), "comment", props); + + NameIdentifier filesetIdent1 = NameIdentifier.of(filesetNs, "fileset31"); + Fileset fileset1 = + dispatcher.createFileset( + filesetIdent1, "comment", Fileset.Type.MANAGED, "fileset31", props); + Assertions.assertEquals("fileset31", fileset1.name()); + Assertions.assertEquals("comment", fileset1.comment()); + testProperties(props, fileset1.properties()); + Assertions.assertEquals(Fileset.Type.MANAGED, fileset1.type()); + Assertions.assertEquals("fileset31", fileset1.storageLocation()); + + boolean dropped = dispatcher.dropFileset(filesetIdent1); + Assertions.assertTrue(dropped); + } + @Test public void testGetCatalogIdentifier() { CatalogOperationDispatcher dispatcher = new CatalogOperationDispatcher(null, null, null); @@ -561,6 +677,7 @@ private void testProperties(Map expectedProps, Map