From 9adb6189a9a63e4ff4c7e904b11ac40c35bb7881 Mon Sep 17 00:00:00 2001 From: Clearvive Date: Tue, 9 Jan 2024 16:35:41 +0800 Subject: [PATCH] [#1339] feat(table): Add index for tables. --- .../com/datastrato/gravitino/rel/Table.java | 6 ++ .../gravitino/rel/TableCatalog.java | 32 ++++++++++- .../rel/expressions/Indexes/Index.java | 16 ++++++ .../gravitino/dto/rel/TableDTO.java | 30 +++++++++- .../dto/rel/indexes/PrimaryKeyDTO.java | 57 +++++++++++++++++++ .../gravitino/dto/rel/indexes/UniqueDTO.java | 57 +++++++++++++++++++ .../dto/requests/TableCreateRequest.java | 24 +++++++- .../gravitino/catalog/rel/BaseTable.java | 16 ++++++ .../catalog/rel/EntityCombinedTable.java | 6 ++ 9 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 api/src/main/java/com/datastrato/gravitino/rel/expressions/Indexes/Index.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/PrimaryKeyDTO.java create mode 100644 common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/UniqueDTO.java diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Table.java b/api/src/main/java/com/datastrato/gravitino/rel/Table.java index 0f3f7c22a44..55daaa03f07 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/Table.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/Table.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.Auditable; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.rel.expressions.Indexes.Index; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; @@ -48,6 +49,11 @@ default Distribution distribution() { return Distributions.NONE; } + /** @return The indexes of the table. If no indexes are specified, Indexes.NONE is returned. */ + default Index[] index() { + return new Index[0]; + } + /** @return The comment of the table. Null is returned if no comment is set. */ @Nullable default String comment() { diff --git a/api/src/main/java/com/datastrato/gravitino/rel/TableCatalog.java b/api/src/main/java/com/datastrato/gravitino/rel/TableCatalog.java index b4231cb93ad..0dd63a7f817 100644 --- a/api/src/main/java/com/datastrato/gravitino/rel/TableCatalog.java +++ b/api/src/main/java/com/datastrato/gravitino/rel/TableCatalog.java @@ -27,6 +27,7 @@ import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.exceptions.TableAlreadyExistsException; +import com.datastrato.gravitino.rel.expressions.Indexes.Index; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.distributions.Distributions; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; @@ -197,7 +198,7 @@ default Table createTable( * @throws NoSuchSchemaException If the schema does not exist. * @throws TableAlreadyExistsException If the table already exists. */ - Table createTable( + default Table createTable( NameIdentifier ident, Column[] columns, String comment, @@ -205,6 +206,35 @@ Table createTable( Transform[] partitions, Distribution distribution, SortOrder[] sortOrders) + throws NoSuchSchemaException, TableAlreadyExistsException { + return createTable( + ident, columns, comment, properties, partitions, distribution, sortOrders, new Index[0]); + } + + /** + * Create a partitioned table in the catalog. + * + * @param ident A table identifier. + * @param columns The columns of the new table. + * @param comment The table comment. + * @param properties The table properties. + * @param distribution The distribution of the table + * @param sortOrders The sort orders of the table + * @param partitions The table partitioning. + * @param indexes The table indexes. + * @return Fhe created table metadata. + * @throws NoSuchSchemaException If the schema does not exist. + * @throws TableAlreadyExistsException If the table already exists. + */ + Table createTable( + NameIdentifier ident, + Column[] columns, + String comment, + Map properties, + Transform[] partitions, + Distribution distribution, + SortOrder[] sortOrders, + Index[] indexes) throws NoSuchSchemaException, TableAlreadyExistsException; /** diff --git a/api/src/main/java/com/datastrato/gravitino/rel/expressions/Indexes/Index.java b/api/src/main/java/com/datastrato/gravitino/rel/expressions/Indexes/Index.java new file mode 100644 index 00000000000..88eaaeccc70 --- /dev/null +++ b/api/src/main/java/com/datastrato/gravitino/rel/expressions/Indexes/Index.java @@ -0,0 +1,16 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.rel.expressions.Indexes; + +/** Represents an index on a table. */ +public interface Index { + + /** @return The name of the index. */ + String name(); + + /** @return Returns the field name under the table contained in the index. eg: table.id */ + String[][] fieldNames(); +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java index bd57c09abb5..877d1c35f66 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.dto.rel.partitions.Partitioning; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.expressions.Indexes.Index; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; @@ -42,6 +43,9 @@ public class TableDTO implements Table { @JsonProperty("partitioning") private Partitioning[] partitioning; + @JsonProperty("indexes") + private Index[] indexes; + private TableDTO() {} /** @@ -53,6 +57,7 @@ private TableDTO() {} * @param properties The properties associated with the table. * @param audit The audit information for the table. * @param partitioning The partitioning of the table. + * @param indexes Teh indexes of the table. */ private TableDTO( String name, @@ -62,7 +67,8 @@ private TableDTO( AuditDTO audit, Partitioning[] partitioning, DistributionDTO distribution, - SortOrderDTO[] sortOrderDTOs) { + SortOrderDTO[] sortOrderDTOs, + Index[] indexes) { this.name = name; this.comment = comment; this.columns = columns; @@ -71,6 +77,7 @@ private TableDTO( this.distribution = distribution; this.sortOrders = sortOrderDTOs; this.partitioning = partitioning; + this.indexes = indexes; } @Override @@ -113,6 +120,11 @@ public Distribution distribution() { return distribution; } + @Override + public Index[] index() { + return indexes; + } + /** * Creates a new Builder to build a Table DTO. * @@ -136,6 +148,7 @@ public static class Builder { protected SortOrderDTO[] sortOrderDTOs; protected DistributionDTO distributionDTO; protected Partitioning[] Partitioning; + protected Index[] indexes; public Builder() {} @@ -209,6 +222,11 @@ public S withPartitioning(Partitioning[] Partitioning) { return (S) this; } + public S withIndex(Index[] indexes) { + this.indexes = indexes; + return (S) this; + } + /** * Builds a Table DTO based on the provided builder parameters. * @@ -222,7 +240,15 @@ public TableDTO build() { Preconditions.checkArgument(audit != null, "audit cannot be null"); return new TableDTO( - name, comment, columns, properties, audit, Partitioning, distributionDTO, sortOrderDTOs); + name, + comment, + columns, + properties, + audit, + Partitioning, + distributionDTO, + sortOrderDTOs, + indexes); } } } diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/PrimaryKeyDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/PrimaryKeyDTO.java new file mode 100644 index 00000000000..6d8327faf83 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/PrimaryKeyDTO.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.rel.indexes; + +import com.datastrato.gravitino.rel.expressions.Indexes.Index; +import com.google.common.base.Preconditions; + +public class PrimaryKeyDTO implements Index { + + private final String name; + private final String fieldName; + + public PrimaryKeyDTO(String name, String fieldName) { + this.name = name; + this.fieldName = fieldName; + } + + @Override + public String name() { + return name; + } + + @Override + public String[][] fieldNames() { + return new String[][] {new String[] {fieldName}}; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + protected String name; + protected String fieldName; + + public Builder() {} + + public S withName(String name) { + this.name = name; + return (S) this; + } + + public S withFieldName(String fieldName) { + this.fieldName = fieldName; + return (S) this; + } + + public PrimaryKeyDTO build() { + Preconditions.checkNotNull(name, "Index name cannot be null"); + Preconditions.checkNotNull(fieldName, "Primary key must specify the field name"); + return new PrimaryKeyDTO(name, fieldName); + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/UniqueDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/UniqueDTO.java new file mode 100644 index 00000000000..bfb3bc551d5 --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/dto/rel/indexes/UniqueDTO.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.dto.rel.indexes; + +import com.datastrato.gravitino.rel.expressions.Indexes.Index; +import com.google.common.base.Preconditions; + +public class UniqueDTO implements Index { + + private final String name; + private final String fieldName; + + public UniqueDTO(String name, String fieldName) { + this.name = name; + this.fieldName = fieldName; + } + + @Override + public String name() { + return name; + } + + @Override + public String[][] fieldNames() { + return new String[][] {new String[] {fieldName}}; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + protected String name; + protected String fieldName; + + public Builder() {} + + public S withName(String name) { + this.name = name; + return (S) this; + } + + public S withFieldName(String fieldName) { + this.fieldName = fieldName; + return (S) this; + } + + public PrimaryKeyDTO build() { + Preconditions.checkNotNull(name, "Index name cannot be null"); + Preconditions.checkNotNull(fieldName, "Unique index must specify the field name"); + return new PrimaryKeyDTO(name, fieldName); + } + } +} diff --git a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java index 4f8e538af7a..194ad13e8f2 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/requests/TableCreateRequest.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.dto.rel.SortOrderDTO; import com.datastrato.gravitino.dto.rel.expressions.FunctionArg; import com.datastrato.gravitino.dto.rel.partitions.Partitioning; +import com.datastrato.gravitino.rel.expressions.Indexes.Index; import com.datastrato.gravitino.rest.RESTRequest; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; @@ -57,6 +58,10 @@ public class TableCreateRequest implements RESTRequest { @JsonProperty("partitioning") private final Partitioning[] partitioning; + @Nullable + @JsonProperty("indexes") + private final Index[] indexes; + public TableCreateRequest() { this(null, null, null, null, null, null, null); } @@ -73,6 +78,17 @@ public TableCreateRequest( new Partitioning[0]); } + public TableCreateRequest( + String name, + String comment, + ColumnDTO[] columns, + Map properties, + SortOrderDTO[] sortOrders, + DistributionDTO distribution, + Partitioning[] partitioning) { + this(name, comment, columns, properties, sortOrders, distribution, partitioning, new Index[0]); + } + public TableCreateRequest( String name, @Nullable String comment, @@ -80,7 +96,8 @@ public TableCreateRequest( @Nullable Map properties, @Nullable SortOrderDTO[] sortOrders, @Nullable DistributionDTO distribution, - @Nullable Partitioning[] partitioning) { + @Nullable Partitioning[] partitioning, + @Nullable Index[] indexes) { this.name = name; this.columns = columns; this.comment = comment; @@ -88,6 +105,7 @@ public TableCreateRequest( this.sortOrders = sortOrders; this.distribution = distribution; this.partitioning = partitioning; + this.indexes = indexes; } @Override @@ -122,5 +140,9 @@ public void validate() throws IllegalArgumentException { autoIncrementCols.size() <= 1, "Only one column can be auto-incremented. There are multiple auto-increment columns in your table: " + autoIncrementColsStr); + + if (indexes != null && indexes.length > 0) { + throw new UnsupportedOperationException("Support for indexing is currently not implemented"); + } } } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseTable.java b/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseTable.java index f5ee6008c31..363384c3df7 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseTable.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/rel/BaseTable.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.expressions.Indexes.Index; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; @@ -34,6 +35,8 @@ public abstract class BaseTable implements Table { @Nullable protected Distribution distribution; + @Nullable protected Index[] indexes; + /** Returns the audit details of the table. */ @Override public AuditInfo auditInfo() { @@ -87,6 +90,11 @@ public Distribution distribution() { return distribution; } + @Override + public Index[] index() { + return indexes; + } + /** * Builder interface for creating instances of {@link BaseTable}. * @@ -111,6 +119,8 @@ interface Builder, T extends BaseTable> { SELF withDistribution(Distribution distribution); + SELF withIndexes(Index[] indexes); + T build(); } @@ -131,6 +141,7 @@ public abstract static class BaseTableBuilder, T e protected SortOrder[] sortOrders; protected Distribution distribution; + protected Index[] indexes; /** * Sets the name of the table. @@ -214,6 +225,11 @@ public SELF withDistribution(Distribution distribution) { return (SELF) this; } + public SELF withIndexes(Index[] indexes) { + this.indexes = indexes; + return (SELF) this; + } + /** * Builds the instance of the table with the provided attributes. * diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/rel/EntityCombinedTable.java b/core/src/main/java/com/datastrato/gravitino/catalog/rel/EntityCombinedTable.java index 7b4c463d26b..1a167aaed7e 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/rel/EntityCombinedTable.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/rel/EntityCombinedTable.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.meta.TableEntity; import com.datastrato.gravitino.rel.Column; import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.expressions.Indexes.Index; import com.datastrato.gravitino.rel.expressions.distributions.Distribution; import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; @@ -84,6 +85,11 @@ public Distribution distribution() { return table.distribution(); } + @Override + public Index[] index() { + return table.index(); + } + @Override public Audit auditInfo() { AuditInfo mergedAudit =