Skip to content

Commit

Permalink
[#1339] feat(table): Add index for tables.
Browse files Browse the repository at this point in the history
  • Loading branch information
Clearvive authored and Clearvive committed Jan 9, 2024
1 parent ad6e2b9 commit 9614ffd
Show file tree
Hide file tree
Showing 14 changed files with 256 additions and 11 deletions.
6 changes: 6 additions & 0 deletions api/src/main/java/com/datastrato/gravitino/rel/Table.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import com.datastrato.gravitino.Auditable;
import com.datastrato.gravitino.Namespace;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.distributions.Distributions;
import com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
Expand Down Expand Up @@ -48,6 +49,11 @@ default Distribution distribution() {
return Distributions.NONE;
}

/** @return The indexes of the table. If no indexes are specified, Indexes.NONE is returned. */
default Index[] index() {
return new Index[0];
}

/** @return The comment of the table. Null is returned if no comment is set. */
@Nullable
default String comment() {
Expand Down
32 changes: 31 additions & 1 deletion api/src/main/java/com/datastrato/gravitino/rel/TableCatalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
import com.datastrato.gravitino.exceptions.NoSuchTableException;
import com.datastrato.gravitino.exceptions.TableAlreadyExistsException;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.distributions.Distributions;
import com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
Expand Down Expand Up @@ -197,14 +198,43 @@ default Table createTable(
* @throws NoSuchSchemaException If the schema does not exist.
* @throws TableAlreadyExistsException If the table already exists.
*/
Table createTable(
default Table createTable(
NameIdentifier ident,
Column[] columns,
String comment,
Map<String, String> properties,
Transform[] partitions,
Distribution distribution,
SortOrder[] sortOrders)
throws NoSuchSchemaException, TableAlreadyExistsException {
return createTable(
ident, columns, comment, properties, partitions, distribution, sortOrders, new Index[0]);
}

/**
* Create a partitioned table in the catalog.
*
* @param ident A table identifier.
* @param columns The columns of the new table.
* @param comment The table comment.
* @param properties The table properties.
* @param distribution The distribution of the table
* @param sortOrders The sort orders of the table
* @param partitions The table partitioning.
* @param indexes The table indexes.
* @return Fhe created table metadata.
* @throws NoSuchSchemaException If the schema does not exist.
* @throws TableAlreadyExistsException If the table already exists.
*/
Table createTable(
NameIdentifier ident,
Column[] columns,
String comment,
Map<String, String> properties,
Transform[] partitions,
Distribution distribution,
SortOrder[] sortOrders,
Index[] indexes)
throws NoSuchSchemaException, TableAlreadyExistsException;

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/

package com.datastrato.gravitino.rel.expressions.Indexes;

/** Represents an index on a table. */
public interface Index {

IndexType type();

/** @return The name of the index. */
String name();

/** @return Returns the field name under the table contained in the index. eg: table.id */
String[][] fieldNames();

enum IndexType {
PRIMARY_KEY,
UNIQUE_KEY,
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.datastrato.gravitino.rel.Table;
import com.datastrato.gravitino.rel.TableCatalog;
import com.datastrato.gravitino.rel.TableChange;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.NamedReference;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.distributions.Distributions;
Expand Down Expand Up @@ -543,6 +544,7 @@ private void validateDistributionAndSort(Distribution distribution, SortOrder[]
* @param comment The comment for the new table.
* @param properties The properties for the new table.
* @param partitioning The partitioning for the new table.
* @param indexes The indexes for the new table.
* @return The newly created HiveTable instance.
* @throws NoSuchSchemaException If the schema for the table does not exist.
* @throws TableAlreadyExistsException If the table with the same name already exists.
Expand All @@ -555,8 +557,11 @@ public Table createTable(
Map<String, String> properties,
Transform[] partitioning,
Distribution distribution,
SortOrder[] sortOrders)
SortOrder[] sortOrders,
Index[] indexes)
throws NoSuchSchemaException, TableAlreadyExistsException {
Preconditions.checkArgument(
null == indexes || indexes.length == 0, "hive-catalog does not support indexes");
NameIdentifier schemaIdent = NameIdentifier.of(tableIdent.namespace().levels());

validatePartitionForCreate(columns, partitioning);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.datastrato.gravitino.rel.Table;
import com.datastrato.gravitino.rel.TableCatalog;
import com.datastrato.gravitino.rel.TableChange;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.distributions.Distributions;
import com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
Expand Down Expand Up @@ -342,6 +343,7 @@ public boolean dropTable(NameIdentifier tableIdent) {
* @param comment The comment for the new table.
* @param properties The properties for the new table.
* @param partitioning The partitioning for the new table.
* @param indexes The indexes for the new table.
* @return The newly created JdbcTable instance.
* @throws NoSuchSchemaException If the schema for the table does not exist.
* @throws TableAlreadyExistsException If the table with the same name already exists.
Expand All @@ -354,8 +356,11 @@ public Table createTable(
Map<String, String> properties,
Transform[] partitioning,
Distribution distribution,
SortOrder[] sortOrders)
SortOrder[] sortOrders,
Index[] indexes)
throws NoSuchSchemaException, TableAlreadyExistsException {
Preconditions.checkArgument(
null == indexes || indexes.length == 0, "jdbc-catalog does not support indexes");
Preconditions.checkArgument(
null == distribution || distribution == Distributions.NONE,
"jdbc-catalog does not support distribution");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.datastrato.gravitino.rel.Table;
import com.datastrato.gravitino.rel.TableCatalog;
import com.datastrato.gravitino.rel.TableChange;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.distributions.Distributions;
import com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
Expand Down Expand Up @@ -468,6 +469,7 @@ public boolean dropTable(NameIdentifier tableIdent) {
* @param comment The comment for the new table.
* @param properties The properties for the new table.
* @param partitioning The partitioning for the new table.
* @param indexes The indexes for the new table.
* @return The newly created IcebergTable instance.
* @throws NoSuchSchemaException If the schema for the table does not exist.
* @throws TableAlreadyExistsException If the table with the same name already exists.
Expand All @@ -480,8 +482,11 @@ public Table createTable(
Map<String, String> properties,
Transform[] partitioning,
Distribution distribution,
SortOrder[] sortOrders)
SortOrder[] sortOrders,
Index[] indexes)
throws NoSuchSchemaException, TableAlreadyExistsException {
Preconditions.checkArgument(
null == indexes || indexes.length == 0, "iceberg-catalog does not support indexes");
try {
if (!Distributions.NONE.equals(distribution)) {
throw new UnsupportedOperationException("Iceberg does not support distribution");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.datastrato.gravitino.rel.Table;
import com.datastrato.gravitino.rel.TableCatalog;
import com.datastrato.gravitino.rel.TableChange;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
import com.datastrato.gravitino.rel.expressions.transforms.Transform;
Expand Down Expand Up @@ -135,6 +136,7 @@ public Table loadTable(NameIdentifier ident) throws NoSuchTableException {
* @param comment The comment of the table.
* @param properties The properties of the table.
* @param partitioning The partitioning of the table.
* @param indexes The indexes of the table.
* @return The created {@link Table}.
* @throws NoSuchSchemaException if the schema with specified namespace does not exist.
* @throws TableAlreadyExistsException if the table with specified identifier already exists.
Expand All @@ -147,7 +149,8 @@ public Table createTable(
Map<String, String> properties,
Transform[] partitioning,
Distribution distribution,
SortOrder[] sortOrders)
SortOrder[] sortOrders,
Index[] indexes)
throws NoSuchSchemaException, TableAlreadyExistsException {
NameIdentifier.checkTable(ident);

Expand All @@ -159,7 +162,8 @@ public Table createTable(
properties,
toDTOs(sortOrders),
toDTO(distribution),
toDTOs(partitioning));
toDTOs(partitioning),
toDTOs(indexes));
req.validate();

TableResponse resp =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import com.datastrato.gravitino.dto.rel.partitions.Partitioning;
import com.datastrato.gravitino.rel.Column;
import com.datastrato.gravitino.rel.Table;
import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.datastrato.gravitino.rel.expressions.distributions.Distribution;
import com.datastrato.gravitino.rel.expressions.sorts.SortOrder;
import com.datastrato.gravitino.rel.expressions.transforms.Transform;
Expand Down Expand Up @@ -42,6 +43,9 @@ public class TableDTO implements Table {
@JsonProperty("partitioning")
private Partitioning[] partitioning;

@JsonProperty("indexes")
private Index[] indexes;

private TableDTO() {}

/**
Expand All @@ -53,6 +57,7 @@ private TableDTO() {}
* @param properties The properties associated with the table.
* @param audit The audit information for the table.
* @param partitioning The partitioning of the table.
* @param indexes Teh indexes of the table.
*/
private TableDTO(
String name,
Expand All @@ -62,7 +67,8 @@ private TableDTO(
AuditDTO audit,
Partitioning[] partitioning,
DistributionDTO distribution,
SortOrderDTO[] sortOrderDTOs) {
SortOrderDTO[] sortOrderDTOs,
Index[] indexes) {
this.name = name;
this.comment = comment;
this.columns = columns;
Expand All @@ -71,6 +77,7 @@ private TableDTO(
this.distribution = distribution;
this.sortOrders = sortOrderDTOs;
this.partitioning = partitioning;
this.indexes = indexes;
}

@Override
Expand Down Expand Up @@ -113,6 +120,11 @@ public Distribution distribution() {
return distribution;
}

@Override
public Index[] index() {
return indexes;
}

/**
* Creates a new Builder to build a Table DTO.
*
Expand All @@ -136,6 +148,7 @@ public static class Builder<S extends Builder> {
protected SortOrderDTO[] sortOrderDTOs;
protected DistributionDTO distributionDTO;
protected Partitioning[] Partitioning;
protected Index[] indexes;

public Builder() {}

Expand Down Expand Up @@ -209,6 +222,11 @@ public S withPartitioning(Partitioning[] Partitioning) {
return (S) this;
}

public S withIndex(Index[] indexes) {
this.indexes = indexes;
return (S) this;
}

/**
* Builds a Table DTO based on the provided builder parameters.
*
Expand All @@ -222,7 +240,15 @@ public TableDTO build() {
Preconditions.checkArgument(audit != null, "audit cannot be null");

return new TableDTO(
name, comment, columns, properties, audit, Partitioning, distributionDTO, sortOrderDTOs);
name,
comment,
columns,
properties,
audit,
Partitioning,
distributionDTO,
sortOrderDTOs,
indexes);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
package com.datastrato.gravitino.dto.rel.indexes;

import com.datastrato.gravitino.rel.expressions.Indexes.Index;
import com.google.common.base.Preconditions;

public class IndexDTO implements Index {

private final IndexType indexType;
private final String name;
private final String[][] fieldNames;

public IndexDTO(IndexType indexType, String name, String[][] fieldNames) {
this.indexType = indexType;
this.name = name;
this.fieldNames = fieldNames;
}

@Override
public IndexType type() {
return indexType;
}

@Override
public String name() {
return name;
}

@Override
public String[][] fieldNames() {
return fieldNames;
}

public static Builder builder() {
return new Builder();
}

public static class Builder<S extends IndexDTO.Builder> {

protected IndexType indexType;

protected String name;
protected String[][] fieldNames;

public Builder() {}

public S withIndexType(IndexType indexType) {
this.indexType = indexType;
return (S) this;
}

public S withName(String name) {
this.name = name;
return (S) this;
}

public S withFieldNames(String[][] fieldNames) {
this.fieldNames = fieldNames;
return (S) this;
}

public IndexDTO build() {
Preconditions.checkNotNull(indexType, "Index type cannot be null");
Preconditions.checkNotNull(name, "Index name cannot be null");
Preconditions.checkNotNull(
fieldNames, "The index must be set with corresponding column names");
return new IndexDTO(indexType, name, fieldNames);
}
}
}
Loading

0 comments on commit 9614ffd

Please sign in to comment.