[ISSUE-3607] Support to sync the entities

apache · May 30, 2024 · 30b5053 · 30b5053
1 parent 844a8e6
commit 30b5053
Show file tree

Hide file tree

Showing 29 changed files with 906 additions and 114 deletions.
diff --git a/api/src/main/java/com/datastrato/gravitino/Schema.java b/api/src/main/java/com/datastrato/gravitino/Schema.java
@@ -33,4 +33,8 @@ default String comment() {
   default Map<String, String> properties() {
     return Collections.emptyMap();
   }
+
+  default boolean imported() {
+    return false;
+  }
 }
diff --git a/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java b/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java
@@ -34,4 +34,8 @@ default String comment() {
   default Map<String, String> properties() {
     return Collections.emptyMap();
   }
+
+  default boolean imported() {
+    return false;
+  }
 }
diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Table.java b/api/src/main/java/com/datastrato/gravitino/rel/Table.java
@@ -81,4 +81,8 @@ default Map<String, String> properties() {
   default SupportsPartitions supportPartitions() throws UnsupportedOperationException {
     throw new UnsupportedOperationException("Table does not support partition operations.");
   }
+
+  default boolean imported() {
+    return false;
+  }
 }
diff --git a/common/src/main/java/com/datastrato/gravitino/dto/MetadataObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/MetadataObjectDTO.java
@@ -0,0 +1,111 @@
+package com.datastrato.gravitino.dto;
+
+import com.datastrato.gravitino.MetadataObject;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
+
+public class MetadataObjectDTO implements MetadataObject {
+
+  @JsonProperty("fullName")
+  private String fullName;
+
+  @JsonProperty("type")
+  private Type type;
+
+  private String parent;
+  private String name;
+
+  /** Default constructor for Jackson deserialization. */
+  protected MetadataObjectDTO() {}
+
+  /**
+   * Creates a new instance of MetadataObject DTO.
+   *
+   * @param fullName The name of the MetadataObject DTO.
+   * @param type The type of the meta data object.
+   */
+  protected MetadataObjectDTO(String fullName, Type type) {
+    this.type = type;
+    this.fullName = fullName;
+    int index = fullName.lastIndexOf(".");
+
+    if (index == -1) {
+      this.parent = null;
+      this.name = fullName;
+    } else {
+      this.parent = fullName.substring(0, index);
+      this.name = fullName.substring(index + 1);
+    }
+  }
+
+  @Nullable
+  @Override
+  public String parent() {
+    return parent;
+  }
+
+  @Override
+  public String name() {
+    return name;
+  }
+
+  @Override
+  public String fullName() {
+    return fullName;
+  }
+
+  @Override
+  public Type type() {
+    return type;
+  }
+
+  /** @return the builder for creating a new instance of MetadataObjectDTO. */
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  /** Builder for {@link MetadataObjectDTO}. */
+  public static class Builder {
+    private String fullName;
+    private Type type;
+
+    /**
+     * Sets the full name of the meta data object.
+     *
+     * @param fullName The full name of the meta data object.
+     * @return The builder instance.
+     */
+    public Builder withFullName(String fullName) {
+      this.fullName = fullName;
+      return this;
+    }
+
+    /**
+     * Sets the type of the meta data object.
+     *
+     * @param type The type of the meta data object.
+     * @return The builder instance.
+     */
+    public Builder withType(Type type) {
+      this.type = type;
+      return this;
+    }
+
+    /**
+     * Builds an instance of MetadataObjectDTO using the builder's properties.
+     *
+     * @return An instance of MetadataObjectDTO.
+     * @throws IllegalArgumentException If the full name or type are not set.
+     */
+    public MetadataObjectDTO build() {
+      Preconditions.checkArgument(
+          StringUtils.isNotBlank(fullName), "full name cannot be null or empty");
+
+      Preconditions.checkArgument(type != null, "type cannot be null");
+
+      return new MetadataObjectDTO(fullName, type);
+    }
+  }
+}
diff --git a/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java
@@ -21,6 +21,9 @@ public class SchemaDTO implements Schema {
   @JsonProperty("properties")
   private Map<String, String> properties;
 
+  @JsonProperty("imported")
+  private boolean imported;
+
   @JsonProperty("audit")
   private AuditDTO audit;
 
@@ -34,10 +37,16 @@ private SchemaDTO() {}
    * @param properties The properties associated with the schema.
    * @param audit The audit information for the schema.
    */
-  private SchemaDTO(String name, String comment, Map<String, String> properties, AuditDTO audit) {
+  private SchemaDTO(
+      String name,
+      String comment,
+      Map<String, String> properties,
+      boolean imported,
+      AuditDTO audit) {
     this.name = name;
     this.comment = comment;
     this.properties = properties;
+    this.imported = imported;
     this.audit = audit;
   }
 
@@ -59,6 +68,11 @@ public Map<String, String> properties() {
     return properties;
   }
 
+  @Override
+  public boolean imported() {
+    return imported;
+  }
+
   /** @return The audit information for the schema. */
   @Override
   public AuditDTO auditInfo() {
@@ -79,6 +93,8 @@ public static class Builder<S extends Builder> {
     protected Map<String, String> properties;
     /** The audit information for the schema. */
     protected AuditDTO audit;
+    /** The imported status for the schema. */
+    protected boolean imported;
 
     /** Constructs a new Builder instance. */
     private Builder() {}
@@ -127,6 +143,11 @@ public S withAudit(AuditDTO audit) {
       return (S) this;
     }
 
+    public S withImported(boolean imported) {
+      this.imported = imported;
+      return (S) this;
+    }
+
     /**
      * Builds a Schema DTO based on the provided builder parameters.
      *
@@ -137,7 +158,7 @@ public SchemaDTO build() {
       Preconditions.checkArgument(name != null && !name.isEmpty(), "name cannot be null or empty");
       Preconditions.checkArgument(audit != null, "audit cannot be null");
 
-      return new SchemaDTO(name, comment, properties, audit);
+      return new SchemaDTO(name, comment, properties, imported, audit);
     }
   }
 

diff --git a/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java
@@ -29,6 +29,9 @@ public static Builder builder() {
   @JsonProperty("properties")
   private Map<String, String> properties;
 
+  @JsonProperty("imported")
+  private boolean imported;
+
   @JsonProperty("audit")
   private AuditDTO audit;
 
@@ -42,7 +45,12 @@ private TopicDTO() {}
    * @param properties The properties associated with the topic.
    * @param audit The audit information for the topic.
    */
-  private TopicDTO(String name, String comment, Map<String, String> properties, AuditDTO audit) {
+  private TopicDTO(
+      String name,
+      String comment,
+      Map<String, String> properties,
+      boolean imported,
+      AuditDTO audit) {
     this.name = name;
     this.comment = comment;
     this.properties = properties;
@@ -65,6 +73,11 @@ public Map<String, String> properties() {
     return properties;
   }
 
+  @Override
+  public boolean imported() {
+    return imported;
+  }
+
   @Override
   public Audit auditInfo() {
     return audit;
@@ -82,12 +95,13 @@ public boolean equals(Object o) {
     return Objects.equals(name, topicDTO.name)
         && Objects.equals(comment, topicDTO.comment)
         && Objects.equals(properties, topicDTO.properties)
+        && Objects.equals(imported, topicDTO.imported)
         && Objects.equals(audit, topicDTO.audit);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(name, comment, properties, audit);
+    return Objects.hash(name, comment, properties, imported, audit);
   }
 
   /** A builder for constructing a Topic DTO. */
@@ -142,6 +156,11 @@ public Builder withAudit(AuditDTO audit) {
       return this;
     }
 
+    public Builder withImported(boolean imported) {
+      topic.imported = imported;
+      return this;
+    }
+
     /** @return The constructed Topic DTO. */
     public TopicDTO build() {
       return topic;

diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java
@@ -47,6 +47,9 @@ public class TableDTO implements Table {
   @JsonProperty("indexes")
   private IndexDTO[] indexes;
 
+  @JsonProperty("imported")
+  private boolean imported;
+
   private TableDTO() {}
 
   /**
@@ -58,7 +61,8 @@ private TableDTO() {}
    * @param properties The properties associated with the table.
    * @param audit The audit information for the table.
    * @param partitioning The partitioning of the table.
-   * @param indexes Teh indexes of the table.
+   * @param indexes The indexes of the table.
+   * @param imported The table is whether to be imported to Gravitino.
    */
   private TableDTO(
       String name,
@@ -69,7 +73,8 @@ private TableDTO(
       Partitioning[] partitioning,
       DistributionDTO distribution,
       SortOrderDTO[] sortOrderDTOs,
-      IndexDTO[] indexes) {
+      IndexDTO[] indexes,
+      boolean imported) {
     this.name = name;
     this.comment = comment;
     this.columns = columns;
@@ -79,6 +84,7 @@ private TableDTO(
     this.sortOrders = sortOrderDTOs;
     this.partitioning = partitioning;
     this.indexes = indexes;
+    this.imported = imported;
   }
 
   /** @return The name of the table. */
@@ -135,6 +141,11 @@ public Index[] index() {
     return indexes;
   }
 
+  @Override
+  public boolean imported() {
+    return imported;
+  }
+
   /**
    * Creates a new Builder to build a Table DTO.
    *
@@ -169,6 +180,8 @@ public static class Builder<S extends Builder> {
     /** The indexes of the table. */
     protected IndexDTO[] indexes;
 
+    protected boolean imported;
+
     /** Default constructor. */
     private Builder() {}
 
@@ -271,6 +284,11 @@ public S withIndex(IndexDTO[] indexes) {
       return (S) this;
     }
 
+    public S withImported(boolean imported) {
+      this.imported = imported;
+      return (S) this;
+    }
+
     /**
      * Builds a Table DTO based on the provided builder parameters.
      *
@@ -292,7 +310,8 @@ public TableDTO build() {
           Partitioning,
           distributionDTO,
           sortOrderDTOs,
-          indexes);
+          indexes,
+          imported);
     }
 
     /**