[ISSUE-3607] Support to sync the entities

apache · May 30, 2024 · 9587cea · 9587cea
1 parent 844a8e6
commit 9587cea
Show file tree

Hide file tree

Showing 29 changed files with 901 additions and 114 deletions.
diff --git a/api/src/main/java/com/datastrato/gravitino/Schema.java b/api/src/main/java/com/datastrato/gravitino/Schema.java
@@ -33,4 +33,8 @@ default String comment() {
   default Map<String, String> properties() {
     return Collections.emptyMap();
   }
+
+  default boolean imported() {
+    return false;
+  }
 }
diff --git a/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java b/api/src/main/java/com/datastrato/gravitino/messaging/Topic.java
@@ -34,4 +34,8 @@ default String comment() {
   default Map<String, String> properties() {
     return Collections.emptyMap();
   }
+
+  default boolean imported() {
+    return false;
+  }
 }
diff --git a/api/src/main/java/com/datastrato/gravitino/rel/Table.java b/api/src/main/java/com/datastrato/gravitino/rel/Table.java
@@ -81,4 +81,8 @@ default Map<String, String> properties() {
   default SupportsPartitions supportPartitions() throws UnsupportedOperationException {
     throw new UnsupportedOperationException("Table does not support partition operations.");
   }
+
+  default boolean imported() {
+    return false;
+  }
 }
diff --git a/common/src/main/java/com/datastrato/gravitino/dto/MetadataObjectDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/MetadataObjectDTO.java
@@ -0,0 +1,113 @@
+package com.datastrato.gravitino.dto;
+
+import com.datastrato.gravitino.MetadataObject;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+
+import javax.annotation.Nullable;
+
+public class MetadataObjectDTO implements MetadataObject {
+
+    @JsonProperty("fullName")
+    private String fullName;
+
+    @JsonProperty("type")
+    private Type type;
+
+    private String parent;
+    private String name;
+
+    /** Default constructor for Jackson deserialization. */
+    protected MetadataObjectDTO() {}
+
+    /**
+     * Creates a new instance of MetadataObject DTO.
+     *
+     * @param fullName The name of the MetadataObject DTO.
+     * @param type The type of the meta data object.
+     */
+    protected MetadataObjectDTO(String fullName, Type type) {
+        this.type = type;
+        this.fullName = fullName;
+        int index = fullName.lastIndexOf(".");
+
+        if (index == -1) {
+            this.parent = null;
+            this.name = fullName;
+        } else {
+            this.parent = fullName.substring(0, index);
+            this.name = fullName.substring(index + 1);
+        }
+    }
+
+    @Nullable
+    @Override
+    public String parent() {
+        return parent;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public String fullName() {
+        return fullName;
+    }
+
+    @Override
+    public Type type() {
+        return type;
+    }
+
+
+    /** @return the builder for creating a new instance of MetadataObjectDTO. */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /** Builder for {@link MetadataObjectDTO}. */
+    public static class Builder {
+        private String fullName;
+        private Type type;
+
+        /**
+         * Sets the full name of the meta data object.
+         *
+         * @param fullName The full name of the meta data object.
+         * @return The builder instance.
+         */
+        public Builder withFullName(String fullName) {
+            this.fullName = fullName;
+            return this;
+        }
+
+        /**
+         * Sets the type of the meta data object.
+         *
+         * @param type The type of the meta data object.
+         * @return The builder instance.
+         */
+        public Builder withType(Type type) {
+            this.type = type;
+            return this;
+        }
+
+        /**
+         * Builds an instance of MetadataObjectDTO using the builder's properties.
+         *
+         * @return An instance of MetadataObjectDTO.
+         * @throws IllegalArgumentException If the full name or type are not set.
+         */
+        public MetadataObjectDTO build() {
+            Preconditions.checkArgument(
+                    StringUtils.isNotBlank(fullName), "full name cannot be null or empty");
+
+            Preconditions.checkArgument(type != null, "type cannot be null");
+
+            return new MetadataObjectDTO(fullName, type);
+        }
+    }
+}
diff --git a/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/SchemaDTO.java
@@ -21,6 +21,9 @@ public class SchemaDTO implements Schema {
   @JsonProperty("properties")
   private Map<String, String> properties;
 
+  @JsonProperty("imported")
+  private boolean imported;
+
   @JsonProperty("audit")
   private AuditDTO audit;
 
@@ -34,10 +37,11 @@ private SchemaDTO() {}
    * @param properties The properties associated with the schema.
    * @param audit The audit information for the schema.
    */
-  private SchemaDTO(String name, String comment, Map<String, String> properties, AuditDTO audit) {
+  private SchemaDTO(String name, String comment, Map<String, String> properties, boolean imported, AuditDTO audit) {
     this.name = name;
     this.comment = comment;
     this.properties = properties;
+    this.imported = imported;
     this.audit = audit;
   }
 
@@ -59,6 +63,11 @@ public Map<String, String> properties() {
     return properties;
   }
 
+  @Override
+  public boolean imported() {
+    return imported;
+  }
+
   /** @return The audit information for the schema. */
   @Override
   public AuditDTO auditInfo() {
@@ -79,6 +88,8 @@ public static class Builder<S extends Builder> {
     protected Map<String, String> properties;
     /** The audit information for the schema. */
     protected AuditDTO audit;
+    /** The imported status for the schema. */
+    protected boolean imported;
 
     /** Constructs a new Builder instance. */
     private Builder() {}
@@ -127,6 +138,11 @@ public S withAudit(AuditDTO audit) {
       return (S) this;
     }
 
+    public S withImported(boolean imported) {
+      this.imported = imported;
+      return (S) this;
+    }
+
     /**
      * Builds a Schema DTO based on the provided builder parameters.
      *
@@ -137,7 +153,7 @@ public SchemaDTO build() {
       Preconditions.checkArgument(name != null && !name.isEmpty(), "name cannot be null or empty");
       Preconditions.checkArgument(audit != null, "audit cannot be null");
 
-      return new SchemaDTO(name, comment, properties, audit);
+      return new SchemaDTO(name, comment, properties, imported, audit);
     }
   }
 

diff --git a/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/messaging/TopicDTO.java
@@ -15,6 +15,7 @@
 /** Represents a topic DTO (Data Transfer Object). */
 public class TopicDTO implements Topic {
 
+
   /** @return a new builder for constructing a Topic DTO. */
   public static Builder builder() {
     return new Builder();
@@ -29,6 +30,9 @@ public static Builder builder() {
   @JsonProperty("properties")
   private Map<String, String> properties;
 
+  @JsonProperty("imported")
+  private boolean imported;
+
   @JsonProperty("audit")
   private AuditDTO audit;
 
@@ -42,7 +46,7 @@ private TopicDTO() {}
    * @param properties The properties associated with the topic.
    * @param audit The audit information for the topic.
    */
-  private TopicDTO(String name, String comment, Map<String, String> properties, AuditDTO audit) {
+  private TopicDTO(String name, String comment, Map<String, String> properties, boolean imported, AuditDTO audit) {
     this.name = name;
     this.comment = comment;
     this.properties = properties;
@@ -65,6 +69,11 @@ public Map<String, String> properties() {
     return properties;
   }
 
+  @Override
+  public boolean imported() {
+    return imported;
+  }
+
   @Override
   public Audit auditInfo() {
     return audit;
@@ -82,12 +91,13 @@ public boolean equals(Object o) {
     return Objects.equals(name, topicDTO.name)
         && Objects.equals(comment, topicDTO.comment)
         && Objects.equals(properties, topicDTO.properties)
+        && Objects.equals(imported, topicDTO.imported)
         && Objects.equals(audit, topicDTO.audit);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(name, comment, properties, audit);
+    return Objects.hash(name, comment, properties, imported, audit);
   }
 
   /** A builder for constructing a Topic DTO. */
@@ -142,6 +152,11 @@ public Builder withAudit(AuditDTO audit) {
       return this;
     }
 
+    public Builder withImported(boolean imported) {
+      topic.imported = imported;
+      return this;
+    }
+
     /** @return The constructed Topic DTO. */
     public TopicDTO build() {
       return topic;

diff --git a/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/rel/TableDTO.java
@@ -47,6 +47,9 @@ public class TableDTO implements Table {
   @JsonProperty("indexes")
   private IndexDTO[] indexes;
 
+  @JsonProperty("imported")
+  private boolean imported;
+
   private TableDTO() {}
 
   /**
@@ -58,7 +61,8 @@ private TableDTO() {}
    * @param properties The properties associated with the table.
    * @param audit The audit information for the table.
    * @param partitioning The partitioning of the table.
-   * @param indexes Teh indexes of the table.
+   * @param indexes The indexes of the table.
+   * @param imported The table is whether to be imported to Gravitino.
    */
   private TableDTO(
       String name,
@@ -69,7 +73,8 @@ private TableDTO(
       Partitioning[] partitioning,
       DistributionDTO distribution,
       SortOrderDTO[] sortOrderDTOs,
-      IndexDTO[] indexes) {
+      IndexDTO[] indexes,
+      boolean imported) {
     this.name = name;
     this.comment = comment;
     this.columns = columns;
@@ -79,6 +84,7 @@ private TableDTO(
     this.sortOrders = sortOrderDTOs;
     this.partitioning = partitioning;
     this.indexes = indexes;
+    this.imported = imported;
   }
 
   /** @return The name of the table. */
@@ -135,6 +141,11 @@ public Index[] index() {
     return indexes;
   }
 
+  @Override
+  public boolean imported() {
+    return imported;
+  }
+
   /**
    * Creates a new Builder to build a Table DTO.
    *
@@ -169,6 +180,8 @@ public static class Builder<S extends Builder> {
     /** The indexes of the table. */
     protected IndexDTO[] indexes;
 
+    protected boolean imported;
+
     /** Default constructor. */
     private Builder() {}
 
@@ -271,6 +284,11 @@ public S withIndex(IndexDTO[] indexes) {
       return (S) this;
     }
 
+    public S withImported(boolean imported) {
+      this.imported = imported;
+      return (S) this;
+    }
+
     /**
      * Builds a Table DTO based on the provided builder parameters.
      *
@@ -292,7 +310,8 @@ public TableDTO build() {
           Partitioning,
           distributionDTO,
           sortOrderDTOs,
-          indexes);
+          indexes,
+          imported);
     }
 
     /**