[#16] feat(schema) add JSON serialize and deserialize support for sch…

…ema (#20) ### What changes were proposed in this pull request? This PR adds JSON serde support for schema system. ### Why are the changes needed? The adds of JSON serde support will help to support REST API for Unified Catalog. Fix: #16 ### Does this PR introduce _any_ user-facing change? NA ### How was this patch tested? And new UTs.
apache · May 23, 2023 · 491746d · 491746d
1 parent 74d2420
commit 491746d
Show file tree

Hide file tree

Showing 13 changed files with 582 additions and 1 deletion.
diff --git a/core/build.gradle.kts b/core/build.gradle.kts
@@ -15,6 +15,7 @@ dependencies {
   implementation(libs.jackson.databind)
   implementation(libs.jackson.annotations)
   implementation(libs.jackson.datatype.jdk8)
+  implementation(libs.jackson.datatype.jsr310)
   implementation(libs.guava)
 
   compileOnly(libs.lombok)

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/AuditInfo.java b/core/src/main/java/com/datastrato/unified_catalog/schema/AuditInfo.java
@@ -9,9 +9,11 @@
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public final class AuditInfo implements Entity {
   public static final Field CREATOR =
       Field.required("creator", String.class, "The name of user who creates the entity");

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/Column.java b/core/src/main/java/com/datastrato/unified_catalog/schema/Column.java
@@ -1,16 +1,21 @@
 package com.datastrato.unified_catalog.schema;
 
+import com.datastrato.unified_catalog.schema.json.JsonUtils;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import io.substrait.type.Type;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public final class Column implements Entity, Auditable {
 
   public static final Field ID =
@@ -47,6 +52,8 @@ public final class Column implements Entity, Auditable {
   private String name;
 
   @JsonProperty("type")
+  @JsonSerialize(using = JsonUtils.TypeSerializer.class)
+  @JsonDeserialize(using = JsonUtils.TypeDeserializer.class)
   private Type type;
 
   @Nullable

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/Lakehouse.java b/core/src/main/java/com/datastrato/unified_catalog/schema/Lakehouse.java
@@ -7,9 +7,11 @@
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public class Lakehouse implements Entity, Auditable {
 
   public static final Field ID =

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/SchemaVersion.java b/core/src/main/java/com/datastrato/unified_catalog/schema/SchemaVersion.java
@@ -1,11 +1,36 @@
 package com.datastrato.unified_catalog.schema;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonFormat(shape = JsonFormat.Shape.OBJECT)
 public enum SchemaVersion {
   V_0_1(0, 1);
 
+  @JsonProperty("major_version")
   public final int majorVersion;
+
+  @JsonProperty("minor_version")
   public final int minorVersion;
 
+  @JsonCreator
+  public static SchemaVersion forValues(
+      @JsonProperty("major_version") int majorVersion,
+      @JsonProperty("minor_version") int minorVersion) {
+    for (SchemaVersion schemaVersion : SchemaVersion.values()) {
+      if (schemaVersion.majorVersion == majorVersion
+          && schemaVersion.minorVersion == minorVersion) {
+        return schemaVersion;
+      }
+    }
+
+    throw new IllegalArgumentException(
+        String.format(
+            "No schema version found for major version %d and minor version %d",
+            majorVersion, minorVersion));
+  }
+
   SchemaVersion(int majorVersion, int minorVersion) {
     this.majorVersion = majorVersion;
     this.minorVersion = minorVersion;

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/Table.java b/core/src/main/java/com/datastrato/unified_catalog/schema/Table.java
@@ -5,11 +5,13 @@
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public class Table implements Entity, Auditable, hasExtraInfo {
-  enum TableType {
+  public enum TableType {
     VIRTUAL("VIRTUAL"),
     VIEW("VIEW"),
     EXTERNAL("EXTERNAL"),

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/Tenant.java b/core/src/main/java/com/datastrato/unified_catalog/schema/Tenant.java
@@ -7,9 +7,11 @@
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public final class Tenant implements Entity, Auditable {
 
   public static final Field ID =

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/VirtualTableInfo.java b/core/src/main/java/com/datastrato/unified_catalog/schema/VirtualTableInfo.java
@@ -6,9 +6,11 @@
 import java.util.Map;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public final class VirtualTableInfo implements hasExtraInfo.ExtraInfo {
 
   public static final Field CONNECTION_ID =
@@ -22,6 +24,12 @@ public final class VirtualTableInfo implements hasExtraInfo.ExtraInfo {
   @JsonProperty("identifier")
   private final List<String> identifier;
 
+  // Only for Jackson deserialization
+  private VirtualTableInfo() {
+    this.connectionId = null;
+    this.identifier = null;
+  }
+
   public VirtualTableInfo(Integer connectionId, List<String> identifier) {
     this.connectionId = connectionId;
     this.identifier = identifier;

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/Zone.java b/core/src/main/java/com/datastrato/unified_catalog/schema/Zone.java
@@ -7,9 +7,11 @@
 import javax.annotation.Nullable;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 @Getter
 @EqualsAndHashCode
+@ToString
 public class Zone implements Entity, Auditable {
 
   public static final Field ID =

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/hasExtraInfo.java b/core/src/main/java/com/datastrato/unified_catalog/schema/hasExtraInfo.java
@@ -1,7 +1,13 @@
 package com.datastrato.unified_catalog.schema;
 
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
 /** Interface for entities that have extra info. */
 public interface hasExtraInfo {
+
+  @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
+  @JsonSubTypes({@JsonSubTypes.Type(value = VirtualTableInfo.class, name = "VIRTUAL")})
   interface ExtraInfo extends Entity {}
 
   /**

diff --git a/core/src/main/java/com/datastrato/unified_catalog/schema/json/JsonUtils.java b/core/src/main/java/com/datastrato/unified_catalog/schema/json/JsonUtils.java
@@ -0,0 +1,57 @@
+package com.datastrato.unified_catalog.schema.json;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.*;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.substrait.type.StringTypeVisitor;
+import io.substrait.type.Type;
+import io.substrait.type.parser.ParseToPojo;
+import io.substrait.type.parser.TypeStringParser;
+import java.io.IOException;
+
+public class JsonUtils {
+  private static ObjectMapper mapper = null;
+
+  public static ObjectMapper objectMapper() {
+    if (mapper == null) {
+      synchronized (JsonUtils.class) {
+        if (mapper == null) {
+          mapper =
+              new ObjectMapper()
+                  .registerModule(new JavaTimeModule())
+                  .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+        }
+      }
+    }
+
+    return mapper;
+  }
+
+  public static class TypeSerializer extends JsonSerializer<io.substrait.type.Type> {
+    private final StringTypeVisitor visitor = new StringTypeVisitor();
+
+    @Override
+    public void serialize(Type value, JsonGenerator gen, SerializerProvider serializers)
+        throws IOException {
+      try {
+        gen.writeString(value.accept(visitor));
+      } catch (Exception e) {
+        throw new IOException("Unable to serialize type " + value, e);
+      }
+    }
+  }
+
+  public static class TypeDeserializer extends JsonDeserializer<io.substrait.type.Type> {
+
+    @Override
+    public Type deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+      String s = p.getValueAsString();
+      try {
+        return TypeStringParser.parse(s, ParseToPojo::type);
+      } catch (Exception e) {
+        throw new IOException("Unable to parse string " + s.replace("\n", " \\n"), e);
+      }
+    }
+  }
+}