Merge pull request #202 from gabrielenizzoli:gabrielenizzoli/issue201

gabrielenizzoli/issue201

.gitignore

@@ -21,6 +21,7 @@
 
 # other stuff
 .idea
+target/*
 */target/*
 *.iml
 

delta-sharing-core/pom.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>delta-sharing-core</artifactId>
+
+    <parent>
+        <groupId>com.spark-engine</groupId>
+        <artifactId>spark-engine-parent</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <name>Spark Engine Delta Sharing Core Module</name>
+    <url>https://www.spark-engine.com</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.delta</groupId>
+            <artifactId>delta-core_${scala.binary.version}</artifactId>
+        </dependency>
+
+        <!-- test -->
+        <dependency>
+            <groupId>com.spark-engine</groupId>
+            <artifactId>spark-test</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>compile</phase>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

delta-sharing-core/src/main/java/sparkengine/delta/sharing/model/Table.java

@@ -0,0 +1,11 @@
+package sparkengine.delta.sharing.model;
+
+import lombok.Value;
+
+@Value
+public class Table {
+
+    TableName tableName;
+    TableMetadata tableMetadata;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/model/TableMetadata.java

@@ -0,0 +1,13 @@
+package sparkengine.delta.sharing.model;
+
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+public class TableMetadata {
+
+    @Nonnull
+    String location;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/model/TableName.java

@@ -0,0 +1,17 @@
+package sparkengine.delta.sharing.model;
+
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+public class TableName {
+
+    @Nonnull
+    String share;
+    @Nonnull
+    String schema;
+    @Nonnull
+    String table;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/File.java

@@ -0,0 +1,27 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Map;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = File.FileBuilder.class)
+public class File {
+
+    @Nonnull
+    String url;
+    @Nonnull
+    String id;
+    @Nonnull
+    Map<String, String> partitionValues;
+    @Nonnull
+    long size;
+    @Nullable
+    String stats;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Format.java

@@ -0,0 +1,22 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Format.FormatBuilder.class)
+public class Format {
+
+    enum Provider {
+        parquet
+    }
+
+    @Nonnull
+    @Builder.Default
+    Provider provider = Provider.parquet;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Metadata.java

@@ -0,0 +1,29 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.List;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Metadata.MetadataBuilder.class)
+public class Metadata {
+
+    @Nonnull
+    String id;
+    @Nullable
+    String name;
+    @Nullable
+    String description;
+    @Nonnull
+    Format format;
+    @Nonnull
+    String schemaString;
+    @Nonnull
+    List<String> partitionColumns;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Protocol.java

@@ -0,0 +1,17 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Protocol.ProtocolBuilder.class)
+public class Protocol {
+
+    @Nonnull
+    int minReaderVersion;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Schema.java

@@ -0,0 +1,19 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Schema.SchemaBuilder.class)
+public class Schema {
+
+    @Nonnull
+    String share;
+    @Nonnull
+    String name;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Schemas.java

@@ -0,0 +1,21 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.List;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Schemas.SchemasBuilder.class)
+public class Schemas {
+
+    @Nonnull
+    List<Schema> items;
+    @Nullable
+    String nextPageToken;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Share.java

@@ -0,0 +1,17 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Share.ShareBuilder.class)
+public class Share {
+
+    @Nonnull
+    String name;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Shares.java

@@ -0,0 +1,21 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.List;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Shares.SharesBuilder.class)
+public class Shares {
+
+    @Nonnull
+    List<Share> items;
+    @Nullable
+    String nextPageToken;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Table.java

@@ -0,0 +1,21 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Table.TableBuilder.class)
+public class Table {
+
+    @Nonnull
+    String share;
+    @Nonnull
+    String schema;
+    @Nonnull
+    String name;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/TableQuery.java

@@ -0,0 +1,20 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nullable;
+import java.util.List;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = TableQuery.TableQueryBuilder.class)
+public class TableQuery {
+
+    @Nullable
+    List<String> predicateHints;
+    @Nullable
+    Long limitHint;
+
+}

delta-sharing-core/src/main/java/sparkengine/delta/sharing/protocol/v1/Tables.java

@@ -0,0 +1,21 @@
+package sparkengine.delta.sharing.protocol.v1;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.Builder;
+import lombok.Value;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.List;
+
+@Value
+@Builder(setterPrefix = "with")
+@JsonDeserialize(builder = Tables.TablesBuilder.class)
+public class Tables {
+
+    @Nonnull
+    List<Table> items;
+    @Nullable
+    String nextPageToken;
+
+}