OpenLineage java model Generator (apache#18)

OpenLineage model generator
add openapi doc gen

Signed-off-by: Julien Le Dem <[email protected]>

.circleci/config.yml

@@ -0,0 +1,9 @@
+version: 2.1
+orbs:
+  gradle: circleci/[email protected]
+workflows:
+  checkout-build-test:
+    jobs:
+      - gradle/test:
+          app_src_directory: client
+

client/.gitignore

@@ -0,0 +1,12 @@
+# Ignore Gradle project-specific cache directory
+.gradle
+
+.project
+.classpath
+.idea/
+.settings
+
+# Ignore Gradle build output directory
+build
+
+bin

client/build.gradle

@@ -0,0 +1,49 @@
+plugins {
+    id "java"
+    id 'maven-publish'
+    id 'application'
+    id 'idea'
+    id 'eclipse'
+    id "org.openapi.generator" version "5.0.1"
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+ext {
+    jackson_version = "2.11.3"
+    jackson_databind_version = "2.11.3"
+}
+
+configurations {
+    codeGenerator
+}
+
+dependencies {
+    compile "com.fasterxml.jackson.core:jackson-core:$jackson_version"
+    compile "com.fasterxml.jackson.core:jackson-annotations:$jackson_version"
+    compile "com.fasterxml.jackson.core:jackson-databind:$jackson_databind_version"
+    codeGenerator project(':generator')
+    testImplementation 'junit:junit:4.13'
+}
+
+task generateCode(type: JavaExec) {
+    classpath configurations.codeGenerator
+    main = 'io.openlineage.client.Generator'
+}
+
+openApiGenerate {
+// openapi-generator generate -i spec/OpenLineage.yml -g html2 -o ../OpenLineage.github.io/
+    generatorName = "html2"
+    inputSpec = "$rootDir/../spec/OpenLineage.yml".toString()
+    outputDir = "$buildDir/docs".toString()
+}
+
+compileJava.dependsOn tasks.generateCode, tasks.openApiGenerate

client/generator/.classpath

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" output="bin/main" path="src/main/java">
+		<attributes>
+			<attribute name="gradle_scope" value="main"/>
+			<attribute name="gradle_used_by_scope" value="main,test"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="bin/main" path="src/main/resources">
+		<attributes>
+			<attribute name="gradle_scope" value="main"/>
+			<attribute name="gradle_used_by_scope" value="main,test"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="bin/test" path="src/test/java">
+		<attributes>
+			<attribute name="gradle_scope" value="test"/>
+			<attribute name="gradle_used_by_scope" value="test"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="bin/test" path="src/test/resources">
+		<attributes>
+			<attribute name="gradle_scope" value="test"/>
+			<attribute name="gradle_used_by_scope" value="test"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-13/"/>
+	<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
+	<classpathentry kind="output" path="bin/default"/>
+</classpath>

client/generator/.project

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>plugin</name>
+	<comment>Project plugin created by Buildship.</comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
+	</natures>
+</projectDescription>

client/generator/.settings/org.eclipse.buildship.core.prefs

@@ -0,0 +1,2 @@
+connection.project.dir=
+eclipse.preferences.version=1

client/generator/.settings/org.eclipse.jdt.ui.prefs

@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.ui.text.custom_code_templates=

client/generator/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This generated file contains a sample Gradle plugin project to get you started.
+ * For more details take a look at the Writing Custom Plugins chapter in the Gradle
+ * User Manual available at https://docs.gradle.org/6.7/userguide/custom_plugins.html
+ */
+
+plugins {
+    // Apply the Java Gradle plugin development plugin to add support for developing Gradle plugins
+    id 'java'
+}
+
+repositories {
+    // Use JCenter for resolving dependencies.
+    jcenter()
+}
+
+dependencies {
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.11.3'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.11.3'
+    compile group: 'com.squareup', name: 'javapoet', version: '1.13.0'
+    compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25'
+    compile group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25'
+
+    // Use JUnit test framework for unit tests
+    testImplementation 'junit:junit:4.13'
+}
+

client/generator/src/main/java/io/openlineage/client/Generator.java

@@ -0,0 +1,39 @@
+package io.openlineage.client;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class Generator {
+
+  public static void main(String[] args) throws JsonParseException, JsonMappingException, IOException {
+    String baseURL = "https://raw.githubusercontent.com/OpenLineage/OpenLineage/jsonschema/spec/OpenLineage.json";
+    File input = new File("../spec/OpenLineage.json");
+    File output = new File("src/main/java/io/openlineage/client/OpenLineage.java");
+    generate(baseURL, input, output);
+  }
+
+  public static void generate(String baseURL, File input, File output) {
+    if (!output.getParentFile().exists()) {
+      if (!output.getParentFile().mkdirs()) {
+        throw new RuntimeException("can't create output " + output.getAbsolutePath());
+      }
+    }
+    try {
+      ObjectMapper mapper = new ObjectMapper();
+      JsonNode schema = mapper.readValue(input, JsonNode.class);
+      TypeResolver typeResolver = new TypeResolver(schema);
+      try (PrintWriter printWriter = new PrintWriter(output)) {
+        new JavaPoetGenerator(typeResolver, baseURL).generate(printWriter);
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}

client/generator/src/main/java/io/openlineage/client/JavaPoetGenerator.java

@@ -0,0 +1,174 @@
+package io.openlineage.client;
+
+import static io.openlineage.client.TypeResolver.titleCase;
+import static javax.lang.model.element.Modifier.FINAL;
+import static javax.lang.model.element.Modifier.PRIVATE;
+import static javax.lang.model.element.Modifier.PUBLIC;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.lang.model.element.Modifier;
+
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.squareup.javapoet.AnnotationSpec;
+import com.squareup.javapoet.ClassName;
+import com.squareup.javapoet.CodeBlock;
+import com.squareup.javapoet.FieldSpec;
+import com.squareup.javapoet.JavaFile;
+import com.squareup.javapoet.MethodSpec;
+import com.squareup.javapoet.MethodSpec.Builder;
+import com.squareup.javapoet.ParameterSpec;
+import com.squareup.javapoet.ParameterizedTypeName;
+import com.squareup.javapoet.TypeName;
+import com.squareup.javapoet.TypeSpec;
+
+import io.openlineage.client.TypeResolver.ArrayType;
+import io.openlineage.client.TypeResolver.Field;
+import io.openlineage.client.TypeResolver.ObjectType;
+import io.openlineage.client.TypeResolver.PrimitiveType;
+import io.openlineage.client.TypeResolver.Type;
+import io.openlineage.client.TypeResolver.TypeVisitor;
+
+/**
+ * Generates a JavaClass with all the types as inner classes
+ */
+public class JavaPoetGenerator {
+
+  private static final String PACKAGE = "io.openlineage.client";
+  private static final String CONTAINER_CLASS_NAME = "OpenLineage";
+  private static final String CONTAINER_CLASS = PACKAGE + "." + CONTAINER_CLASS_NAME;
+  private final TypeResolver typeResolver;
+  private final String baseURL;
+
+  public JavaPoetGenerator(TypeResolver typeResolver, String baseURL) {
+    this.typeResolver = typeResolver;
+    this.baseURL = baseURL;
+  }
+
+  public void generate(PrintWriter printWriter) throws IOException {
+
+    TypeSpec.Builder builder = TypeSpec.classBuilder(CONTAINER_CLASS_NAME)
+        .addModifiers(Modifier.PUBLIC, Modifier.FINAL);
+    generateTypes(builder);
+    TypeSpec openLineage = builder.build();
+
+    JavaFile javaFile = JavaFile.builder(PACKAGE, openLineage)
+        .build();
+
+    javaFile.writeTo(printWriter);
+  }
+
+  private void generateTypes(TypeSpec.Builder builder) {
+    Collection<ObjectType> types = typeResolver.getTypes();
+    for (ObjectType type : types) {
+
+      if (typeResolver.getBaseTypes().contains(type.getName())) {
+        TypeSpec.Builder interfaceBuilder = TypeSpec.interfaceBuilder(type.getName())
+            .addModifiers(Modifier.STATIC);
+        for (Field f : type.getProperties()) {
+          MethodSpec getter = getter(f)
+              .addModifiers(Modifier.ABSTRACT, Modifier.PUBLIC)
+              .build();
+          interfaceBuilder.addMethod(getter);
+        }
+        TypeSpec intrfc = interfaceBuilder.build();
+        builder.addType(intrfc);
+      } else {
+        TypeSpec.Builder classBuilder = TypeSpec.classBuilder(type.getName())
+            .addModifiers(Modifier.STATIC, Modifier.FINAL);
+        for (String parent : type.getParents()) {
+          classBuilder.addSuperinterface(ClassName.get(CONTAINER_CLASS, parent));
+        }
+        MethodSpec.Builder constructor = MethodSpec.constructorBuilder();
+        constructor.addAnnotation(JsonCreator.class);
+        List<String> fieldNames = new ArrayList<String>();
+        for (Field f : type.getProperties()) {
+          classBuilder.addField(getTypeName(f.getType()), f.getName(), PRIVATE, FINAL);
+          fieldNames.add(f.getName());
+          if (f.getName().equals("_schemaURL")) {
+            String schemaURL = baseURL + "#/definitions/" + type.getName();
+            constructor.addCode("this.$N = $S;\n", f.getName(), schemaURL);
+          } else {
+            constructor.addJavadoc("@param $N $N\n", f.getName(), f.getDescription() == null ? "" : f.getDescription());
+            constructor.addParameter(
+                ParameterSpec.builder(getTypeName(f.getType()), f.getName())
+                .addAnnotation(AnnotationSpec.builder(JsonProperty.class).addMember("value", "$S", f.getName()).build())
+                .build());
+            constructor.addCode("this.$N = $N;\n", f.getName(), f.getName());
+          }
+          MethodSpec getter = getter(f)
+              .addModifiers(Modifier.PUBLIC)
+              .addCode("return $N;", f.getName())
+              .build();
+          classBuilder.addMethod(getter);
+        }
+         // additionalFields
+        if (type.isHasAdditionalProperties()) {
+          String fieldName = "additionalProperties";
+          TypeName additionalPropertiesValueType = type.getAdditionalPropertiesType() == null ? ClassName.get(Object.class) : getTypeName(type.getAdditionalPropertiesType());
+          TypeName additionalPropertiesType = ParameterizedTypeName.get(ClassName.get(Map.class), ClassName.get(String.class), additionalPropertiesValueType);
+          classBuilder.addMethod(MethodSpec
+              .methodBuilder("get" + titleCase(fieldName))
+              .addJavadoc("additional properties")
+              .returns(additionalPropertiesType)
+              .addModifiers(Modifier.PUBLIC)
+              .addCode("return $N;", fieldName)
+              .addAnnotation(AnnotationSpec.builder(JsonAnyGetter.class).build())
+              .build());
+          classBuilder.addField(
+              FieldSpec.builder(additionalPropertiesType, fieldName, PUBLIC, FINAL)
+              .addAnnotation(JsonAnySetter.class)
+              .build());
+
+          constructor.addCode(CodeBlock.builder().addStatement("this.$N = new $T<>();\n", fieldName, HashMap.class).build());
+        }
+        classBuilder.addMethod(constructor.build());
+        TypeSpec clss = classBuilder.build();
+        builder.addType(clss);
+      }
+    }
+
+  }
+
+  private Builder getter(Field f) {
+    Builder builder = MethodSpec
+        .methodBuilder("get" + titleCase(f.getName()))
+        .returns(getTypeName(f.getType()));
+    if (f.getDescription() != null) {
+      builder.addJavadoc("$N", f.getDescription());
+    }
+    return builder;
+  }
+
+  public static TypeName getTypeName(Type type) {
+    return type.accept(new TypeVisitor<TypeName>(){
+
+      @Override
+      public TypeName visit(PrimitiveType primitiveType) {
+        if (primitiveType.getName().equals("string")) {
+          return ClassName.get(String.class);
+        }
+        throw new RuntimeException("Unknown primitive: " + primitiveType.getName());
+      }
+
+      @Override
+      public TypeName visit(ObjectType objectType) {
+        return ClassName.get(CONTAINER_CLASS, objectType.getName());
+      }
+
+      @Override
+      public TypeName visit(ArrayType arrayType) {
+        return ParameterizedTypeName.get(ClassName.get(List.class), getTypeName(arrayType.getItems()));
+      }
+    });
+  }
+}