Skip to content

Commit

Permalink
[apache#3885] feat(catalog-lakehouse-paimon): Add a basic code skelet…
Browse files Browse the repository at this point in the history
…on for Paimon catalog
  • Loading branch information
caican committed Jun 19, 2024
1 parent 2d13784 commit 99fbdde
Show file tree
Hide file tree
Showing 14 changed files with 464 additions and 0 deletions.
1 change: 1 addition & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,7 @@ tasks {
dependsOn(
":catalogs:catalog-hive:copyLibAndConfig",
":catalogs:catalog-lakehouse-iceberg:copyLibAndConfig",
":catalogs:catalog-lakehouse-paimon:copyLibAndConfig",
":catalogs:catalog-jdbc-doris:copyLibAndConfig",
":catalogs:catalog-jdbc-mysql:copyLibAndConfig",
":catalogs:catalog-jdbc-postgresql:copyLibAndConfig",
Expand Down
2 changes: 2 additions & 0 deletions catalogs/bundled-catalog/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ dependencies {
implementation(project(":catalogs:catalog-jdbc-mysql"))
implementation(project(":catalogs:catalog-jdbc-postgresql"))
implementation(project(":catalogs:catalog-lakehouse-iceberg"))
implementation(project(":catalogs:catalog-lakehouse-paimon"))
implementation(project(":core"))
implementation(libs.slf4j.api)
}
Expand Down Expand Up @@ -80,6 +81,7 @@ tasks.jar {
tasks.compileJava {
dependsOn(":catalogs:catalog-jdbc-postgresql:runtimeJars")
dependsOn(":catalogs:catalog-lakehouse-iceberg:runtimeJars")
dependsOn(":catalogs:catalog-lakehouse-paimon:runtimeJars")
dependsOn(":catalogs:catalog-jdbc-mysql:runtimeJars")
dependsOn(":catalogs:catalog-hive:runtimeJars")
dependsOn(":catalogs:catalog-hadoop:runtimeJars")
Expand Down
110 changes: 110 additions & 0 deletions catalogs/catalog-lakehouse-paimon/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
description = "catalog-lakehouse-paimon"

plugins {
`maven-publish`
id("java")
id("idea")
}

dependencies {
implementation(project(":api"))
implementation(project(":common"))
implementation(project(":core"))
implementation(libs.bundles.paimon)
implementation(libs.guava)
implementation(libs.hadoop2.common) {
exclude("com.github.spotbugs")
}
implementation(libs.hadoop2.hdfs)
implementation(libs.hive2.exec) {
artifact {
classifier = "core"
}
exclude("com.google.code.findbugs", "jsr305")
exclude("com.google.protobuf")
exclude("org.apache.avro")
exclude("org.apache.calcite")
exclude("org.apache.calcite.avatica")
exclude("org.apache.curator")
exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager")
exclude("org.apache.logging.log4j")
exclude("org.apache.zookeeper")
exclude("org.eclipse.jetty.aggregate", "jetty-all")
exclude("org.eclipse.jetty.orbit", "javax.servlet")
exclude("org.openjdk.jol")
exclude("org.pentaho")
exclude("org.slf4j")
}
implementation(libs.hive2.metastore) {
exclude("co.cask.tephra")
exclude("com.github.spotbugs")
exclude("com.google.code.findbugs", "jsr305")
exclude("com.tdunning", "json")
exclude("javax.transaction", "transaction-api")
exclude("org.apache.avro", "avro")
exclude("org.apache.hbase")
exclude("org.apache.hadoop", "hadoop-yarn-api")
exclude("org.apache.hadoop", "hadoop-yarn-server-applicationhistoryservice")
exclude("org.apache.hadoop", "hadoop-yarn-server-common")
exclude("org.apache.hadoop", "hadoop-yarn-server-resourcemanager")
exclude("org.apache.hadoop", "hadoop-yarn-server-web-proxy")
exclude("org.apache.logging.log4j")
exclude("org.apache.parquet", "parquet-hadoop-bundle")
exclude("org.apache.zookeeper")
exclude("org.eclipse.jetty.aggregate", "jetty-all")
exclude("org.eclipse.jetty.orbit", "javax.servlet")
exclude("org.pentaho") // missing dependency
exclude("org.slf4j", "slf4j-log4j12")
exclude("com.zaxxer", "HikariCP")
exclude("com.sun.jersey", "jersey-server")
}

annotationProcessor(libs.lombok)
compileOnly(libs.lombok)
}

tasks {
val runtimeJars by registering(Copy::class) {
from(configurations.runtimeClasspath)
into("build/libs")
}

val copyCatalogLibs by registering(Copy::class) {
dependsOn("jar", "runtimeJars")
from("build/libs")
into("$rootDir/distribution/package/catalogs/lakehouse-paimon/libs")
}

val copyCatalogConfig by registering(Copy::class) {
from("src/main/resources")
into("$rootDir/distribution/package/catalogs/lakehouse-paimon/conf")

include("lakehouse-paimon.conf")
include("core-site.xml.template")
include("hdfs-site.xml.template")

rename { original ->
if (original.endsWith(".template")) {
original.replace(".template", "")
} else {
original
}
}

exclude { details ->
details.file.isDirectory()
}
}

register("copyLibAndConfig", Copy::class) {
dependsOn(copyCatalogLibs, copyCatalogConfig)
}
}

tasks.getByName("generateMetadataFileForMavenJavaPublication") {
dependsOn("runtimeJars")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
package com.datastrato.gravitino.catalog.lakehouse.paimon;

import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.connector.BaseCatalog;
import com.datastrato.gravitino.connector.CatalogOperations;
import com.datastrato.gravitino.connector.PropertiesMetadata;
import com.datastrato.gravitino.connector.capability.Capability;
import java.util.Map;

/** Implementation of {@link Catalog} that represents a Paimon catalog in Gravitino. */
public class PaimonCatalog extends BaseCatalog<PaimonCatalog> {

/** @return The short name of the catalog. */
@Override
public String shortName() {
return "lakehouse-paimon";
}

/**
* Creates a new instance of {@link PaimonCatalogOperations} with the provided configuration.
*
* @param config The configuration map for the Paimon catalog operations.
* @return A new instance of {@link PaimonCatalogOperations}.
*/
@Override
protected CatalogOperations newOps(Map<String, String> config) {
return new PaimonCatalogOperations();
}

@Override
public Capability newCapability() {
return new PaimonCatalogCapability();
}

@Override
public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationException {
throw new UnsupportedOperationException(
"The catalog does not support table properties metadata");
}

@Override
public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException {
throw new UnsupportedOperationException(
"The catalog does not support catalog properties metadata");
}

@Override
public PropertiesMetadata schemaPropertiesMetadata() throws UnsupportedOperationException {
throw new UnsupportedOperationException(
"The catalog does not support schema properties metadata");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
package com.datastrato.gravitino.catalog.lakehouse.paimon;

import com.datastrato.gravitino.connector.capability.Capability;

public class PaimonCatalogCapability implements Capability {}
Loading

0 comments on commit 99fbdde

Please sign in to comment.