From dc3ee9462aa0bb83132de54833633a244be72a39 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 18 Oct 2023 17:28:46 +0800 Subject: [PATCH] support Paimon create table --- extensions/spark/kyuubi-spark-authz/pom.xml | 6 ++ ...imonCatalogRangerSparkExtensionSuite.scala | 83 +++++++++++++++++++ .../org/apache/kyuubi/tags/PaimonTest.java | 29 +++++++ pom.xml | 7 ++ 4 files changed, 125 insertions(+) create mode 100644 extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala create mode 100644 kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml b/extensions/spark/kyuubi-spark-authz/pom.xml index 97145e51485..0b28942bcda 100644 --- a/extensions/spark/kyuubi-spark-authz/pom.xml +++ b/extensions/spark/kyuubi-spark-authz/pom.xml @@ -323,6 +323,12 @@ scala-collection-compat_${scala.binary.version} test + + + org.apache.paimon + paimon-spark-${spark.binary.version} + test + diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala new file mode 100644 index 00000000000..13ff54f6481 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.plugin.spark.authz.ranger + +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.tags.PaimonTest +import org.apache.kyuubi.util.AssertionUtils._ + +/** + * Tests for RangerSparkExtensionSuite on Paimon + */ +@PaimonTest +class PaimonCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { + override protected val catalogImpl: String = "hive" + private def isSupportedVersion = !isSparkV35OrGreater + + val catalogV2 = "paimon_catalog" + val namespace1 = "paimon_ns" + val table1 = "table1" + + override def withFixture(test: NoArgTest): Outcome = { + assume(isSupportedVersion) + test() + } + + override def beforeAll(): Unit = { + if (isSupportedVersion) { + spark.conf.set(s"spark.sql.catalog.$catalogV2", "org.apache.paimon.spark.SparkCatalog") + spark.conf.set( + s"spark.sql.catalog.$catalogV2.warehouse", + Utils.createTempDir("iceberg-hadoop").toString) + super.beforeAll() + } + + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) + } + + override def afterAll(): Unit = { + if (isSupportedVersion) { + super.afterAll() + spark.sessionState.catalog.reset() + spark.sessionState.conf.clear() + } + } + + test("CreateTable") { + withCleanTmpResources(Seq((s"$catalogV2.$namespace1.$table1", "table"))) { + val createTable = + s""" + |CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1 + |(id int, name string, city string) + |USING paimon + |OPTIONS ( + | primaryKey = 'id' + |) + |""".stripMargin + + interceptContains[AccessControlException] { + doAs(someone, sql(createTable)) + }(s"does not have [create] privilege on [$namespace1/$table1]") + doAs(admin, createTable) + } + } +} diff --git a/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java new file mode 100644 index 00000000000..e3e807584e7 --- /dev/null +++ b/kyuubi-util-scala/src/test/java/org/apache/kyuubi/tags/PaimonTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface PaimonTest {} diff --git a/pom.xml b/pom.xml index 00ee354a19e..fbe96789425 100644 --- a/pom.xml +++ b/pom.xml @@ -184,6 +184,7 @@ --> 4.1.93.Final 0.12.0 + 0.5.0-incubating 1.10.1 6.0.0 0.16.0 @@ -1484,6 +1485,12 @@ hudi-spark${hudi.spark.binary.version}-bundle_${scala.binary.version} ${hudi.version} + + + org.apache.paimon + paimon-spark-${spark.binary.version} + ${paimon.version} +