[KYUUBI #6577] Add spark sql engine plugin module and define a sql stringify plugin

extensions/spark/kyuubi-spark-engine-plugin/pom.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.kyuubi</groupId>
+        <artifactId>kyuubi-parent</artifactId>
+        <version>1.10.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>kyuubi-spark-engine-plugin</artifactId>
+    <packaging>jar</packaging>
+    <name>Kyuubi Project Spark Engine Plugin</name>
+    <url>https://kyuubi.apache.org/</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

extensions/spark/kyuubi-spark-engine-plugin/src/main/java/org/apache/kyuubi/engine/spark/plugin/PlanOnlyExecutor.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.engine.spark.plugin;
+
+public interface PlanOnlyExecutor {
+
+  String mode();
+
+  default String execute(SparkPlans plans) {
+    return execute(plans, "plain");
+  }
+
+  /**
+   * execute the parsed plan with plan only mode
+   *
+   * @param plans the spark plans
+   * @param style the style of the result (plain or json)
+   * @return the result in the specified style
+   */
+  String execute(SparkPlans plans, String style);
+}

extensions/spark/kyuubi-spark-engine-plugin/src/main/java/org/apache/kyuubi/engine/spark/plugin/SparkPlans.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.engine.spark.plugin;
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
+import org.apache.spark.sql.execution.SparkPlan;
+
+public interface SparkPlans {
+
+    LogicalPlan parsedPlan();
+    LogicalPlan analyzedPlan();
+    LogicalPlan optimizedPlan();
+    SparkPlan sparkPlan();
+    SparkPlan executedPlan();
+
+}

externals/kyuubi-spark-sql-engine/pom.xml

@@ -49,6 +49,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-spark-engine-plugin</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-core</artifactId>

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala

@@ -17,6 +17,8 @@
 
 package org.apache.kyuubi.engine.spark.operation
 
+import java.util.Locale
+
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import org.apache.spark.kyuubi.SparkUtilsHelper
@@ -28,6 +30,7 @@ import org.apache.spark.sql.types.StructType
 import org.apache.kyuubi.KyuubiSQLException
 import org.apache.kyuubi.config.KyuubiConf.{LINEAGE_PARSER_PLUGIN_PROVIDER, OPERATION_PLAN_ONLY_EXCLUDES, OPERATION_PLAN_ONLY_OUT_STYLE}
 import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.getSessionConf
+import org.apache.kyuubi.engine.spark.operation.planonly.{PlanOnlyExecutors, SparkPlansImpl}
 import org.apache.kyuubi.operation.{AnalyzeMode, ArrayFetchIterator, ExecutionMode, IterableFetchIterator, JsonStyle, LineageMode, OperationHandle, OptimizeMode, OptimizeWithStatsMode, ParseMode, PhysicalMode, PlainStyle, PlanOnlyMode, PlanOnlyStyle, UnknownMode, UnknownStyle}
 import org.apache.kyuubi.operation.PlanOnlyMode.{notSupportedModeError, unknownModeError}
 import org.apache.kyuubi.operation.PlanOnlyStyle.{notSupportedStyleError, unknownStyleError}
@@ -150,6 +153,10 @@ class PlanOnlyStatement(
       case LineageMode =>
         val result = parseLineage(spark, plan)
         iter = new IterableFetchIterator(Seq(Row(result)))
+      case PlanOnlyExecutors(executor) =>
+        val sparkPlans = SparkPlansImpl(spark, plan)
+        val result = executor.execute(sparkPlans, style.name.toLowerCase(Locale.ROOT))
+        iter = new IterableFetchIterator(Seq(Row(result)))
       case UnknownMode => throw unknownModeError(mode)
       case _ =>
         throw KyuubiSQLException(s"The operation mode $mode" +

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/planonly/PlanOnlyExecutors.scala

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.engine.spark.operation.planonly
+
+import java.util.Locale
+
+import org.apache.kyuubi.engine.spark.plugin.PlanOnlyExecutor
+import org.apache.kyuubi.operation.PlanOnlyMode
+import org.apache.kyuubi.util.reflect.ReflectUtils
+
+object PlanOnlyExecutors {
+
+  private lazy val executors: Map[String, PlanOnlyExecutor] = {
+    ReflectUtils.loadFromServiceLoader[PlanOnlyExecutor]()
+      .map(e => e.mode().toLowerCase(Locale.ROOT) -> e).toMap
+  }
+
+  def getExecutor(mode: PlanOnlyMode): Option[PlanOnlyExecutor] = {
+    executors.get(mode.name.toLowerCase(Locale.ROOT))
+  }
+
+  def unapply(mode: PlanOnlyMode): Option[PlanOnlyExecutor] = getExecutor(mode)
+
+}

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/planonly/SparkPlansImpl.scala

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.engine.spark.operation.planonly
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.execution.{CommandExecutionMode, QueryExecution, SparkPlan}
+
+import org.apache.kyuubi.engine.spark.plugin.SparkPlans
+
+class SparkPlansImpl(queryExecution: QueryExecution) extends SparkPlans {
+  assert(
+    queryExecution.mode == CommandExecutionMode.SKIP,
+    "Query execution mode must be SKIP to use PlanOnlyMode.")
+
+  override def parsedPlan: LogicalPlan = queryExecution.logical
+
+  override def analyzedPlan: LogicalPlan = queryExecution.analyzed
+
+  override def optimizedPlan: LogicalPlan = queryExecution.optimizedPlan
+
+  override def sparkPlan: SparkPlan = queryExecution.sparkPlan
+
+  override def executedPlan: SparkPlan = queryExecution.executedPlan
+
+}
+
+object SparkPlansImpl {
+
+  def apply(queryExecution: QueryExecution): SparkPlansImpl = {
+    new SparkPlansImpl(queryExecution)
+  }
+
+  def apply(spark: SparkSession, parsedPlan: LogicalPlan): SparkPlansImpl = {
+    val qe = spark.sessionState.executePlan(parsedPlan, CommandExecutionMode.SKIP)
+    apply(qe)
+  }
+}

pom.xml

@@ -60,6 +60,7 @@
         <module>extensions/spark/kyuubi-spark-connector-common</module>
         <module>extensions/spark/kyuubi-spark-connector-tpcds</module>
         <module>extensions/spark/kyuubi-spark-connector-tpch</module>
+        <module>extensions/spark/kyuubi-spark-engine-plugin</module>
         <module>extensions/spark/kyuubi-spark-lineage</module>
         <module>externals/kyuubi-chat-engine</module>
         <module>externals/kyuubi-download</module>