Skip to content

Add metrics for batch pending max elapse time #22282

Add metrics for batch pending max elapse time

Add metrics for batch pending max elapse time #22282

Workflow file for this run

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: CI
on:
push:
branches:
- master
- branch-*
pull_request:
branches:
- master
- branch-*
concurrency:
group: test-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
MAVEN_ARGS: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded,gen-policy -Dmaven.plugin.download.cache.path=/tmp/engine-archives
KUBERNETES_VERSION: v1.30.0
MINIKUBE_VERSION: v1.33.1
jobs:
default:
name: Kyuubi and Spark Test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
max-parallel: 6
matrix:
java:
- 8
- 17
spark:
- '3.3'
- '3.4'
- '3.5'
spark-archive: [""]
exclude-tags: [""]
comment: ["normal"]
include:
- java: 21
spark: '4.0'
spark-archive: '-Pscala-2.13'
exclude-tags: ''
comment: 'normal'
- java: 8
spark: '3.5'
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.2-binary'
- java: 8
spark: '3.5'
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.3-binary'
- java: 8
spark: '3.5'
spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3 -Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.4-binary'
- java: 17
spark: '3.5'
spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.0-preview2 -Dspark.archive.name=spark-4.0.0-preview2-bin-hadoop3.tgz'
exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-4.0-binary'
env:
SPARK_LOCAL_IP: localhost
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.9'
- name: Build and test Kyuubi and Spark with maven w/o linters
run: |
if [[ "${{ matrix.java }}" == "8" && "${{ matrix.spark }}" == "3.5" && "${{ matrix.spark-archive }}" == "" ]]; then
export MAVEN_ARGS="${MAVEN_ARGS} -Pcodecov -Dmaven.javadoc.skip=false -Dmaven.scaladoc.skip=false"
fi
TEST_MODULES="dev/kyuubi-codecov"
./build/mvn clean install -pl ${TEST_MODULES} -am \
-Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }} ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }}
- name: Code coverage
if: |
matrix.java == 8 &&
matrix.spark == '3.5' &&
matrix.spark-archive == ''
uses: codecov/codecov-action@v3
with:
verbose: true
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-java-${{ matrix.java }}-spark-${{ matrix.spark }}-${{ matrix.comment }}
path: |
**/target/unit-tests.log
**/kyuubi-spark-sql-engine.log*
**/kyuubi-spark-batch-submit.log*
scala-test:
name: Scala Test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
scala:
- '2.13'
java:
- '8'
spark:
- '3.5'
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Build on Scala ${{ matrix.scala }}
run: |
TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it"
./build/mvn clean install -pl ${TEST_MODULES} -am \
-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.SparkLocalClusterTest \
-Pscala-${{ matrix.scala }} -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }}
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-scala-${{ matrix.scala }}-java-${{ matrix.java }}-spark-${{ matrix.spark }}
path: |
**/target/unit-tests.log
**/kyuubi-spark-sql-engine.log*
**/kyuubi-spark-batch-submit.log*
**/kyuubi-jdbc-engine.log*
**/kyuubi-hive-sql-engine.log*
spark-connector-cross-version-test:
name: Spark Connector Cross Version Test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
max-parallel: 1
matrix:
java:
- 17
scala:
- '2.12'
- '2.13'
spark-compile:
- '3.5'
spark-runtime:
- '3.4'
- '3.3'
comment: [ "normal" ]
include:
- java: 17
scala: "2.13"
spark-compile: "3.5"
spark-runtime: "4.0"
comment: "normal"
env:
SPARK_LOCAL_IP: localhost
TEST_MODULES: "extensions/spark/kyuubi-spark-connector-hive,\
extensions/spark/kyuubi-spark-connector-tpcds,\
extensions/spark/kyuubi-spark-connector-tpch"
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Build Kyuubi Spark Connectors with Spark-${{ matrix.spark-compile }}
run: |
./build/mvn clean install -pl ${TEST_MODULES} -am -Pjava-${{ matrix.java }} \
-Pscala-${{ matrix.scala }} -Pspark-${{ matrix.spark-compile }} -DskipTests
- name: Test Kyuubi Spark Connectors with Spark-${{ matrix.spark-runtime }}
run: |
./build/mvn test -pl ${TEST_MODULES} -Pjava-${{ matrix.java }} \
-Pscala-${{ matrix.scala }} -Pspark-${{ matrix.spark-runtime }} -Pcross-version-test
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: "unit-tests-log-java-${{ matrix.java }}-scala-${{ matrix.scala }}\
-spark-compile-${{ matrix.spark-compile }}-spark-runtime-${{ matrix.spark-runtime }}\
-${{ matrix.comment }}"
path: |
**/target/unit-tests.log
flink-it:
name: Flink Test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
max-parallel: 2
matrix:
java:
- 8
flink:
- '1.17'
- '1.18'
- '1.19'
- '1.20'
flink-archive: [ "" ]
comment: [ "normal" ]
include:
- java: 8
flink: '1.20'
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2 -Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
comment: 'verify-on-flink-1.17-binary'
- java: 8
flink: '1.20'
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1 -Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
comment: 'verify-on-flink-1.18-binary'
- java: 8
flink: '1.20'
flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1 -Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
comment: 'verify-on-flink-1.19-binary'
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Build Flink with maven w/o linters
run: |
TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it"
./build/mvn -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} -am clean install -DskipTests
- name: Test Flink
if: matrix.flink-archive == ''
run: |
TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it"
./build/mvn -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} test
- name: Cross-version test Flink
if: matrix.flink-archive != ''
run: |
IT_FLINK=`echo "${{ matrix.flink-archive }}" | grep -E 'flink\-([0-9]+\.[0-9]+.[0-9]+)\-bin' -o | grep -E '[0-9]+\.[0-9]+' -o`
IT_MODULE="integration-tests/kyuubi-flink-it"
./build/mvn -pl ${IT_MODULE} -Pflink-${IT_FLINK} ${{ matrix.flink-archive }} test
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-java-${{ matrix.java }}-flink-${{ matrix.flink }}-${{ matrix.comment }}
path: |
**/target/unit-tests.log
**/kyuubi-flink-sql-engine.log*
hive-it:
name: Hive Test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
max-parallel: 1
matrix:
java:
- 8
hive-archive: [ "" ]
comment: [ "normal" ]
include:
- java: 8
hive-archive: '-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10 -Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
comment: 'verify-on-hive-2.3-binary'
- java: 8
hive-archive: '-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release -Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz'
comment: 'verify-on-hive-2.1-cdh6-binary'
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Build and test Hive with maven w/o linters
run: |
TEST_MODULES="externals/kyuubi-hive-sql-engine,integration-tests/kyuubi-hive-it"
./build/mvn ${{ matrix.hive-archive }} -pl ${TEST_MODULES} -am clean install -DskipTests
# unpack avatica-1.8.0.jar and delete Jackson classes to workaround CALCITE-1224
if [[ "${{ matrix.hive-archive }}" == *apache-hive-2.3.10-bin.tar.gz* ]]; then
HIVE_2310_LIB="$PWD/externals/kyuubi-download/target/apache-hive-2.3.10-bin/lib"
mkdir avatica-1.8.0-unpack
cd avatica-1.8.0-unpack
jar xf $HIVE_2310_LIB/avatica-1.8.0.jar
rm -rf com/fasterxml/jackson org/slf4j
jar cf avatica-1.8.0-patched.jar .
rm $HIVE_2310_LIB/avatica-1.8.0.jar
mv avatica-1.8.0-patched.jar $HIVE_2310_LIB/
cd ..
rm -rf avatica-1.8.0-unpack
fi
./build/mvn ${{ matrix.hive-archive }} -pl ${TEST_MODULES} test
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-java-${{ matrix.java }}-hive-${{ matrix.comment }}
path: |
**/target/unit-tests.log
**/kyuubi-hive-sql-engine.log*
jdbc-trino-tpc-it:
name: JDBC Trino TPC Tests
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
java: [ 8 ]
comment: [ "normal" ]
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Build and test JDBC and Trino engines, TPC-H/TPC-DS with maven w/o linters
run: |
JDBC_TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it"
TRINO_TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it"
TPC_TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch"
./build/mvn -pl ${JDBC_TEST_MODULES},${TRINO_TEST_MODULES},${TPC_TEST_MODULES} -am clean install -DskipTests
./build/mvn -pl ${JDBC_TEST_MODULES} test
./build/mvn -pl ${TRINO_TEST_MODULES} test
./build/mvn -pl ${TPC_TEST_MODULES} test \
-Dmaven.plugin.scalatest.exclude.tags='' \
-Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-java-${{ matrix.java }}-hive-${{ matrix.comment }}
path: |
**/target/unit-tests.log
**/kyuubi-jdbc-engine.log*
**/kyuubi-trino-engine.log*
kyuubi-on-k8s-it:
name: Kyuubi Server On Kubernetes Integration Test
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
# https://github.com/docker/build-push-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build Kyuubi Docker Image
uses: docker/build-push-action@v6
with:
# passthrough CI into build container
build-args: |
CI=${CI}
MVN_ARG=--flink-provided --hive-provided
context: .
file: build/Dockerfile.CI
load: true
tags: apache/kyuubi:latest
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Setup Minikube
run: |
# https://minikube.sigs.k8s.io/docs/start/
curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube
minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
# https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
minikube image load apache/kyuubi:latest
# pre-install spark into minikube
docker pull apache/spark:3.5.2
minikube image load apache/spark:3.5.2
- name: kubectl pre-check
run: |
kubectl get nodes
kubectl get serviceaccount
kubectl create serviceaccount kyuubi
kubectl create clusterrolebinding kyuubi-role --clusterrole=edit --serviceaccount=default:kyuubi
kubectl get serviceaccount
- name: start kyuubi
run: kubectl apply -f integration-tests/kyuubi-kubernetes-it/src/test/resources/kyuubi-server.yaml
- name: kyuubi pod check
run: |
kubectl get pods
kubectl describe pods kyuubi-test
- name: integration tests
run: >-
./build/mvn clean install
-pl integration-tests/kyuubi-kubernetes-it -am
-Pkubernetes-it
-Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment,org.apache.kyuubi.kubernetes.test.KubernetesUtilsTest
- name: Cat kyuubi server log
if: failure()
run: kubectl logs kyuubi-test
- name: Copy spark engine log from kyuubi pod
if: failure()
run: kubectl cp kyuubi-test:/opt/kyuubi/work ./target/work
- name: Cat spark driver log
if: failure()
run: |
kubectl get pods
kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {}
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-kyuubi-on-k8s-it
path: |
**/target/unit-tests.log
**/target/work/**
spark-on-k8s-it:
name: Spark Engine On Kubernetes Integration Test
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- name: Free up disk space
run: ./.github/scripts/free_disk_space.sh
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: Setup JDK 11
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 11
cache: 'maven'
- name: Setup Minikube
run: |
# https://minikube.sigs.k8s.io/docs/start/
curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube
minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
# in case: https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac
- name: Create Service Account
run: |
kubectl create serviceaccount spark
kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default
kubectl get serviceaccount
# TODO Support more Spark version
- name: integration tests
run: >-
./build/mvn clean install
-Pflink-provided,hive-provided
-Pkubernetes-it
-Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.spark
- name: Print Driver Pod logs
if: failure()
run: |
kubectl get pods
kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I {} kubectl logs {}
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-spark-on-k8s-it
path: |
**/target/unit-tests.log
**/kyuubi-spark-sql-engine.log*
**/kyuubi-spark-batch-submit.log*
zookeeper-it:
name: Zookeeper Integration Test
runs-on: ubuntu-24.04
strategy:
fail-fast: false
max-parallel: 1
matrix:
java: [ 8 ]
zookeeper: ["3.4", "3.5", "3.6", "3.7" ]
comment: [ "normal" ]
steps:
- uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: 'maven'
- name: Setup Maven
uses: ./.github/actions/setup-maven
- name: Cache Engine Archives
uses: ./.github/actions/cache-engine-archives
- name: zookeeper integration tests
run: |
export KYUUBI_IT_ZOOKEEPER_VERSION=${{ matrix.zookeeper }}
TEST_MODULES="integration-tests/kyuubi-zookeeper-it"
./build/mvn -pl ${TEST_MODULES} -am clean install -DskipTests
./build/mvn -pl ${TEST_MODULES} test
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: unit-tests-log-java-${{ matrix.java }}-zookeeper-${{ matrix.comment }}
path: |
**/target/unit-tests.log