From 265dcb47372623e4493f8c4a34cc3d4b16e3176e Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sat, 22 Jun 2024 10:22:56 +0800 Subject: [PATCH] HBASE-28679 Upgrade yetus to a newer version (#6012) Signed-off-by: Nick Dimiduk (cherry picked from commit 17ce7c3b4c5e58af1f70d17e62940c9dd6fb4660) --- dev-support/Jenkinsfile | 8 ++-- dev-support/Jenkinsfile_GitHub | 38 ++++++++++--------- dev-support/docker/Dockerfile | 2 + dev-support/hbase_nightly_yetus.sh | 10 ++--- dev-support/jenkins_precommit_github_yetus.sh | 13 ++++--- 5 files changed, 38 insertions(+), 33 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 19256ccb9b12..e514b31702cc 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -31,7 +31,7 @@ pipeline { disableConcurrentBuilds() } environment { - YETUS_RELEASE = '0.12.0' + YETUS_RELEASE = '0.15.0' // where we'll write everything from different steps. Need a copy here so the final step can check for success/failure. OUTPUT_DIR_RELATIVE_GENERAL = 'output-general' OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2' @@ -42,12 +42,12 @@ pipeline { PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh' PERSONALITY_FILE = 'tools/personality.sh' // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag. - AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh' - WHITESPACE_IGNORE_LIST = '.*/generated/.*' + AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc' + BLANKS_IGNORE_LIST = '.*/generated/.*' // output from surefire; sadly the archive function in yetus only works on file names. ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump' // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure. - TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite' + TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop' EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes" // TODO does hadoopcheck need to be jdk specific? SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check' diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index 44443cd697ee..a2b953464c04 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -36,14 +36,16 @@ pipeline { YETUS_REL = 'yetus' DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile" YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh" - // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' - YETUS_VERSION = 'rel/0.12.0' + YETUS_VERSION = '0.15.0' GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit' JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit' + // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag. + AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc' + BLANKS_IGNORE_LIST = '.*/generated/.*' // output from surefire; sadly the archive function in yetus only works on file names. ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump' // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure. - TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite' + TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop' EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes" // set build parallel BUILD_THREAD = 4 @@ -101,11 +103,11 @@ pipeline { checkout scm } dir("${YETUSDIR}") { - checkout([ - $class : 'GitSCM', - branches : [[name: "${YETUS_VERSION}"]], - userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]] - ) + sh'''#!/usr/bin/env bash + wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \ + tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \ + rm apache-yetus-${YETUS_VERSION}-bin.tar.gz + ''' } dir("${WORKDIR}") { withCredentials([ @@ -228,11 +230,11 @@ pipeline { checkout scm } dir("${YETUSDIR}") { - checkout([ - $class : 'GitSCM', - branches : [[name: "${YETUS_VERSION}"]], - userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]] - ) + sh'''#!/usr/bin/env bash + wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \ + tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \ + rm apache-yetus-${YETUS_VERSION}-bin.tar.gz + ''' } dir("${WORKDIR}") { withCredentials([ @@ -365,11 +367,11 @@ pipeline { checkout scm } dir("${YETUSDIR}") { - checkout([ - $class : 'GitSCM', - branches : [[name: "${YETUS_VERSION}"]], - userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]] - ) + sh'''#!/usr/bin/env bash + wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \ + tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \ + rm apache-yetus-${YETUS_VERSION}-bin.tar.gz + ''' } dir("${WORKDIR}") { withCredentials([ diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1f841d1d8672..bc8dee088ae3 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \ ruby=1:3.0* \ ruby-dev=1:3.0* \ shellcheck='0.8.0-*' \ + libxml2-dev='2.9.13+dfsg-*' \ + libxml2-utils='2.9.13+dfsg-*' \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh index 4c671dcfef65..3423a7e73c55 100755 --- a/dev-support/hbase_nightly_yetus.sh +++ b/dev-support/hbase_nightly_yetus.sh @@ -20,7 +20,7 @@ declare -i missing_env=0 # Validate params for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \ "OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \ - "WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \ + "BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \ "USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do if [ -z "${!required_env}" ]; then echo "[ERROR] Required environment variable '${required_env}' is not set." @@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}") YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}") YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}") YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}") -YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}") -YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}") +YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}") +YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}") YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}") YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}") YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}") @@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then fi if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then - YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}") TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch" else - YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py" "${YETUS_ARGS[@]}") - TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh" + TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh" fi echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index d22351bcf5a8..95ddc0ed4aad 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -41,6 +41,8 @@ declare -a required_envs=( "SOURCEDIR" "TESTS_FILTER" "YETUSDIR" + "AUTHOR_IGNORE_LIST" + "BLANKS_IGNORE_LIST" ) # Validate params for required_env in "${required_envs[@]}"; do @@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then fi # TODO (HBASE-23900): cannot assume test-patch runs directly from sources -TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh" +TESTPATCHBIN="${YETUSDIR}/bin/test-patch" # this must be clean for every run rm -rf "${PATCHDIR}" @@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt") YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") # enable writing back to Github -YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}") -YETUS_ARGS+=("--github-user=${GITHUB_USER}") +YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}") +YETUS_ARGS+=("--github-write-comment") # auto-kill any surefire stragglers during unit test runs YETUS_ARGS+=("--reapermode=kill") # set relatively high limits for ASF machines @@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker") YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") YETUS_ARGS+=("--mvn-custom-repos") YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}") -YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*") -YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*") +YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}") +YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}") +YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*") YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}") YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh") YETUS_ARGS+=("--quick-hadoopcheck")