HBASE-28679 Upgrade yetus to a newer version (#6012)

Signed-off-by: Nick Dimiduk <[email protected]>
(cherry picked from commit 17ce7c3)

dev-support/Jenkinsfile

@@ -31,7 +31,7 @@ pipeline {
     disableConcurrentBuilds()
   }
   environment {
-    YETUS_RELEASE = '0.12.0'
+    YETUS_RELEASE = '0.15.0'
     // where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
     OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
     OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
@@ -42,12 +42,12 @@ pipeline {
     PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
     PERSONALITY_FILE = 'tools/personality.sh'
     // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
-    AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
-    WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+    AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+    BLANKS_IGNORE_LIST = '.*/generated/.*'
     // output from surefire; sadly the archive function in yetus only works on file names.
     ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
     // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
-    TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+    TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
     EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
       // TODO does hadoopcheck need to be jdk specific?
     SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'

dev-support/Jenkinsfile_GitHub

@@ -36,14 +36,16 @@ pipeline {
         YETUS_REL = 'yetus'
         DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
         YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
-        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION = 'rel/0.12.0'
+        YETUS_VERSION = '0.15.0'
         GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
         JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
+        // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
+        AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+        BLANKS_IGNORE_LIST = '.*/generated/.*'
         // output from surefire; sadly the archive function in yetus only works on file names.
         ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
         // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
-        TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+        TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
         EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
         // set build parallel
         BUILD_THREAD = 4
@@ -101,11 +103,11 @@ pipeline {
                             checkout scm
                         }
                         dir("${YETUSDIR}") {
-                            checkout([
-                              $class           : 'GitSCM',
-                              branches         : [[name: "${YETUS_VERSION}"]],
-                              userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
-                            )
+                            sh'''#!/usr/bin/env bash
+                              wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+                              tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+                              rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+                            '''
                         }
                         dir("${WORKDIR}") {
                             withCredentials([
@@ -228,11 +230,11 @@ pipeline {
                             checkout scm
                         }
                         dir("${YETUSDIR}") {
-                            checkout([
-                              $class           : 'GitSCM',
-                              branches         : [[name: "${YETUS_VERSION}"]],
-                              userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
-                            )
+                            sh'''#!/usr/bin/env bash
+                              wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+                              tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+                              rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+                            '''
                         }
                         dir("${WORKDIR}") {
                             withCredentials([
@@ -365,11 +367,11 @@ pipeline {
                             checkout scm
                         }
                         dir("${YETUSDIR}") {
-                            checkout([
-                              $class           : 'GitSCM',
-                              branches         : [[name: "${YETUS_VERSION}"]],
-                              userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
-                            )
+                            sh'''#!/usr/bin/env bash
+                              wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+                              tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+                              rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+                            '''
                         }
                         dir("${WORKDIR}") {
                             withCredentials([

dev-support/docker/Dockerfile

@@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
     ruby=1:3.0* \
     ruby-dev=1:3.0* \
     shellcheck='0.8.0-*' \
+    libxml2-dev='2.9.13+dfsg-*' \
+    libxml2-utils='2.9.13+dfsg-*' \
     && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \

dev-support/hbase_nightly_yetus.sh

@@ -20,7 +20,7 @@ declare -i missing_env=0
 # Validate params
 for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \
                     "OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \
-                    "WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
+                    "BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
                     "USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
   if [ -z "${!required_env}" ]; then
     echo "[ERROR] Required environment variable '${required_env}' is not set."
@@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
@@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then
 fi
 
 if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
-  YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
   TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
 else
-  YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py" "${YETUS_ARGS[@]}")
-  TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh"
+  TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh"
 fi
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"

dev-support/jenkins_precommit_github_yetus.sh

@@ -41,6 +41,8 @@ declare -a required_envs=(
   "SOURCEDIR"
   "TESTS_FILTER"
   "YETUSDIR"
+  "AUTHOR_IGNORE_LIST"
+  "BLANKS_IGNORE_LIST"
 )
 # Validate params
 for required_env in "${required_envs[@]}"; do
@@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then
 fi
 
 # TODO (HBASE-23900): cannot assume test-patch runs directly from sources
-TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+TESTPATCHBIN="${YETUSDIR}/bin/test-patch"
 
 # this must be clean for every run
 rm -rf "${PATCHDIR}"
@@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
 YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
 YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
 # enable writing back to Github
-YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
-YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-write-comment")
 # auto-kill any surefire stragglers during unit test runs
 YETUS_ARGS+=("--reapermode=kill")
 # set relatively high limits for ASF machines
@@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker")
 YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 YETUS_ARGS+=("--mvn-custom-repos")
 YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
-YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
-YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
+YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*")
 YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
 YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
 YETUS_ARGS+=("--quick-hadoopcheck")