HBASE-27449 Use xmllint to parse version and revision in release scri…

…pts (#4850)

Also upgrade our Dockerfile for release scripts to use ubuntu 22.04,
maven 3.8.6 and yetus 0.14.1.
Use python3 only.

Signed-off-by: Guanghao Zhang <[email protected]>
Signed-off-by: Nick Dimiduk <[email protected]>

dev-support/create-release/hbase-rm/Dockerfile

@@ -15,11 +15,11 @@
 # limitations under the License.
 #
 
-# Image for building HBase releases. Based on Ubuntu 18.04.
+# Image for building HBase releases. Based on Ubuntu 22.04.
 #
 # Includes:
 # * Java 8
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 
 # Install extra needed repos and refresh.
@@ -28,40 +28,46 @@ FROM ubuntu:18.04
 # the most current package versions (instead of potentially using old versions cached by docker).
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
   && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-    curl='7.58.0-*' \
-    git='1:2.17.1-*' \
-    gnupg='2.2.4-*' \
-    libcurl4-openssl-dev='7.58.0-*' \
-    libxml2-dev='2.9.4+dfsg1-*' \
-    lsof='4.89+dfsg-*' \
+    curl='7.81.0-*' \
+    git='1:2.34.1-*' \
+    gnupg='2.2.27-*' \
+    libcurl4-openssl-dev='7.81.0-*' \
+    libxml2-dev='2.9.13+dfsg-*' \
+    libxml2-utils='2.9.13+dfsg-*' \
+    lsof='4.93.2+dfsg-*' \
     openjdk-8-jdk='8u*' \
-    python-pip='9.0.1-*' \
-    subversion='1.9.7-*' \
-    wget='1.19.4-*' \
+    python3-pip='22.0.2+dfsg-*' \
+    subversion='1.14.1-*' \
+    wget='1.21.2-*' \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
   && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
-  && pip install \
-    python-dateutil==2.8.1
+  && pip3 install \
+    python-dateutil==2.8.2
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# Install mvn 3.6.3.
-ARG MAVEN_VERSION=3.6.3
-ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0
-ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+# Install maven
+ENV MAVEN_VERSION=3.8.6
+ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
+ARG MAVEN_SHA512="f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26"
 RUN mkdir -p /opt/maven \
-  && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
-  && echo "${SHA}  /tmp/apache-maven.tar.gz" | sha512sum -c - \
+  && curl -fsSL -o /tmp/apache-maven.tar.gz "${MAVEN_URL}" \
+  && echo "${MAVEN_SHA512}  /tmp/apache-maven.tar.gz" | sha512sum -c - \
   && tar -xzf /tmp/apache-maven.tar.gz -C /opt/maven --strip-components=1 \
   && rm -f /tmp/apache-maven.tar.gz \
   && ln -s /opt/maven/bin/mvn /usr/bin/mvn
 
 # Install Apache Yetus
-ENV YETUS_VERSION 0.12.0
-RUN curl -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \
-        tar xvz -C /opt
-ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
+ENV YETUS_VERSION 0.14.1
+ARG YETUS_URL="https://archive.apache.org/dist/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz"
+ARG YETUS_SHA512="068c8426a5c7ecc9281b988123d223a3d772fd725365303bde8287ae1515e623764b89f09a4a9cdc01e7f30631814086b71408b625025a51bdb3c58b213b4109"
+RUN mkdir -p /opt/yetus \
+  && curl -fsSL -o /tmp/apache-yetus.tar.gz "${YETUS_URL}" \
+  && echo "${YETUS_SHA512} */tmp/apache-yetus.tar.gz" | sha512sum -c - \
+  && tar -xzf /tmp/apache-yetus.tar.gz -C /opt/yetus --strip-components=1 \
+  && rm -f /tmp/apache-yetus.tar.gz
+ENV YETUS_HOME /opt/yetus
 
 ARG UID
 ARG RM_USER

dev-support/create-release/prepend_changes.py

@@ -24,7 +24,7 @@
   print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
   sys.exit(1)
 
-pattern = re.compile(r'^## Release .+ - Unreleased .+$')
+pattern = re.compile(r'^## Release .+$')
 with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
   line = prev_r.readline()
   while line:

dev-support/create-release/release-build.sh

@@ -149,7 +149,7 @@ if [[ "$1" == "tag" ]]; then
   # check their position when generating release data. We can not put them under the source code
   # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
   # generating src release tarball, we will reset the git repo
-  if [[ $(is_tracked "CHANGES.md") == 0 ]]; then
+  if is_tracked "CHANGES.md"; then
     git add RELEASENOTES.md CHANGES.md
     git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
     cp CHANGES.md ../

dev-support/create-release/release-util.sh

@@ -60,8 +60,11 @@ function read_config {
 }
 
 function parse_version {
-  grep -e '<version>.*</version>' | \
-    head -n 2 | tail -n 1 | cut -d'>' -f2 | cut -d '<' -f1
+  xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" -
+}
+
+function parse_revision {
+  xmllint --xpath "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='revision']/text()" -
 }
 
 function banner {
@@ -135,6 +138,8 @@ function get_api_diff_version {
 # Get all branches that begin with 'branch-', the hbase convention for
 # release branches, sort them and then pop off the most recent.
 function get_release_info {
+  init_xmllint
+
   PROJECT="$(read_config "PROJECT" "$PROJECT")"
   export PROJECT
 
@@ -164,6 +169,14 @@ function get_release_info {
   local version
   version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
     parse_version)"
+  # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+  # If we use ${revision} as placeholder, we need to parse the revision property to
+  # get maven version
+  # shellcheck disable=SC2016
+  if [[ "${version}" == '${revision}' ]]; then
+    version="$(curl -s "$ASF_REPO_WEBUI;a=blob_plain;f=pom.xml;hb=refs/heads/$GIT_BRANCH" |
+      parse_revision)"
+  fi
   log "Current branch VERSION is $version."
 
   NEXT_VERSION="$version"
@@ -344,6 +357,17 @@ function init_locale {
   export LANG="$locale_value"
 }
 
+# Check whether xmllint is available
+function init_xmllint {
+  if ! [ -x "$(command -v xmllint)"  ]; then
+    log "Error: xmllint is not available, we need to use it for parsing pom.xml." >&2
+    log "Ubuntu: apt install libxml2-utils" >&2
+    log "CentOS: yum install xmlstarlet" >&2
+    log "Mac OS: brew install xmlstarlet" >&2
+    exit 1
+  fi
+}
+
 # Initializes JAVA_VERSION to the version of the JVM in use.
 function init_java {
   if [ -z "$JAVA_HOME" ]; then
@@ -355,10 +379,10 @@ function init_java {
 }
 
 function init_python {
-  if ! [ -x "$(command -v python2)"  ]; then
-    error 'python2 needed by yetus. Install or add link? E.g: sudo ln -sf /usr/bin/python2.7 /usr/local/bin/python2'
+  if ! [ -x "$(command -v python3)"  ]; then
+    error 'python3 needed by yetus and api report. Install or add link?'
   fi
-  log "python version: $(python2 --version)"
+  log "python3 version: $(python3 --version)"
 }
 
 # Set MVN
@@ -440,8 +464,8 @@ function git_clone_overwrite {
     log "Clone will be of the gitbox repo for ${PROJECT}."
     if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
       # Ugly!
-      encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''', '')")
-      encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''', '')")
+      encoded_username=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_USERNAME''', ''))")
+      encoded_password=$(python3 -c "from urllib.parse import quote; print(quote('''$ASF_PASSWORD''', ''))")
       GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
     else
       GIT_REPO="https://${asf_repo}"
@@ -533,7 +557,7 @@ function get_jira_name {
 # Update the CHANGES.md
 # DOES NOT DO COMMITS! Caller should do that.
 # requires yetus to have a defined home already.
-# yetus requires python2 to be on the path.
+# yetus requires python3 to be on the path.
 function update_releasenotes {
   local project_dir="$1"
   local jira_fix_version="$2"
@@ -686,28 +710,13 @@ function kick_gpg_agent {
 function maven_set_version { #input: <version_to_set>
   local this_version="$1"
   local use_revision='false'
-  local -a version_splits=()
-  IFS='.' read -ar version_splits <<< "$(maven_get_version)"
-
-  # Do the right thing based on project and release line.
-  if [ "${PROJECT}" = 'hbase' ] ; then
-    if [ "${version_splits[0]}" -le 1 ] ; then
-      use_revision='false'
-    elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -le 4 ] ; then
-      use_revision='false'
-    elif [ "${version_splits[0]}" -eq 2 ] && [ "${version_splits[1]}" -ge 5 ] ; then
-      use_revision='true'
-    elif [ "${version_splits[0]}" -ge 3 ] ; then
-      use_revision='true'
-    fi
-  elif [ "${PROJECT}" = 'hbase-thirdparty' ] ; then
-    use_revision='false'
-  elif [ "${PROJECT}" = 'hbase-connectors' ] ; then
+  local maven_version
+  maven_version="$(parse_version < pom.xml)"
+  # We do not want to expand ${revision} here, see https://maven.apache.org/maven-ci-friendly.html
+  # If we use ${revision} as placeholder, the way to bump maven version will be different
+  # shellcheck disable=SC2016
+  if [[ "${maven_version}" == '${revision}' ]]; then
     use_revision='true'
-  elif [ "${PROJECT}" = 'hbase-filesystem' ] ; then
-    use_revision='false'
-  elif [ "${PROJECT}" = 'hbase-operator-tools' ] ; then
-    use_revision='false'
   fi
 
   if [ "${use_revision}" = 'false' ] ; then