HBASE-27448 Add an admin method to get replication enabled state

dev-support/Dockerfile

@@ -20,16 +20,14 @@
 #
 # Specifically, it's used for the flaky test reporting job defined in
 # dev-support/flaky-tests/flaky-reporting.Jenkinsfile
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 COPY . /hbase/dev-support
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
     && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-      curl='7.58.0-*' \
-      python2.7='2.7.17-*' \
-      python-pip='9.0.1-*' \
-      python-setuptools='39.0.1-*' \
+      curl='7.81.0-*' \
+      python3-pip='22.0.2+dfsg-*' \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-RUN pip install -r /hbase/dev-support/python-requirements.txt
+RUN pip3 install -r /hbase/dev-support/python-requirements.txt

dev-support/Jenkinsfile

@@ -267,14 +267,14 @@ pipeline {
               if [ -d "${OUTPUT_DIR}/branch-site" ]; then
                 echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
                 rm -rf "${OUTPUT_DIR}/branch-site"
-                python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
+                python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
               else
                 echo "No branch-site, skipping"
               fi
               if [ -d "${OUTPUT_DIR}/patch-site" ]; then
                 echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
                 rm -rf "${OUTPUT_DIR}/patch-site"
-                python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
+                python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
               else
                 echo "No patch-site, skipping"
               fi
@@ -384,7 +384,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -493,7 +493,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -604,7 +604,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -773,7 +773,7 @@ pipeline {
                 if [ -f "${SRC_TAR}" ]; then
                   echo "Remove ${SRC_TAR} for saving space"
                   rm -rf "${SRC_TAR}"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
                 else
                   echo "No hbase-src.tar.gz, skipping"
                 fi

dev-support/Jenkinsfile_GitHub

@@ -143,14 +143,14 @@ pipeline {
                             if [ -d "${PATCHDIR}/branch-site" ]; then
                               echo "Remove ${PATCHDIR}/branch-site for saving space"
                               rm -rf "${PATCHDIR}/branch-site"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
                             else
                               echo "No branch-site, skipping"
                             fi
                             if [ -d "${PATCHDIR}/patch-site" ]; then
                               echo "Remove ${PATCHDIR}/patch-site for saving space"
                               rm -rf "${PATCHDIR}/patch-site"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
                             else
                               echo "No patch-site, skipping"
                             fi
@@ -282,7 +282,7 @@ pipeline {
                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
                               rm -rf "${PATCHDIR}/test_logs.zip"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
                             else
                               echo "No test_logs.zip, skipping"
                             fi
@@ -414,7 +414,7 @@ pipeline {
                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
                               rm -rf "${PATCHDIR}/test_logs.zip"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
                             else
                               echo "No test_logs.zip, skipping"
                             fi

dev-support/checkcompatibility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -41,7 +41,9 @@
 import shutil
 import subprocess
 import sys
-import urllib2
+import urllib.request
+import urllib.error
+import urllib.parse
 from collections import namedtuple
 try:
     import argparse
@@ -55,11 +57,11 @@
 
 
 def check_output(*popenargs, **kwargs):
-    """ Run command with arguments and return its output as a byte string.
-    Backported from Python 2.7 as it's implemented as pure python on stdlib.
-    >>> check_output(['/usr/bin/python', '--version'])
-    Python 2.6.2 """
-    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    """ Run command with arguments and return its output as a byte string. """
+    process = subprocess.Popen(stdout=subprocess.PIPE,
+                               universal_newlines=True,
+                               *popenargs,
+                               **kwargs)
     output, _ = process.communicate()
     retcode = process.poll()
     if retcode:
@@ -69,7 +71,7 @@ def check_output(*popenargs, **kwargs):
         error = subprocess.CalledProcessError(retcode, cmd)
         error.output = output
         raise error
-    return output
+    return output.strip()
 
 
 def get_repo_dir():
@@ -161,7 +163,7 @@ def checkout_java_acc(force):
     url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz"
     scratch_dir = get_scratch_dir()
     path = os.path.join(scratch_dir, os.path.basename(url))
-    jacc = urllib2.urlopen(url)
+    jacc = urllib.request.urlopen(url)
     with open(path, 'wb') as w:
         w.write(jacc.read())
 
@@ -196,8 +198,8 @@ def ascii_encode_dict(data):
     """ Iterate through a dictionary of data and convert all unicode to ascii.
     This method was taken from
     stackoverflow.com/questions/9590382/forcing-python-json-module-to-work-with-ascii """
-    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
-    return dict(map(ascii_encode, pair) for pair in data.items())
+    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, str) else x
+    return dict(list(map(ascii_encode, pair)) for pair in list(data.items()))
 
 
 def process_json(path):
@@ -229,8 +231,8 @@ def compare_results(tool_results, known_issues, compare_warnings):
     unexpected_issues = [unexpected_issue(check=check,  issue_type=issue_type,
                                       known_count=known_count,
                                       observed_count=tool_results[check][issue_type])
-                     for check, known_issue_counts in known_issues.items()
-                        for issue_type, known_count in known_issue_counts.items()
+                     for check, known_issue_counts in list(known_issues.items())
+                        for issue_type, known_count in list(known_issue_counts.items())
                            if compare_tool_results_count(tool_results, check, issue_type, known_count)]
 
     if not compare_warnings:
@@ -309,14 +311,14 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annot
         logging.info("Annotations are: %s", annotations)
         annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
         logging.info("Annotations path: %s", annotations_path)
-        with file(annotations_path, "w") as f:
+        with open(annotations_path, "w") as f:
             f.write('\n'.join(annotations))
         args.extend(["-annotations-list", annotations_path])
 
     if skip_annotations is not None:
         skip_annotations_path = os.path.join(
             get_scratch_dir(), "skip_annotations.txt")
-        with file(skip_annotations_path, "w") as f:
+        with open(skip_annotations_path, "w") as f:
             f.write('\n'.join(skip_annotations))
         args.extend(["-skip-annotations-list", skip_annotations_path])
 

dev-support/checkstyle_report.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -29,8 +29,8 @@
 from collections import defaultdict
 
 if len(sys.argv) != 3 :
-  print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
-  exit(1)
+  print("usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0])
+  sys.exit(1)
 
 def path_key(x):
   path = x.attrib['name']
@@ -40,41 +40,41 @@ def error_name(x):
   error_class = x.attrib['source']
   return error_class[error_class.rfind(".") + 1:]
 
-def print_row(path, error, master_errors, patch_errors):
-    print '%s\t%s\t%s\t%s' % (path,error, master_errors,patch_errors)
+def print_row(path, err, master_errors, patch_errors):
+  print('%s\t%s\t%s\t%s' % (path, err, master_errors, patch_errors))
 
 master = etree.parse(sys.argv[1])
 patch = etree.parse(sys.argv[2])
 
 master_dict = defaultdict(int)
 ret_value = 0
 
-for child in master.getroot().getchildren():
+for child in list(master.getroot()):
     if child.tag != 'file':
-        continue
+      continue
     file = path_key(child)
-    for error_tag in child.getchildren():
-        error = error_name(error_tag)
-        if (file, error) in master_dict:
-            master_dict[(file, error)] += 1
-        else:
-            master_dict[(file, error)] = 1
+    for error_tag in list(child):
+      error = error_name(error_tag)
+      if (file, error) in master_dict:
+        master_dict[(file, error)] += 1
+      else:
+        master_dict[(file, error)] = 1
 
-for child in patch.getroot().getchildren():
-    if child.tag != 'file':
-        continue
-    temp_dict = defaultdict(int)
-    for error_tag in child.getchildren():
-        error = error_name(error_tag)
-        if error in temp_dict:
-            temp_dict[error] += 1
-        else:
-            temp_dict[error] = 1
+for child in list(patch.getroot()):
+  if child.tag != 'file':
+    continue
+  temp_dict = defaultdict(int)
+  for error_tag in list(child):
+    error = error_name(error_tag)
+    if error in temp_dict:
+      temp_dict[error] += 1
+    else:
+      temp_dict[error] = 1
 
-    file = path_key(child)
-    for error, count in temp_dict.iteritems():
-        if count > master_dict[(file, error)]:
-            print_row(file, error, master_dict[(file, error)], count)
-            ret_value = 1
+  file = path_key(child)
+  for error, count in temp_dict.items():
+    if count > master_dict[(file, error)]:
+      print_row(file, error, master_dict[(file, error)], count)
+      ret_value = 1
 
 sys.exit(ret_value)

dev-support/create-release/hbase-rm/Dockerfile

@@ -15,11 +15,11 @@
 # limitations under the License.
 #
 
-# Image for building HBase releases. Based on Ubuntu 18.04.
+# Image for building HBase releases. Based on Ubuntu 22.04.
 #
 # Includes:
 # * Java 8
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 
 # Install extra needed repos and refresh.
@@ -28,40 +28,46 @@ FROM ubuntu:18.04
 # the most current package versions (instead of potentially using old versions cached by docker).
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
   && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-    curl='7.58.0-*' \
-    git='1:2.17.1-*' \
-    gnupg='2.2.4-*' \
-    libcurl4-openssl-dev='7.58.0-*' \
-    libxml2-dev='2.9.4+dfsg1-*' \
-    lsof='4.89+dfsg-*' \
+    curl='7.81.0-*' \
+    git='1:2.34.1-*' \
+    gnupg='2.2.27-*' \
+    libcurl4-openssl-dev='7.81.0-*' \
+    libxml2-dev='2.9.13+dfsg-*' \
+    libxml2-utils='2.9.13+dfsg-*' \
+    lsof='4.93.2+dfsg-*' \
     openjdk-8-jdk='8u*' \
-    python-pip='9.0.1-*' \
-    subversion='1.9.7-*' \
-    wget='1.19.4-*' \
+    python3-pip='22.0.2+dfsg-*' \
+    subversion='1.14.1-*' \
+    wget='1.21.2-*' \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
   && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \
-  && pip install \
-    python-dateutil==2.8.1
+  && pip3 install \
+    python-dateutil==2.8.2
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# Install mvn 3.6.3.
-ARG MAVEN_VERSION=3.6.3
-ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0
-ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries
+# Install maven
+ENV MAVEN_VERSION=3.8.6
+ARG MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
+ARG MAVEN_SHA512="f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26"
 RUN mkdir -p /opt/maven \
-  && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
-  && echo "${SHA}  /tmp/apache-maven.tar.gz" | sha512sum -c - \
+  && curl -fsSL -o /tmp/apache-maven.tar.gz "${MAVEN_URL}" \
+  && echo "${MAVEN_SHA512}  /tmp/apache-maven.tar.gz" | sha512sum -c - \
   && tar -xzf /tmp/apache-maven.tar.gz -C /opt/maven --strip-components=1 \
   && rm -f /tmp/apache-maven.tar.gz \
   && ln -s /opt/maven/bin/mvn /usr/bin/mvn
 
 # Install Apache Yetus
-ENV YETUS_VERSION 0.12.0
-RUN curl -L "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz" | \
-        tar xvz -C /opt
-ENV YETUS_HOME /opt/apache-yetus-${YETUS_VERSION}
+ENV YETUS_VERSION 0.14.1
+ARG YETUS_URL="https://archive.apache.org/dist/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz"
+ARG YETUS_SHA512="068c8426a5c7ecc9281b988123d223a3d772fd725365303bde8287ae1515e623764b89f09a4a9cdc01e7f30631814086b71408b625025a51bdb3c58b213b4109"
+RUN mkdir -p /opt/yetus \
+  && curl -fsSL -o /tmp/apache-yetus.tar.gz "${YETUS_URL}" \
+  && echo "${YETUS_SHA512} */tmp/apache-yetus.tar.gz" | sha512sum -c - \
+  && tar -xzf /tmp/apache-yetus.tar.gz -C /opt/yetus --strip-components=1 \
+  && rm -f /tmp/apache-yetus.tar.gz
+ENV YETUS_HOME /opt/yetus
 
 ARG UID
 ARG RM_USER

dev-support/create-release/prepend_changes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -22,9 +22,9 @@
 
 if len(sys.argv) != 3:
   print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
-  exit(1)
+  sys.exit(1)
 
-pattern = re.compile(r'^## Release .+ - Unreleased .+$')
+pattern = re.compile(r'^## Release .+$')
 with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
   line = prev_r.readline()
   while line:

dev-support/create-release/prepend_releasenotes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -22,7 +22,7 @@
 
 if len(sys.argv) != 3:
   print("usage: %s <NEW_RELEASENOTES.md> <PREV_RELEASENOTES.md>" % sys.argv[0])
-  exit(1)
+  sys.exit(1)
 
 pattern = re.compile(r'^# .+ Release Notes$')
 with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w: