HBASE-27450 Update all our python scripts to use python3 (apache#4851)

Signed-off-by: Guanghao Zhang <[email protected]>
vinayakphegde · Oct 28, 2022 · cdabfd3 · cdabfd3
1 parent d800536
commit cdabfd3
Show file tree

Hide file tree

Showing 12 changed files with 101 additions and 412 deletions.
diff --git a/dev-support/Dockerfile b/dev-support/Dockerfile
@@ -20,16 +20,14 @@
 #
 # Specifically, it's used for the flaky test reporting job defined in
 # dev-support/flaky-tests/flaky-reporting.Jenkinsfile
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
 COPY . /hbase/dev-support
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -qq -y update \
     && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install --no-install-recommends \
-      curl='7.58.0-*' \
-      python2.7='2.7.17-*' \
-      python-pip='9.0.1-*' \
-      python-setuptools='39.0.1-*' \
+      curl='7.81.0-*' \
+      python3-pip='22.0.2+dfsg-*' \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-RUN pip install -r /hbase/dev-support/python-requirements.txt
+RUN pip3 install -r /hbase/dev-support/python-requirements.txt
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
@@ -267,14 +267,14 @@ pipeline {
               if [ -d "${OUTPUT_DIR}/branch-site" ]; then
                 echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
                 rm -rf "${OUTPUT_DIR}/branch-site"
-                python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
+                python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
               else
                 echo "No branch-site, skipping"
               fi
               if [ -d "${OUTPUT_DIR}/patch-site" ]; then
                 echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
                 rm -rf "${OUTPUT_DIR}/patch-site"
-                python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
+                python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
               else
                 echo "No patch-site, skipping"
               fi
@@ -384,7 +384,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -493,7 +493,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -604,7 +604,7 @@ pipeline {
                 if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
                   echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
                   rm -rf "${OUTPUT_DIR}/test_logs.zip"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
                 else
                   echo "No test_logs.zip, skipping"
                 fi
@@ -773,7 +773,7 @@ pipeline {
                 if [ -f "${SRC_TAR}" ]; then
                   echo "Remove ${SRC_TAR} for saving space"
                   rm -rf "${SRC_TAR}"
-                  python2 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
+                  python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/output-srctarball" > "${WORKSPACE}/output-srctarball/hbase-src.html"
                 else
                   echo "No hbase-src.tar.gz, skipping"
                 fi

diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
@@ -143,14 +143,14 @@ pipeline {
                             if [ -d "${PATCHDIR}/branch-site" ]; then
                               echo "Remove ${PATCHDIR}/branch-site for saving space"
                               rm -rf "${PATCHDIR}/branch-site"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
                             else
                               echo "No branch-site, skipping"
                             fi
                             if [ -d "${PATCHDIR}/patch-site" ]; then
                               echo "Remove ${PATCHDIR}/patch-site for saving space"
                               rm -rf "${PATCHDIR}/patch-site"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
                             else
                               echo "No patch-site, skipping"
                             fi
@@ -282,7 +282,7 @@ pipeline {
                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
                               rm -rf "${PATCHDIR}/test_logs.zip"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
                             else
                               echo "No test_logs.zip, skipping"
                             fi
@@ -414,7 +414,7 @@ pipeline {
                             if [ -f "${PATCHDIR}/test_logs.zip" ]; then
                               echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
                               rm -rf "${PATCHDIR}/test_logs.zip"
-                              python2 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+                              python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
                             else
                               echo "No test_logs.zip, skipping"
                             fi

diff --git a/dev-support/checkcompatibility.py b/dev-support/checkcompatibility.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -41,7 +41,9 @@
 import shutil
 import subprocess
 import sys
-import urllib2
+import urllib.request
+import urllib.error
+import urllib.parse
 from collections import namedtuple
 try:
     import argparse
@@ -55,11 +57,11 @@
 
 
 def check_output(*popenargs, **kwargs):
-    """ Run command with arguments and return its output as a byte string.
-    Backported from Python 2.7 as it's implemented as pure python on stdlib.
-    >>> check_output(['/usr/bin/python', '--version'])
-    Python 2.6.2 """
-    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    """ Run command with arguments and return its output as a byte string. """
+    process = subprocess.Popen(stdout=subprocess.PIPE,
+                               universal_newlines=True,
+                               *popenargs,
+                               **kwargs)
     output, _ = process.communicate()
     retcode = process.poll()
     if retcode:
@@ -69,7 +71,7 @@ def check_output(*popenargs, **kwargs):
         error = subprocess.CalledProcessError(retcode, cmd)
         error.output = output
         raise error
-    return output
+    return output.strip()
 
 
 def get_repo_dir():
@@ -161,7 +163,7 @@ def checkout_java_acc(force):
     url = "https://github.com/lvc/japi-compliance-checker/archive/2.4.tar.gz"
     scratch_dir = get_scratch_dir()
     path = os.path.join(scratch_dir, os.path.basename(url))
-    jacc = urllib2.urlopen(url)
+    jacc = urllib.request.urlopen(url)
     with open(path, 'wb') as w:
         w.write(jacc.read())
 
@@ -196,8 +198,8 @@ def ascii_encode_dict(data):
     """ Iterate through a dictionary of data and convert all unicode to ascii.
     This method was taken from
     stackoverflow.com/questions/9590382/forcing-python-json-module-to-work-with-ascii """
-    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, unicode) else x
-    return dict(map(ascii_encode, pair) for pair in data.items())
+    ascii_encode = lambda x: x.encode('ascii') if isinstance(x, str) else x
+    return dict(list(map(ascii_encode, pair)) for pair in list(data.items()))
 
 
 def process_json(path):
@@ -229,8 +231,8 @@ def compare_results(tool_results, known_issues, compare_warnings):
     unexpected_issues = [unexpected_issue(check=check,  issue_type=issue_type,
                                       known_count=known_count,
                                       observed_count=tool_results[check][issue_type])
-                     for check, known_issue_counts in known_issues.items()
-                        for issue_type, known_count in known_issue_counts.items()
+                     for check, known_issue_counts in list(known_issues.items())
+                        for issue_type, known_count in list(known_issue_counts.items())
                            if compare_tool_results_count(tool_results, check, issue_type, known_count)]
 
     if not compare_warnings:
@@ -309,14 +311,14 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations, skip_annot
         logging.info("Annotations are: %s", annotations)
         annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
         logging.info("Annotations path: %s", annotations_path)
-        with file(annotations_path, "w") as f:
+        with open(annotations_path, "w") as f:
             f.write('\n'.join(annotations))
         args.extend(["-annotations-list", annotations_path])
 
     if skip_annotations is not None:
         skip_annotations_path = os.path.join(
             get_scratch_dir(), "skip_annotations.txt")
-        with file(skip_annotations_path, "w") as f:
+        with open(skip_annotations_path, "w") as f:
             f.write('\n'.join(skip_annotations))
         args.extend(["-skip-annotations-list", skip_annotations_path])
 

diff --git a/dev-support/checkstyle_report.py b/dev-support/checkstyle_report.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -29,8 +29,8 @@
 from collections import defaultdict
 
 if len(sys.argv) != 3 :
-  print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
-  exit(1)
+  print("usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0])
+  sys.exit(1)
 
 def path_key(x):
   path = x.attrib['name']
@@ -40,41 +40,41 @@ def error_name(x):
   error_class = x.attrib['source']
   return error_class[error_class.rfind(".") + 1:]
 
-def print_row(path, error, master_errors, patch_errors):
-    print '%s\t%s\t%s\t%s' % (path,error, master_errors,patch_errors)
+def print_row(path, err, master_errors, patch_errors):
+  print('%s\t%s\t%s\t%s' % (path, err, master_errors, patch_errors))
 
 master = etree.parse(sys.argv[1])
 patch = etree.parse(sys.argv[2])
 
 master_dict = defaultdict(int)
 ret_value = 0
 
-for child in master.getroot().getchildren():
+for child in list(master.getroot()):
     if child.tag != 'file':
-        continue
+      continue
     file = path_key(child)
-    for error_tag in child.getchildren():
-        error = error_name(error_tag)
-        if (file, error) in master_dict:
-            master_dict[(file, error)] += 1
-        else:
-            master_dict[(file, error)] = 1
+    for error_tag in list(child):
+      error = error_name(error_tag)
+      if (file, error) in master_dict:
+        master_dict[(file, error)] += 1
+      else:
+        master_dict[(file, error)] = 1
 
-for child in patch.getroot().getchildren():
-    if child.tag != 'file':
-        continue
-    temp_dict = defaultdict(int)
-    for error_tag in child.getchildren():
-        error = error_name(error_tag)
-        if error in temp_dict:
-            temp_dict[error] += 1
-        else:
-            temp_dict[error] = 1
+for child in list(patch.getroot()):
+  if child.tag != 'file':
+    continue
+  temp_dict = defaultdict(int)
+  for error_tag in list(child):
+    error = error_name(error_tag)
+    if error in temp_dict:
+      temp_dict[error] += 1
+    else:
+      temp_dict[error] = 1
 
-    file = path_key(child)
-    for error, count in temp_dict.iteritems():
-        if count > master_dict[(file, error)]:
-            print_row(file, error, master_dict[(file, error)], count)
-            ret_value = 1
+  file = path_key(child)
+  for error, count in temp_dict.items():
+    if count > master_dict[(file, error)]:
+      print_row(file, error, master_dict[(file, error)], count)
+      ret_value = 1
 
 sys.exit(ret_value)
diff --git a/dev-support/create-release/prepend_changes.py b/dev-support/create-release/prepend_changes.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -22,7 +22,7 @@
 
 if len(sys.argv) != 3:
   print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
-  exit(1)
+  sys.exit(1)
 
 pattern = re.compile(r'^## Release .+ - Unreleased .+$')
 with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:

diff --git a/dev-support/create-release/prepend_releasenotes.py b/dev-support/create-release/prepend_releasenotes.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 ##
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -22,7 +22,7 @@
 
 if len(sys.argv) != 3:
   print("usage: %s <NEW_RELEASENOTES.md> <PREV_RELEASENOTES.md>" % sys.argv[0])
-  exit(1)
+  sys.exit(1)
 
 pattern = re.compile(r'^# .+ Release Notes$')
 with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w: