Initial changes to support Python 3.10

* Correct CI / unit tests to work with python 3.10

This also corrects the version of `black` used in the commit hook to
work with Python 3.10.

The version of pytest needs to be at least 6.2.5 in order to work with
Python 3.10.  As a result of this change, we need to correct one test
with its use of `ExceptiionInfo` objects.

Unfortunately, the "usage" message created by the `argparse` module
changed to use "options:" instead of "optional arguments:". We ensure
that only for the two legacy tests that use a python command the output
emitted by older versions of Python3 look like Python 3.10 and later.

* Stop using deprecated setuptools method of install

The new way to install our Python3 interfaces is:

    /usr/bin/python3 -m pip install --prefix=${dir} -e .

See https://docs.python.org/3.6/installing/index.html#installing-index.

The new interface allows for the `agent/Makefile` and `server/Makefile`
to create the python CLI interfaces on environments running Python 3.10
and later.

Why do we use `/usr/bin/python3`?  If you invoke pip inside a
virtualenv to install a package, it automatically uninstalls the
previous one from where it was installed, and installs it again in the
`--prefix` path. That is wrong on so many levels. So, to work around
this terrible behavior we have to invoke the non-virtualenv
`/usr/bin/python3` so that the installation works without touching the
virtualenv.

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
     hooks:
      - id: flake8
   - repo: https://github.com/python/black.git
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
         - id: black
           language_version: python3

agent/Makefile

@@ -250,10 +250,13 @@ install-lib:
 	${INSTALL} ${INSTALLOPTS} ${LIBDIR}
 	${COPY} lib ${DESTDIR}
 
+# NOTE: we deliberately use `/usr/bin/python3` to make sure that when this is
+# invoked inside a python virtualenv the pip installation won't uninstall the
+# existing pbench module, breaking the environment.
 install-python3-setup: install-util-scripts install-lib
 	${COPY} requirements.txt ${DESTDIR}
 	mkdir -p ${DESTDIR}/python3
-	(cd ..; SKIP_GENERATE_AUTHORS=1 SKIP_WRITE_GIT_CHANGELOG=1 python3 setup.py install --prefix=${DESTDIR}/python3)
+	(cd ..; /usr/bin/python3 -m pip install --prefix=${DESTDIR}/python3 -e .)
 	${COPY} $(addprefix ${DESTDIR}/python3/bin/, ${click-scripts}) ${UTILDIR}/
 	rm -rf ${DESTDIR}/python3
 	${COPY} ../lib/pbench ${LIBDIR}/

agent/test-requirements.txt

@@ -2,9 +2,9 @@ coverage
 filelock
 gitpython
 mock>=3.0.5
-pytest>=4.6.3, < 5
+pytest>=6.2.5
 pytest-cov>=2.7.1
 pytest-helpers-namespace==2019.1.8
-pytest-mock>=1.10.4, < 3.3  # Upper limit required for compatibility with pytest limit
+pytest-mock
 pytest-xdist
 responses

agent/util-scripts/gold/pbench-tool-meister-start/test-54.txt

@@ -8,7 +8,7 @@ positional arguments:
   tool_group            The tool group name of tools to be run by the Tool
                         Meisters.
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   --sysinfo SYSINFO     The list of system information items to be collected.
   --orchestrate {create,existing}

agent/util-scripts/gold/pbench-tool-meister-stop/test-55.txt

@@ -7,7 +7,7 @@ positional arguments:
   tool_group            The tool group name of tools being run in the Tool
                         Meisters.
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   --sysinfo SYSINFO     The list of system information items to be collected.
   --interrupt           Whether or not the stop operation is in response to an

agent/util-scripts/unittests

@@ -615,6 +615,8 @@ declare -A post_hooks=(
     [test-07]='cat ${_testdir}/42-iter/sample42/tools-foobar/testhost.example.com/postprocess.log >> ${_testout} 2>&1'
     [test-19]='rm ${_testopt}/unittest-scripts/pbench-tool-meister-client'
     [test-53]='sort_testlog; sort_tmlogs; filter_tmerrs'
+    [test-54]='sed -Ei "s/^optional arguments:/options:/" ${_testout}'
+    [test-55]='sed -Ei "s/^optional arguments:/options:/" ${_testout}'
     [test-56]='sort_testlog; sort_tdslog; sort_tmlogs; filter_tmerrs'
     [test-57]='sort_testlog; sort_tdslog; sort_tmlogs; filter_tmerrs'
     [test-61]='sort_testlog; sort_tdslog; sort_tmlogs; filter_tmerrs'

exec-unittests

@@ -103,12 +103,14 @@ function verify_make_source_tree {
     mkdir ${_dir} || return ${?}
 
     printf -- "\n\n\nVerifying %s source tree build\n\n" "${_major^}"
-    (cd ${_major} && make DESTDIR=${_dir} install)
+    (cd ${_major} && make DESTDIR=${_dir} install > ${_dir}.log 2>&1)
     _rc=${?}
     if [[ ${_rc} -ne 0 ]]; then
+        cat ${_dir}.log
         printf -- "\n%s source tree build failed with '%s'\n\n" "${_major^}" "${_rc}"
     else
         printf -- "\n%s source tree build succeeded\n\n" "${_major^}"
+        rm -rf ${_dir} ${_dir}.log
     fi
     return ${_rc}
 }

lib/pbench/test/unit/agent/task/test_copy_result_tb.py

@@ -48,13 +48,13 @@ def test_bad_tar(self, caplog):
             f"http://pbench.example.com/api/v1/upload/{bad_tarball}",
             status=HTTPStatus.OK,
         )
-        expected_error_message = (
-            f"FileNotFoundError: Tar ball '{bad_tarball}' does not exist"
-        )
+        expected_error_message = f"Tar ball '{bad_tarball}' does not exist"
         caplog.set_level(logging.ERROR, logger=self.logger.name)
-        with pytest.raises(FileNotFoundError) as e:
+        with pytest.raises(FileNotFoundError) as excinfo:
             crt = CopyResultTb(
                 "controller", bad_tarball, 0, "ignoremd5", self.config, self.logger
             )
             crt.copy_result_tb("token")
-        assert str(e).endswith(expected_error_message)
+        assert str(excinfo.value).endswith(
+            expected_error_message
+        ), f"expected='...{expected_error_message}', found='{str(excinfo.value)}'"

server/Makefile

@@ -44,10 +44,13 @@ install-dirs:
 	${INSTALL} ${INSTALLOPTS} ${LIBDIR}/pbench
 	${INSTALL} ${INSTALLOPTS} ${LIBDIR}/pbench/cli
 
+# NOTE: we deliberately use `/usr/bin/python3` to make sure that when this is
+# invoked inside a python virtualenv the pip installation won't uninstall the
+# existing pbench module, breaking the environment.
 install-python3-setup: install-bin install-lib
 	mkdir -p ${DESTDIR}/python3
 	${COPY} requirements.txt ${DESTDIR}
-	(cd ..; SKIP_GENERATE_AUTHORS=1 SKIP_WRITE_GIT_CHANGELOG=1 python3 setup.py install --prefix=${DESTDIR}/python3)
+	(cd ..; /usr/bin/python3 -m pip install --prefix=${DESTDIR}/python3 -e .)
 	${COPY} $(addprefix ${DESTDIR}/python3/bin/, pbench-config pbench-server ${click-scripts}) ${BINDIR}/
 	${RM} -r ${DESTDIR}/python3
 	${COPY} ../lib/pbench ${LIBDIR}/

server/test-requirements.txt

@@ -2,11 +2,11 @@ coverage
 filelock
 gitpython
 mock>=3.0.5
-pytest>=4.6.3, < 5
+pytest>=6.2.5
 pytest-cov>=2.7.1
 pytest-freezegun
 pytest-helpers-namespace==2019.1.8
-pytest-mock>=1.10.4, < 3.3  # Upper limit required for compatibility with pytest limit
+pytest-mock
 pytest-xdist
 requests_mock
 responses

utils/rpm.mk

@@ -42,7 +42,7 @@ srpm: spec patches tarball
 spec: rpm-dirs ${prog}.spec.j2
 	if [ -e ./seqno ] ;then expr ${seqno} + 1 > ./seqno ;fi
 	jinja2 ${prog}.spec.j2 -D version=${VERSION} -D gdist=g${sha1} -D seqno=${seqno} > ${RPMSPEC}/${prog}.spec
-	rpmlint --file ${PBENCHTOP}/utils/rpmlint ${RPMSPEC}/${prog}.spec
+	rpmlint --rpmlintrc ${PBENCHTOP}/utils/rpmlint ${RPMSPEC}/${prog}.spec
 
 .PHONY: patches
 patches: rpm-dirs

utils/rpmlint

@@ -1,3 +1,2 @@
 addFilter("setup-not-quiet")
 addFilter("invalid-url Source0")
-addFilter("macro-in-comment")