Correct dependencies in agent spec file

This effectively reverts PR #2893. Simplify the python dependencies as much as possible: - RHEL7, RHEL9, CentOS-Stream-9 and the Fedoras have a `python3` package available from the standard distro repos. The actual version varies by distro: 3.6 for RHEL7, 3.9.10 for RHEL9/CentOS-Stream-9, 3.10.0-1 for Fedora 35, 3.10.4-1 for Fedora 36. - RHEL8 and CentOS-Stream-8 have `python36`, `python38` and `python39` packages available through the module system. We use the generic `python3` package on most distros, and standardize on `python39` for RHEL8/CentOS-Stream-8. That is mostly treated using a jinja2 loop and a (new) `target.json` file that generates the separate conditionals for RHEL/CentOS Stream 8 on one hand and the rest on the other. Unfortunately, I had to treat RHEL7 separately from the rest because it is missing a python `cffi` package that all the others have. I haven't found a clean way to deal with that in `jinja2`, so it is an explicit RPM conditional in the spec file. Also, the Fedoras are treated separately, at least for now. I am testing by setting the release in subscription-manager now (wherever possible: there is no 7.9 release which I don't understand - this needs investigation - and there is no 9.1 release yet which is understandable). However, that means that RHEL 8.4 and 8.5 have an `ansible` package and 8.6 has an `ansible-core` package available. I've hard-coded `ansible-core` in the spec file (except on RHEL7) which breaks the installation on 8.4 and 8.5. I don't know how to deal with this problem. Modify the base docker file to enable `python39`. Since we standardized on `python39` for CentOS-Stream-8, we enable that module and stop worrying about `python36`.
distributed-system-analysis · Jun 25, 2022 · 32c8526 · 32c8526
1 parent edd4ade
commit 32c8526
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 33 deletions.
diff --git a/agent/containers/images/Dockerfile.base.j2 b/agent/containers/images/Dockerfile.base.j2
@@ -12,8 +12,7 @@ COPY ./{{ pbench_repo_file }} /etc/yum.repos.d/pbench.repo
 # ... and finally, ensure the proper pbench-agent environment variables are set up.
 RUN \
 {% if is_centos_8 %}
-    {{ pkgmgr }} module -y enable python36 && \
-    {{ pkgmgr }} module -y disable python38 && \
+    {{ pkgmgr }} module -y enable python39 && \
 {% endif %}
 {% if image_name == 'centos' %}
     {{ pkgmgr }} install -y --setopt=tsflags=nodocs https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ image_rev }}.noarch.rpm && \

diff --git a/agent/rpm/pbench-agent.spec.j2 b/agent/rpm/pbench-agent.spec.j2
@@ -8,61 +8,93 @@ URL:            https://github.com/distributed-system-analysis/pbench
 Source0:        pbench-agent-%{version}.tar.gz
 Buildarch:      noarch
 
-# EPEL provides ansible (a curated set of roles with a dependency on ansible-core)
-# on RHEL 8 and RHEL9. Only CentOS-Stream seems to not have an ansible package
-# available.
 
-%if 0%{?centos} >= 8
-Requires:  ansible-core
+# RHEL7, RHEL9 (and CentOS-Stream-9) and the Fedoras provide a python3 package
+# through the standard distro repos.
+# RHEL8 (and CentOS-Stream-8) provide python36, python38 and python39 as modules.
+# We stick with python39 for them for now.
+# N.B. The condition catches both RHEL8 and CentOS-Stream-8)
+%if 0%{?rhel} == 8
+
+Requires:  python39
+# This is used by the shebang processor
+%define __python3 /usr/bin/python3.9
+
 %else
-Requires:  ansible
+
+Requires:  python3
+
 %endif
 
 
+# RHEL7 is a special case: it does not conform to the (slightly) more
+# general pattern of the rest
+
 %if 0%{?rhel} == 7
-Requires:  python3, python3-pip
-%endif
 
-# This condition will be true on a CentOS-Stream-8 system as well
-%if 0%{?rhel} == 8
-Requires:  python36, python3-pip
 # RPMs for modules in requirements.txt
-Requires:  python3-cffi, python3-click, python3-requests
-# RPMs for module dependencies
-Requires:  python3-docutils, python3-psutil
-%endif
+Requires: python3-pip, python3-requests
+#RPMS for module dependencies
+Requires: python3-psutil
+
+# RHEL7 also does not define __python3 - we need it for installing
+# requirements through pip so we define it here
+%define __python3 /usr/bin/python3
+
+%else
+
+{# Jinja2 template for python dependencies - see the targets.json file for the input data #}
+{%for tname, tcond in targets.items() -%}
 
-# This condition will be true on a CentOS-Stream-9 system as well
-%if 0%{?rhel} == 9
-Requires:  python3-pip
+%if {{ tcond }}
 # RPMs for modules in requirements.txt
-Requires:  python3-cffi, python3-requests
-# RPMs for module dependencies
-Requires:  python3-docutils, python3-psutil
+Requires: {{ tname }}-pip, {{ tname }}-cffi, {{ tname }}-requests
+#RPMS for module dependencies
+Requires: {{ tname }}-psutil
 %endif
 
+{% endfor %}
+{# end of template #}
 
+%endif
+
+# docutils is not available for RHEL7 - it is also *only* available as
+# `python3-docutil's on everything else, which is not good for
+# RHEL8/CentOS Stream 8 where we would want `python39-docutils', but
+# this does not exist. We need to handle it specially: check for
+# RHEL9/CentOS Stream 9 OR Fedora
+
+%if 0%{?rhel} > 8 || 0%{?fedora} != 0
+Requires: python3-docutils
+%endif
+
+# additional packages that Fedora builds but the RHELs don't
 %if 0%{?fedora} != 0
-Requires:  python3, python3-pip
 # RPMs for modules in requirements.txt
-Requires:  python3-bottle, python3-cffi, python3-click, python3-daemon
-Requires:  python3-jinja2, python3-redis, python3-requests, python3-sh
+Requires:  python3-bottle, python3-click, python3-daemon
+Requires:  python3-jinja2, python3-redis, python3-sh
+%endif
 
 %if 0%{?fedora} >= 36
 Requires: python3-ifaddr
 %endif
 
-# RPMs for module dependencies
-Requires:  python3-psutil
-%endif
-
+# Common requirements
 Requires:  perl, perl-Data-UUID, perl-JSON, perl-JSON-XS
 Requires:  perl-Time-HiRes
 
 Requires:  bc, bzip2, hostname, iproute, iputils, net-tools, numactl
 Requires:  openssh-clients, openssh-server, procps-ng, psmisc, redis
 Requires:  rpmdevtools, rsync, screen, sos, tar, xz
 
+# The condition here should be 7 OR 8.4 OR 8.5 but we don't have a way to deal with minor versions.
+# The resulting RPM will NOT work on RHEL8.[45].
+%if 0%{?rhel} == 7
+Requires:  ansible
+%else
+Requires:  ansible-core
+%endif
+
 Obsoletes: pbench <= 0.34
 Conflicts: pbench <= 0.34
 # configtools is packaged with pbench-agent, so we specifically do NOT want
@@ -98,8 +130,7 @@ if pip3 show configtools > /dev/null 2>&1 ;then pip3 uninstall -y configtools ;f
 %post
 
 # Install python dependencies
-python3 -m pip install --upgrade pip 2>&1 > /%{installdir}/pip3-upgrade.log
-python3 -m pip --no-cache-dir install --prefix=/%{installdir} -r /%{installdir}/requirements.txt > /%{installdir}/pip3-install.log 2>&1
+%{__python3} -m pip --no-cache-dir install --prefix=/%{installdir} -r /%{installdir}/requirements.txt > /%{installdir}/pip3-install.log 2>&1
 
 # link the pbench profile, so it'll automatically be sourced on login
 ln -sf /%{installdir}/profile /etc/profile.d/pbench-agent.sh

diff --git a/agent/rpm/targets.json b/agent/rpm/targets.json
@@ -0,0 +1,7 @@
+{
+    "targets": {
+        "python3": "0%{?rhel} != 8",
+        "python39": "0%{?rhel} == 8"
+    }
+}
+
diff --git a/utils/rpm.mk b/utils/rpm.mk
@@ -41,7 +41,7 @@ srpm: spec patches tarball
 .PHONY: spec
 spec: rpm-dirs ${prog}.spec.j2
 	if [ -e ./seqno ] ;then expr ${seqno} + 1 > ./seqno ;fi
-	jinja2 ${prog}.spec.j2 -D version=${VERSION} -D gdist=g${sha1} -D seqno=${seqno} > ${RPMSPEC}/${prog}.spec
+	jinja2 ${prog}.spec.j2 targets.json -D version=${VERSION} -D gdist=g${sha1} -D seqno=${seqno} > ${RPMSPEC}/${prog}.spec
 	cp ${PBENCHTOP}/utils/rpmlint ${RPMSPEC}/pbench-common.rpmlintrc
 	XDG_CONFIG_HOME=${PBENCHTOP}/utils rpmlint ${RPMSPEC}/${prog}.spec