Add yum element override for dnf5 support

In order to build images with dnf5 out of the box, like Fedora 41,
we need to fix yum element to support dnf5 syntax.

This can be dropped as soon as [1] will be merged.

[1] https://review.opendev.org/c/openstack/diskimage-builder/+/934332

yum/README.rst

@@ -0,0 +1,43 @@
+===
+yum
+===
+Provide yum specific image building glue.
+
+RHEL/Fedora/CentOS and other yum based distributions need specific yum
+customizations.
+
+Customizations include caching of downloaded yum packages outside of the build
+chroot so that they can be reused by subsequent image builds.  The cache
+increases image building speed when building multiple images, especially on
+slow connections.  This is more effective than using an HTTP proxy as a yum
+cache since the same rpm from different mirrors is often requested.
+
+Custom yum repository configurations can also be applied by defining
+`DIB_YUM_REPO_CONF` to a space separated list of repo configuration files. The
+files will be copied to /etc/yum.repos.d/ during the image build, and then
+removed at the end of the build. Each repo file should be named differently to
+avoid a filename collision.
+
+The yum repository can also be configured by defining `DIB_YUM_REPO_PACKAGE` as
+a yum available package or a URL to an rpm file. This package can install repo
+files with any associated keys and certificates.
+
+Environment Variables
+---------------------
+
+DIB_DNF_MODULE_STREAMS
+  :Required: No
+  :Default: None
+  :Description: The following environment variable is used to select module streams
+                to be enabled during an image build on Yum/DNF based distributions.Any existing
+                stream for the given module is first disabled prior to enabling the specified
+                stream.
+  :Example: ``DIB_DNF_MODULE_STREAMS='virt:8.2 container-tools:3.0'``
+
+DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE
+  :Required: No
+  :Default: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+  :Description: The environment variable is used to override default value in pre-install 
+                stage to install epel repository from custom source
+  :Example: ``DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE=http://repos.example.com/epel/epel-latest-7.noarch.rpm``
+

yum/bin/install-packages

@@ -0,0 +1,164 @@
+#!/bin/bash
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+EXTRA_ARGS=
+ACTION=install
+MAP_ELEMENT=
+
+# allow override for dnf, as shipped by default with >=F22
+YUM=${YUM:-yum}
+YUM_FLAGS=${YUM_FLAGS:--y -d}
+
+# save global xtrace state
+_xtrace=$(set +o | grep xtrace)
+
+SCRIPTNAME=$(basename $0)
+function show_options () {
+    echo "Usage: $SCRIPTNAME [package ...]"
+    echo
+    echo "Options:"
+    echo "    -u          -- update all packages"
+    echo "    -d dir      -- download the packages to directory"
+    echo "    -e          -- erase/remove packages"
+    echo "    -m map      -- use custom element package map (Example: -m nova)"
+    exit 0
+}
+
+while getopts "hud:em:" opt; do
+    case "$opt" in
+        u)
+            ${YUM} -y update;
+            exit 0
+            ;;
+        d)
+            ACTION="download"
+            DOWNLOAD_PATH=$OPTARG
+            ;;
+        e)
+            ACTION="remove"
+            ;;
+        m)
+            MAP_ELEMENT=$OPTARG
+            ;;
+        h)
+            show_options
+            ;;
+        *)
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+
+# Packages that aren't available in the distro but requested for installation
+# can be ignored by adding them to the exclude list
+BLACKLIST=$(cat /tmp/yum-blacklist 2>/dev/null || echo "")
+WHITELIST=""
+for i in "$@"
+do
+    PKG_NAME=$i
+    if [ -n "$MAP_ELEMENT" ]; then
+        if !  PKG_NAME=$(pkg-map --element $MAP_ELEMENT $i); then
+            echo "bin/pkg-map error. $PKG_NAME"
+            exit 1
+        fi
+    fi
+    if [[ ! ${BLACKLIST[*]} =~ $PKG_NAME ]]; then
+        WHITELIST="$WHITELIST $i"
+    else
+        echo "The package $i is not available and will not be installed"
+    fi
+done
+if [ -n "$WHITELIST" ]; then
+    if [ -f /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release ]; then
+        rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
+    fi
+    if [ -n "$MAP_ELEMENT" ]; then
+        if ! PKGS=$(pkg-map --element $MAP_ELEMENT $WHITELIST); then
+            echo "bin/pkg-map error. $PKGS"
+            exit 1
+        fi
+    else
+        PKGS=$(map-packages $WHITELIST)
+    fi
+
+    if [ -z "${PKGS}" ]; then
+        echo "Not running install-packages $ACTION with empty packages list"
+        exit 0
+    fi
+
+    echo "Running install-packages ${ACTION}."
+
+    if [ "$ACTION" == "download" ]; then
+        mkdir -p $DOWNLOAD_PATH
+        if [ ${YUM} == "dnf" ]; then
+            dnf download --destdir=$DOWNLOAD_PATH $PKGS
+        else
+            # note; you don't want to use yum --download only here.
+            # Firstly that only puts things in the yum cache
+            # directory, and secondly it acts funny if old versions
+            # are already in the cache.
+            if [ ! -f /usr/bin/yumdownloader ]; then
+                yum install -y yum-utils
+            fi
+            yumdownloader --destdir=$DOWNLOAD_PATH $PKGS
+        fi
+        exit 0
+    fi
+
+    # yum & dnf have a feature/bug where missing packages in a list of
+    # packages for install/erase do not raise an error exit [1].
+    # There is a corner case when specifying just *one* package, where
+    # it will fail if the package is missing.  Thus install-packages
+    # follows this behaviour; we may need to re-evaluate this at some
+    # point.
+    #
+    # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567
+    set -o xtrace
+
+    DNF5=$(which dnf5 2>/dev/null)
+    if [ ${YUM} == 'dnf' ] && [ -n "$DNF5" ]; then
+        # dnf5 does not support debug flag
+        YUM_FLAGS=${YUM_FLAGS//-d}
+    fi
+
+    ${YUM} $YUM_FLAGS $ACTION $EXTRA_ARGS $PKGS
+
+    if [ "$ACTION" == "install" ]; then
+        # dnf5 does not support "mark install"
+        if [ ${YUM} == "dnf" ] && [ -z "$DNF5" ]; then
+            # Make sure dnf won't autoremove these packages
+            dnf mark install $PKGS
+        fi
+    fi
+    $_xtrace
+
+    # probably not the right place for this; but python-pip package on
+    # fedora/rh calls pip "pip-python" while the rest of the work
+    # expects it to be just called "pip"
+    for pkg in "$@"; do
+        if [ "$pkg" = "python-pip" ] ; then
+            alternatives --install /usr/bin/pip pip /usr/bin/pip-python 10
+        fi
+    done
+fi

yum/cleanup.d/99-remove-yum-repo-conf

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# exit directly if DIB_YUM_REPO_CONF is not defined properly
+if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then
+    echo "DIB_YUM_REPO_CONF is not set - no repo configurations will be cleaned up"
+    exit 0
+fi
+
+for file in $DIB_YUM_REPO_CONF; do
+    sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/$(basename $file)
+done

yum/element-deps

@@ -0,0 +1 @@
+install-bin

yum/extra-data.d/99-yum-repo-conf

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Add additional yum repo configuration(s) with $DIB_YUM_REPO_CONF
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# exit directly if DIB_YUM_REPO_CONF is not defined properly
+if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then
+    echo "DIB_YUM_REPO_CONF is not set - no repo configuration will be copied in"
+    exit 0
+fi
+
+for file in $DIB_YUM_REPO_CONF; do
+    if [ ! -f $file ]; then
+        echo "$file is not a valid yum repo configuration file."
+        echo "You should assign a list of proper yum repo configuration"
+        echo "files in DIB_YUM_REPO_CONF."
+        exit 1
+    fi
+
+    # copy the yum repo configuration
+    sudo cp -L -f $file $TMP_MOUNT_PATH/etc/yum.repos.d
+
+done

yum/post-install.d/99-reset-yum-conf

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [[ ${YUM} == "dnf" ]]; then
+    cfg=/etc/dnf/dnf.conf
+    cachedir=/var/cache/dnf
+else
+    cfg=/etc/yum.conf
+    cachedir=/var/cache/yum
+fi
+
+sed -i "/^keepcache/c\keepcache=0" $cfg
+sed -i "/^cachedir/c\cachedir=${cachedir}" $cfg

yum/pre-install.d/00-01-yum-keepcache

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [[ ${YUM} == "dnf" ]]; then
+    # dnf is slightly different; doesn't have these by default
+    cfg=/etc/dnf/dnf.conf
+    echo "keepcache=1" >> $cfg
+    echo "cachedir=/tmp/yum" >> $cfg
+else
+    cfg=/etc/yum.conf
+    sed -i 's/keepcache=0/keepcache=1/' $cfg
+    sed -i 's/cachedir=\/var\/cache\/yum/cachedir=\/tmp\/yum/' $cfg
+fi
+

yum/pre-install.d/00-dnf-update

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -eq 22 ]]; then
+    # we need dnf mark command from dnf 1.1.1, which is pretty recent
+    dnf --refresh update -y dnf
+fi

yum/pre-install.d/00-install-repo-package

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [ -n "${DIB_YUM_REPO_PACKAGE:-}" ] ; then
+    rpm -ivh ${DIB_YUM_REPO_PACKAGE}
+fi

yum/pre-install.d/01-00-centos-python3

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# Ensure the python3 interpreter and YAML libraries are installed
+# early (even before package-installs, which is written in Python and
+# uses YAML).
+
+if [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} == 7 ]]; then
+    # Our package map and install stuff doesn't have a way to say
+    # "install this from EPEL".  So we hack in an install of it here
+    # from EPEL.  Nothing else should have installed EPEL at this
+    # early stage.
+    yum install -y python3
+    # NOTE(dpawlik) The epel-release package is not available in RHEL.
+    yum install -y ${DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE:-https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm}
+    yum install -y python36-PyYAML
+    yum remove -y epel-release
+elif [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} > 7 ]]; then
+    # For 8 and above ensure the "user" python3 package is installed
+    # so we have /usr/bin/python3 and pyyaml.
+    dnf install -y python3 python3-pyyaml
+fi

yum/pre-install.d/01-module-configuration

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# When building RHEL overcloud images, certain modules may be preferred
+# or even required.
+#
+# For example, container-tools:2.0 is notably a requirement when running
+# the Train releases on RHEL 8.2.
+#
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+
+set -eu
+set -o pipefail
+
+export DIB_DNF_MODULE_STREAMS=${DIB_DNF_MODULE_STREAMS:-}
+
+if [ "${YUM}" == "dnf" ]; then
+    for m in ${DIB_DNF_MODULE_STREAMS}; do
+        ${YUM} -y module disable ${m/:*/}
+        ${YUM} -y module enable ${m}
+    done
+fi

yum/root.d/50-yum-cache

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "$TARGET_ROOT" ]
+
+YUM_CACHE_DIR=$DIB_IMAGE_CACHE/yum
+mkdir -p $YUM_CACHE_DIR
+
+sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum
+sudo mount --bind $YUM_CACHE_DIR $TMP_MOUNT_PATH/tmp/yum