Merge pull request #9 from citynetwork/bugfix/fedora41

Add yum element override for dnf5 support

yum/README.rst

@@ -0,0 +1,43 @@
+===
+yum
+===
+Provide yum specific image building glue.
+
+RHEL/Fedora/CentOS and other yum based distributions need specific yum
+customizations.
+
+Customizations include caching of downloaded yum packages outside of the build
+chroot so that they can be reused by subsequent image builds.  The cache
+increases image building speed when building multiple images, especially on
+slow connections.  This is more effective than using an HTTP proxy as a yum
+cache since the same rpm from different mirrors is often requested.
+
+Custom yum repository configurations can also be applied by defining
+`DIB_YUM_REPO_CONF` to a space separated list of repo configuration files. The
+files will be copied to /etc/yum.repos.d/ during the image build, and then
+removed at the end of the build. Each repo file should be named differently to
+avoid a filename collision.
+
+The yum repository can also be configured by defining `DIB_YUM_REPO_PACKAGE` as
+a yum available package or a URL to an rpm file. This package can install repo
+files with any associated keys and certificates.
+
+Environment Variables
+---------------------
+
+DIB_DNF_MODULE_STREAMS
+  :Required: No
+  :Default: None
+  :Description: The following environment variable is used to select module streams
+                to be enabled during an image build on Yum/DNF based distributions.Any existing
+                stream for the given module is first disabled prior to enabling the specified
+                stream.
+  :Example: ``DIB_DNF_MODULE_STREAMS='virt:8.2 container-tools:3.0'``
+
+DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE
+  :Required: No
+  :Default: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+  :Description: The environment variable is used to override default value in pre-install 
+                stage to install epel repository from custom source
+  :Example: ``DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE=http://repos.example.com/epel/epel-latest-7.noarch.rpm``
+

yum/bin/install-packages

@@ -0,0 +1,164 @@
+#!/bin/bash
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+EXTRA_ARGS=
+ACTION=install
+MAP_ELEMENT=
+
+# allow override for dnf, as shipped by default with >=F22
+YUM=${YUM:-yum}
+YUM_FLAGS=${YUM_FLAGS:--y -d}
+
+# save global xtrace state
+_xtrace=$(set +o | grep xtrace)
+
+SCRIPTNAME=$(basename $0)
+function show_options () {
+    echo "Usage: $SCRIPTNAME [package ...]"
+    echo
+    echo "Options:"
+    echo "    -u          -- update all packages"
+    echo "    -d dir      -- download the packages to directory"
+    echo "    -e          -- erase/remove packages"
+    echo "    -m map      -- use custom element package map (Example: -m nova)"
+    exit 0
+}
+
+while getopts "hud:em:" opt; do
+    case "$opt" in
+        u)
+            ${YUM} -y update;
+            exit 0
+            ;;
+        d)
+            ACTION="download"
+            DOWNLOAD_PATH=$OPTARG
+            ;;
+        e)
+            ACTION="remove"
+            ;;
+        m)
+            MAP_ELEMENT=$OPTARG
+            ;;
+        h)
+            show_options
+            ;;
+        *)
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+
+# Packages that aren't available in the distro but requested for installation
+# can be ignored by adding them to the exclude list
+BLACKLIST=$(cat /tmp/yum-blacklist 2>/dev/null || echo "")
+WHITELIST=""
+for i in "$@"
+do
+    PKG_NAME=$i
+    if [ -n "$MAP_ELEMENT" ]; then
+        if !  PKG_NAME=$(pkg-map --element $MAP_ELEMENT $i); then
+            echo "bin/pkg-map error. $PKG_NAME"
+            exit 1
+        fi
+    fi
+    if [[ ! ${BLACKLIST[*]} =~ $PKG_NAME ]]; then
+        WHITELIST="$WHITELIST $i"
+    else
+        echo "The package $i is not available and will not be installed"
+    fi
+done
+if [ -n "$WHITELIST" ]; then
+    if [ -f /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release ]; then
+        rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
+    fi
+    if [ -n "$MAP_ELEMENT" ]; then
+        if ! PKGS=$(pkg-map --element $MAP_ELEMENT $WHITELIST); then
+            echo "bin/pkg-map error. $PKGS"
+            exit 1
+        fi
+    else
+        PKGS=$(map-packages $WHITELIST)
+    fi
+
+    if [ -z "${PKGS}" ]; then
+        echo "Not running install-packages $ACTION with empty packages list"
+        exit 0
+    fi
+
+    echo "Running install-packages ${ACTION}."
+
+    if [ "$ACTION" == "download" ]; then
+        mkdir -p $DOWNLOAD_PATH
+        if [ ${YUM} == "dnf" ]; then
+            dnf download --destdir=$DOWNLOAD_PATH $PKGS
+        else
+            # note; you don't want to use yum --download only here.
+            # Firstly that only puts things in the yum cache
+            # directory, and secondly it acts funny if old versions
+            # are already in the cache.
+            if [ ! -f /usr/bin/yumdownloader ]; then
+                yum install -y yum-utils
+            fi
+            yumdownloader --destdir=$DOWNLOAD_PATH $PKGS
+        fi
+        exit 0
+    fi
+
+    # yum & dnf have a feature/bug where missing packages in a list of
+    # packages for install/erase do not raise an error exit [1].
+    # There is a corner case when specifying just *one* package, where
+    # it will fail if the package is missing.  Thus install-packages
+    # follows this behaviour; we may need to re-evaluate this at some
+    # point.
+    #
+    # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567
+    set -o xtrace
+
+    DNF5=$(which dnf5 2>/dev/null)
+    if [ ${YUM} == 'dnf' ] && [ -n "$DNF5" ]; then
+        # dnf5 does not support debug flag
+        YUM_FLAGS=${YUM_FLAGS//-d}
+    fi
+
+    ${YUM} $YUM_FLAGS $ACTION $EXTRA_ARGS $PKGS
+
+    if [ "$ACTION" == "install" ]; then
+        # dnf5 does not support "mark install"
+        if [ ${YUM} == "dnf" ] && [ -z "$DNF5" ]; then
+            # Make sure dnf won't autoremove these packages
+            dnf mark install $PKGS
+        fi
+    fi
+    $_xtrace
+
+    # probably not the right place for this; but python-pip package on
+    # fedora/rh calls pip "pip-python" while the rest of the work
+    # expects it to be just called "pip"
+    for pkg in "$@"; do
+        if [ "$pkg" = "python-pip" ] ; then
+            alternatives --install /usr/bin/pip pip /usr/bin/pip-python 10
+        fi
+    done
+fi

yum/cleanup.d/99-remove-yum-repo-conf

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# exit directly if DIB_YUM_REPO_CONF is not defined properly
+if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then
+    echo "DIB_YUM_REPO_CONF is not set - no repo configurations will be cleaned up"
+    exit 0
+fi
+
+for file in $DIB_YUM_REPO_CONF; do
+    sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/$(basename $file)
+done

yum/element-deps

@@ -0,0 +1 @@
+install-bin

yum/extra-data.d/99-yum-repo-conf

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Add additional yum repo configuration(s) with $DIB_YUM_REPO_CONF
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# exit directly if DIB_YUM_REPO_CONF is not defined properly
+if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then
+    echo "DIB_YUM_REPO_CONF is not set - no repo configuration will be copied in"
+    exit 0
+fi
+
+for file in $DIB_YUM_REPO_CONF; do
+    if [ ! -f $file ]; then
+        echo "$file is not a valid yum repo configuration file."
+        echo "You should assign a list of proper yum repo configuration"
+        echo "files in DIB_YUM_REPO_CONF."
+        exit 1
+    fi
+
+    # copy the yum repo configuration
+    sudo cp -L -f $file $TMP_MOUNT_PATH/etc/yum.repos.d
+
+done

yum/post-install.d/99-reset-yum-conf

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [[ ${YUM} == "dnf" ]]; then
+    cfg=/etc/dnf/dnf.conf
+    cachedir=/var/cache/dnf
+else
+    cfg=/etc/yum.conf
+    cachedir=/var/cache/yum
+fi
+
+sed -i "/^keepcache/c\keepcache=0" $cfg
+sed -i "/^cachedir/c\cachedir=${cachedir}" $cfg

yum/pre-install.d/00-01-yum-keepcache

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [[ ${YUM} == "dnf" ]]; then
+    # dnf is slightly different; doesn't have these by default
+    cfg=/etc/dnf/dnf.conf
+    echo "keepcache=1" >> $cfg
+    echo "cachedir=/tmp/yum" >> $cfg
+else
+    cfg=/etc/yum.conf
+    sed -i 's/keepcache=0/keepcache=1/' $cfg
+    sed -i 's/cachedir=\/var\/cache\/yum/cachedir=\/tmp\/yum/' $cfg
+fi
+

yum/pre-install.d/00-dnf-update

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -eq 22 ]]; then
+    # we need dnf mark command from dnf 1.1.1, which is pretty recent
+    dnf --refresh update -y dnf
+fi

yum/pre-install.d/00-install-repo-package

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+if [ -n "${DIB_YUM_REPO_PACKAGE:-}" ] ; then
+    rpm -ivh ${DIB_YUM_REPO_PACKAGE}
+fi

yum/pre-install.d/01-00-centos-python3

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# Ensure the python3 interpreter and YAML libraries are installed
+# early (even before package-installs, which is written in Python and
+# uses YAML).
+
+if [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} == 7 ]]; then
+    # Our package map and install stuff doesn't have a way to say
+    # "install this from EPEL".  So we hack in an install of it here
+    # from EPEL.  Nothing else should have installed EPEL at this
+    # early stage.
+    yum install -y python3
+    # NOTE(dpawlik) The epel-release package is not available in RHEL.
+    yum install -y ${DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE:-https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm}
+    yum install -y python36-PyYAML
+    yum remove -y epel-release
+elif [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} > 7 ]]; then
+    # For 8 and above ensure the "user" python3 package is installed
+    # so we have /usr/bin/python3 and pyyaml.
+    dnf install -y python3 python3-pyyaml
+fi

yum/pre-install.d/01-module-configuration

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# When building RHEL overcloud images, certain modules may be preferred
+# or even required.
+#
+# For example, container-tools:2.0 is notably a requirement when running
+# the Train releases on RHEL 8.2.
+#
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+
+set -eu
+set -o pipefail
+
+export DIB_DNF_MODULE_STREAMS=${DIB_DNF_MODULE_STREAMS:-}
+
+if [ "${YUM}" == "dnf" ]; then
+    for m in ${DIB_DNF_MODULE_STREAMS}; do
+        ${YUM} -y module disable ${m/:*/}
+        ${YUM} -y module enable ${m}
+    done
+fi

yum/root.d/50-yum-cache

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "$TARGET_ROOT" ]
+
+YUM_CACHE_DIR=$DIB_IMAGE_CACHE/yum
+mkdir -p $YUM_CACHE_DIR
+
+sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum
+sudo mount --bind $YUM_CACHE_DIR $TMP_MOUNT_PATH/tmp/yum