diff --git a/yum/README.rst b/yum/README.rst new file mode 100644 index 0000000..6ee06d4 --- /dev/null +++ b/yum/README.rst @@ -0,0 +1,43 @@ +=== +yum +=== +Provide yum specific image building glue. + +RHEL/Fedora/CentOS and other yum based distributions need specific yum +customizations. + +Customizations include caching of downloaded yum packages outside of the build +chroot so that they can be reused by subsequent image builds. The cache +increases image building speed when building multiple images, especially on +slow connections. This is more effective than using an HTTP proxy as a yum +cache since the same rpm from different mirrors is often requested. + +Custom yum repository configurations can also be applied by defining +`DIB_YUM_REPO_CONF` to a space separated list of repo configuration files. The +files will be copied to /etc/yum.repos.d/ during the image build, and then +removed at the end of the build. Each repo file should be named differently to +avoid a filename collision. + +The yum repository can also be configured by defining `DIB_YUM_REPO_PACKAGE` as +a yum available package or a URL to an rpm file. This package can install repo +files with any associated keys and certificates. + +Environment Variables +--------------------- + +DIB_DNF_MODULE_STREAMS + :Required: No + :Default: None + :Description: The following environment variable is used to select module streams + to be enabled during an image build on Yum/DNF based distributions.Any existing + stream for the given module is first disabled prior to enabling the specified + stream. + :Example: ``DIB_DNF_MODULE_STREAMS='virt:8.2 container-tools:3.0'`` + +DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE + :Required: No + :Default: https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + :Description: The environment variable is used to override default value in pre-install + stage to install epel repository from custom source + :Example: ``DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE=http://repos.example.com/epel/epel-latest-7.noarch.rpm`` + diff --git a/yum/bin/install-packages b/yum/bin/install-packages new file mode 100755 index 0000000..8d7dc15 --- /dev/null +++ b/yum/bin/install-packages @@ -0,0 +1,164 @@ +#!/bin/bash + +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then + set -x +fi +set -eu +set -o pipefail + +EXTRA_ARGS= +ACTION=install +MAP_ELEMENT= + +# allow override for dnf, as shipped by default with >=F22 +YUM=${YUM:-yum} +YUM_FLAGS=${YUM_FLAGS:--y -d} + +# save global xtrace state +_xtrace=$(set +o | grep xtrace) + +SCRIPTNAME=$(basename $0) +function show_options () { + echo "Usage: $SCRIPTNAME [package ...]" + echo + echo "Options:" + echo " -u -- update all packages" + echo " -d dir -- download the packages to directory" + echo " -e -- erase/remove packages" + echo " -m map -- use custom element package map (Example: -m nova)" + exit 0 +} + +while getopts "hud:em:" opt; do + case "$opt" in + u) + ${YUM} -y update; + exit 0 + ;; + d) + ACTION="download" + DOWNLOAD_PATH=$OPTARG + ;; + e) + ACTION="remove" + ;; + m) + MAP_ELEMENT=$OPTARG + ;; + h) + show_options + ;; + *) + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + + +# Packages that aren't available in the distro but requested for installation +# can be ignored by adding them to the exclude list +BLACKLIST=$(cat /tmp/yum-blacklist 2>/dev/null || echo "") +WHITELIST="" +for i in "$@" +do + PKG_NAME=$i + if [ -n "$MAP_ELEMENT" ]; then + if ! PKG_NAME=$(pkg-map --element $MAP_ELEMENT $i); then + echo "bin/pkg-map error. $PKG_NAME" + exit 1 + fi + fi + if [[ ! ${BLACKLIST[*]} =~ $PKG_NAME ]]; then + WHITELIST="$WHITELIST $i" + else + echo "The package $i is not available and will not be installed" + fi +done +if [ -n "$WHITELIST" ]; then + if [ -f /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release ]; then + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release + fi + if [ -n "$MAP_ELEMENT" ]; then + if ! PKGS=$(pkg-map --element $MAP_ELEMENT $WHITELIST); then + echo "bin/pkg-map error. $PKGS" + exit 1 + fi + else + PKGS=$(map-packages $WHITELIST) + fi + + if [ -z "${PKGS}" ]; then + echo "Not running install-packages $ACTION with empty packages list" + exit 0 + fi + + echo "Running install-packages ${ACTION}." + + if [ "$ACTION" == "download" ]; then + mkdir -p $DOWNLOAD_PATH + if [ ${YUM} == "dnf" ]; then + dnf download --destdir=$DOWNLOAD_PATH $PKGS + else + # note; you don't want to use yum --download only here. + # Firstly that only puts things in the yum cache + # directory, and secondly it acts funny if old versions + # are already in the cache. + if [ ! -f /usr/bin/yumdownloader ]; then + yum install -y yum-utils + fi + yumdownloader --destdir=$DOWNLOAD_PATH $PKGS + fi + exit 0 + fi + + # yum & dnf have a feature/bug where missing packages in a list of + # packages for install/erase do not raise an error exit [1]. + # There is a corner case when specifying just *one* package, where + # it will fail if the package is missing. Thus install-packages + # follows this behaviour; we may need to re-evaluate this at some + # point. + # + # [1] https://bugzilla.redhat.com/show_bug.cgi?id=965567 + set -o xtrace + + DNF5=$(which dnf5 2>/dev/null) + if [ ${YUM} == 'dnf' ] && [ -n "$DNF5" ]; then + # dnf5 does not support debug flag + YUM_FLAGS=${YUM_FLAGS//-d} + fi + + ${YUM} $YUM_FLAGS $ACTION $EXTRA_ARGS $PKGS + + if [ "$ACTION" == "install" ]; then + # dnf5 does not support "mark install" + if [ ${YUM} == "dnf" ] && [ -z "$DNF5" ]; then + # Make sure dnf won't autoremove these packages + dnf mark install $PKGS + fi + fi + $_xtrace + + # probably not the right place for this; but python-pip package on + # fedora/rh calls pip "pip-python" while the rest of the work + # expects it to be just called "pip" + for pkg in "$@"; do + if [ "$pkg" = "python-pip" ] ; then + alternatives --install /usr/bin/pip pip /usr/bin/pip-python 10 + fi + done +fi diff --git a/yum/cleanup.d/99-remove-yum-repo-conf b/yum/cleanup.d/99-remove-yum-repo-conf new file mode 100755 index 0000000..7795146 --- /dev/null +++ b/yum/cleanup.d/99-remove-yum-repo-conf @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# exit directly if DIB_YUM_REPO_CONF is not defined properly +if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then + echo "DIB_YUM_REPO_CONF is not set - no repo configurations will be cleaned up" + exit 0 +fi + +for file in $DIB_YUM_REPO_CONF; do + sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/$(basename $file) +done diff --git a/yum/element-deps b/yum/element-deps new file mode 100644 index 0000000..d888ae5 --- /dev/null +++ b/yum/element-deps @@ -0,0 +1 @@ +install-bin diff --git a/yum/extra-data.d/99-yum-repo-conf b/yum/extra-data.d/99-yum-repo-conf new file mode 100755 index 0000000..b5b6e98 --- /dev/null +++ b/yum/extra-data.d/99-yum-repo-conf @@ -0,0 +1,27 @@ +#!/bin/bash +# Add additional yum repo configuration(s) with $DIB_YUM_REPO_CONF + +if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# exit directly if DIB_YUM_REPO_CONF is not defined properly +if [ -z "${DIB_YUM_REPO_CONF:-}" ] ; then + echo "DIB_YUM_REPO_CONF is not set - no repo configuration will be copied in" + exit 0 +fi + +for file in $DIB_YUM_REPO_CONF; do + if [ ! -f $file ]; then + echo "$file is not a valid yum repo configuration file." + echo "You should assign a list of proper yum repo configuration" + echo "files in DIB_YUM_REPO_CONF." + exit 1 + fi + + # copy the yum repo configuration + sudo cp -L -f $file $TMP_MOUNT_PATH/etc/yum.repos.d + +done diff --git a/yum/post-install.d/99-reset-yum-conf b/yum/post-install.d/99-reset-yum-conf new file mode 100755 index 0000000..d4fad1c --- /dev/null +++ b/yum/post-install.d/99-reset-yum-conf @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +if [[ ${YUM} == "dnf" ]]; then + cfg=/etc/dnf/dnf.conf + cachedir=/var/cache/dnf +else + cfg=/etc/yum.conf + cachedir=/var/cache/yum +fi + +sed -i "/^keepcache/c\keepcache=0" $cfg +sed -i "/^cachedir/c\cachedir=${cachedir}" $cfg diff --git a/yum/pre-install.d/00-01-yum-keepcache b/yum/pre-install.d/00-01-yum-keepcache new file mode 100755 index 0000000..d471d0c --- /dev/null +++ b/yum/pre-install.d/00-01-yum-keepcache @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +if [[ ${YUM} == "dnf" ]]; then + # dnf is slightly different; doesn't have these by default + cfg=/etc/dnf/dnf.conf + echo "keepcache=1" >> $cfg + echo "cachedir=/tmp/yum" >> $cfg +else + cfg=/etc/yum.conf + sed -i 's/keepcache=0/keepcache=1/' $cfg + sed -i 's/cachedir=\/var\/cache\/yum/cachedir=\/tmp\/yum/' $cfg +fi + diff --git a/yum/pre-install.d/00-dnf-update b/yum/pre-install.d/00-dnf-update new file mode 100755 index 0000000..30560a5 --- /dev/null +++ b/yum/pre-install.d/00-dnf-update @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -eq 22 ]]; then + # we need dnf mark command from dnf 1.1.1, which is pretty recent + dnf --refresh update -y dnf +fi diff --git a/yum/pre-install.d/00-install-repo-package b/yum/pre-install.d/00-install-repo-package new file mode 100755 index 0000000..1a6ae1a --- /dev/null +++ b/yum/pre-install.d/00-install-repo-package @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +if [ -n "${DIB_YUM_REPO_PACKAGE:-}" ] ; then + rpm -ivh ${DIB_YUM_REPO_PACKAGE} +fi diff --git a/yum/pre-install.d/01-00-centos-python3 b/yum/pre-install.d/01-00-centos-python3 new file mode 100755 index 0000000..d023353 --- /dev/null +++ b/yum/pre-install.d/01-00-centos-python3 @@ -0,0 +1,27 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# Ensure the python3 interpreter and YAML libraries are installed +# early (even before package-installs, which is written in Python and +# uses YAML). + +if [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} == 7 ]]; then + # Our package map and install stuff doesn't have a way to say + # "install this from EPEL". So we hack in an install of it here + # from EPEL. Nothing else should have installed EPEL at this + # early stage. + yum install -y python3 + # NOTE(dpawlik) The epel-release package is not available in RHEL. + yum install -y ${DIB_CENTOS_7_PREINSTALL_EPEL_URL_PACKAGE:-https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm} + yum install -y python36-PyYAML + yum remove -y epel-release +elif [[ ${DISTRO_NAME} =~ (centos|rhel) && ${DIB_RELEASE} > 7 ]]; then + # For 8 and above ensure the "user" python3 package is installed + # so we have /usr/bin/python3 and pyyaml. + dnf install -y python3 python3-pyyaml +fi diff --git a/yum/pre-install.d/01-module-configuration b/yum/pre-install.d/01-module-configuration new file mode 100755 index 0000000..b498e47 --- /dev/null +++ b/yum/pre-install.d/01-module-configuration @@ -0,0 +1,24 @@ +#!/bin/bash +# +# When building RHEL overcloud images, certain modules may be preferred +# or even required. +# +# For example, container-tools:2.0 is notably a requirement when running +# the Train releases on RHEL 8.2. +# + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi + +set -eu +set -o pipefail + +export DIB_DNF_MODULE_STREAMS=${DIB_DNF_MODULE_STREAMS:-} + +if [ "${YUM}" == "dnf" ]; then + for m in ${DIB_DNF_MODULE_STREAMS}; do + ${YUM} -y module disable ${m/:*/} + ${YUM} -y module enable ${m} + done +fi diff --git a/yum/root.d/50-yum-cache b/yum/root.d/50-yum-cache new file mode 100755 index 0000000..542a87f --- /dev/null +++ b/yum/root.d/50-yum-cache @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] + +YUM_CACHE_DIR=$DIB_IMAGE_CACHE/yum +mkdir -p $YUM_CACHE_DIR + +sudo mkdir -p $TMP_MOUNT_PATH/tmp/yum +sudo mount --bind $YUM_CACHE_DIR $TMP_MOUNT_PATH/tmp/yum