From 4606f7c0db1a9cf7b590e47f1c47d072868563b1 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 21 Aug 2024 10:46:12 -0700 Subject: [PATCH 1/2] docker_run_checks: run from source base for scripts problem: we can't access scripts from docker builds with the current configuration solution: switch to the active directory being the source root and referencing the dockerfile with -f --- src/test/docker/docker-run-checks.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/docker/docker-run-checks.sh b/src/test/docker/docker-run-checks.sh index 6fdbb3fa3..498442f36 100755 --- a/src/test/docker/docker-run-checks.sh +++ b/src/test/docker/docker-run-checks.sh @@ -156,7 +156,8 @@ checks_group "Building image $IMAGE for user $USER $(id -u) group=$(id -g)" \ --build-arg FLUX_SECURITY_VERSION=$FLUX_SECURITY_VERSION \ ${BUILD_ARG:- } \ -t ${BUILD_IMAGE} \ - ${DOCKERFILE} \ + -f ${DOCKERFILE}/Dockerfile \ + . \ || die "docker build failed" if [[ -n "$MOUNT_HOME_ARGS" ]]; then From 564d1f56d48edd548f1f14deaa90f5b5bac59a96 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 21 Aug 2024 10:35:30 -0700 Subject: [PATCH 2/2] deps: add install-deps.sh script, use in docker problem: it's easy to accidentally miss adding or removing a dependency in the various dockerfiles and the readme solution: much like core, encode them as a script, then use it --- README.md | 14 ++---- scripts/add_docker_user.sh | 14 ++++++ scripts/install-deps.sh | 67 +++++++++++++++++++++++++++++ src/test/docker/alpine/Dockerfile | 19 ++------ src/test/docker/bookworm/Dockerfile | 23 +++------- src/test/docker/el9/Dockerfile | 27 +++--------- src/test/docker/fedora40/Dockerfile | 24 ++--------- src/test/docker/jammy/Dockerfile | 23 +++------- src/test/docker/noble/Dockerfile | 22 +++------- 9 files changed, 114 insertions(+), 119 deletions(-) create mode 100755 scripts/add_docker_user.sh create mode 100755 scripts/install-deps.sh diff --git a/README.md b/README.md index 78f749dbc..2d2e2416f 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ Fluxion requires an installed flux-core package. Instructions for installing flux-core can be found in [the flux-core README](https://github.com/flux-framework/flux-core/blob/master/README.md). +To install our other dependencies on a dnf, apk or apt-using distro you should +be able to use ./scripts/install-deps.sh to install all necessary dependencies. +
Click to expand and see our full dependency table @@ -60,15 +63,6 @@ libedit-devel | libedit-dev | >= 3.0 | python3-pyyaml | python3-yaml | >= 3.10 | yaml-cpp-devel | libyaml-cpp-dev | >= 0.5.1 | -*Note 1 - Boost package versions 1.54-1.58 contain a bug that -leads to compilation error.* - -The following optional dependencies enable additional testing: - -**redhat** | **ubuntu** | **version** ----------- | ---------- | ----------- -valgrind-devel | valgrind | -jq | jq |
##### Installing RedHat/CentOS Packages @@ -156,7 +150,7 @@ from the source directory or use the usual ctest options to filter by regex: ```bash $ cd build/t -$ ../../t/t9001-golang-basic.t +$ ../../t/t9001-golang-basic.t ok 1 - match allocate 1 slot: 1 socket: 1 core (pol=default) ok 2 - match allocate 2 slots: 2 sockets: 5 cores 1 gpu 6 memory # passed all 2 test(s) diff --git a/scripts/add_docker_user.sh b/scripts/add_docker_user.sh new file mode 100755 index 000000000..dd8f6002b --- /dev/null +++ b/scripts/add_docker_user.sh @@ -0,0 +1,14 @@ +#!/bin/sh +if test "$USER" != "flux"; then + sudo groupadd -g $UID $USER + sudo useradd -g $USER -u $UID -d /home/$USER -m $USER + sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" + case "$ID" in + ubuntu|debian) + sudo adduser $USER sudo + ;; + fedora|rocky|alma|rhel|centos|alpine) + sudo usermod -G wheel $USER + ;; + esac +fi diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh new file mode 100755 index 000000000..e85353898 --- /dev/null +++ b/scripts/install-deps.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +if [ -z "$ID" ] ; then + while IFS== read -r key value; do + value="${value%\"}" + value="${value#\"}" + export "$key"="$value" + done > /etc/sudoers" \ - && sudo usermod -G wheel $USER; \ - fi +RUN /scripts/add_docker_user.sh USER $USER WORKDIR /home/$USER diff --git a/src/test/docker/bookworm/Dockerfile b/src/test/docker/bookworm/Dockerfile index b28a1803b..bb31df984 100644 --- a/src/test/docker/bookworm/Dockerfile +++ b/src/test/docker/bookworm/Dockerfile @@ -4,27 +4,16 @@ ARG USER=flux ARG UID=1000 USER root +# copy scripts into image +COPY scripts/ /scripts # Install extra buildrequires for flux-sched: -RUN apt-get update -RUN apt-get -qq install -y --no-install-recommends \ - libboost-graph-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-regex-dev \ - libyaml-cpp-dev \ - libedit-dev \ - ninja-build \ - python3-yaml +RUN sudo apt update \ + && sudo /scripts/install-deps.sh -y \ + && sudo rm -rf /var/lib/apt/lists/* # Add configured user to image with sudo access: # -RUN \ - if test "$USER" != "flux"; then \ - groupadd -g $UID $USER \ - && useradd -g $USER -u $UID -d /home/$USER -m $USER \ - && sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ - && adduser $USER sudo ; \ - fi +RUN /scripts/add_docker_user.sh USER $USER WORKDIR /home/$USER diff --git a/src/test/docker/el9/Dockerfile b/src/test/docker/el9/Dockerfile index 109a4a8e7..85709878d 100644 --- a/src/test/docker/el9/Dockerfile +++ b/src/test/docker/el9/Dockerfile @@ -4,32 +4,15 @@ ARG USER=flux ARG UID=1000 USER root +# copy scripts into image +COPY scripts/ /scripts # Install extra buildrequires for flux-sched: -RUN sudo dnf -y install \ - # We need this to allow curl or others to upgrade base packages - --allowerasing \ - boost-devel \ - boost-graph \ - boost-system \ - boost-filesystem \ - boost-regex \ - python3-pyyaml \ - yaml-cpp-devel \ - libedit-devel \ - ninja-build \ - gcc-toolset-13-{gcc,gcc-c++,gdb} \ - cmake \ - curl +RUN sudo /scripts/install-deps.sh -y --allowerasing gcc-toolset-13-{gcc,gcc-c++,gdb} \ + && sudo dnf clean all # Add configured user to image with sudo access: # -RUN \ - if test "$USER" != "flux"; then \ - sudo groupadd -g $UID $USER \ - && sudo useradd -g $USER -u $UID -d /home/$USER -m $USER \ - && sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ - && sudo usermod -G wheel $USER; \ - fi +RUN /scripts/add_docker_user.sh USER $USER WORKDIR /home/$USER diff --git a/src/test/docker/fedora40/Dockerfile b/src/test/docker/fedora40/Dockerfile index df18026db..83b657754 100644 --- a/src/test/docker/fedora40/Dockerfile +++ b/src/test/docker/fedora40/Dockerfile @@ -5,31 +5,15 @@ LABEL maintainer="Tom Scogland " ARG USER=flux ARG UID=1000 +# copy scripts into image +COPY scripts/ /scripts # Install flux-sched dependencies -RUN sudo dnf -y install \ - boost-devel \ - boost-graph \ - boost-system \ - boost-filesystem \ - boost-regex \ - python3-pyyaml \ - yaml-cpp-devel \ - libedit-devel \ - ninja-build \ - cmake \ - clang \ - lcov \ +RUN sudo /scripts/install-deps.sh -y \ && sudo dnf clean all # Add configured user to image with sudo access: # -RUN \ - if test "$USER" != "flux"; then \ - sudo groupadd -g $UID $USER \ - && sudo useradd -g $USER -u $UID -d /home/$USER -m $USER \ - && sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ - && sudo usermod -G wheel $USER; \ - fi +RUN /scripts/add_docker_user.sh USER $USER WORKDIR /home/$USER diff --git a/src/test/docker/jammy/Dockerfile b/src/test/docker/jammy/Dockerfile index b8c3e18c2..4b5477f76 100644 --- a/src/test/docker/jammy/Dockerfile +++ b/src/test/docker/jammy/Dockerfile @@ -3,27 +3,16 @@ FROM fluxrm/flux-core:jammy ARG USER=flux ARG UID=1000 +# copy scripts into image +COPY scripts/ /scripts # Install extra buildrequires for flux-sched: -RUN sudo apt-get update -RUN sudo apt-get -qq install -y --no-install-recommends \ - libboost-graph-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-regex-dev \ - python3-yaml \ - libyaml-cpp-dev \ - libedit-dev \ - ninja-build +RUN sudo apt update \ + && sudo /scripts/install-deps.sh -y \ + && sudo rm -rf /var/lib/apt/lists/* # Add configured user to image with sudo access: # -RUN \ - if test "$USER" != "flux"; then \ - sudo groupadd -g $UID $USER \ - && sudo useradd -g $USER -u $UID -d /home/$USER -m $USER \ - && sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ - && sudo adduser $USER sudo ; \ - fi +RUN /scripts/add_docker_user.sh USER $USER WORKDIR /home/$USER diff --git a/src/test/docker/noble/Dockerfile b/src/test/docker/noble/Dockerfile index bc1cbd152..10c2f4beb 100644 --- a/src/test/docker/noble/Dockerfile +++ b/src/test/docker/noble/Dockerfile @@ -4,28 +4,16 @@ ARG USER=flux ARG UID=1000 USER root +# copy scripts into image +COPY scripts/ /scripts # Install extra buildrequires for flux-sched: -RUN sudo apt-get update -RUN sudo apt-get -qq install -y --no-install-recommends \ - libboost-graph-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-regex-dev \ - python3-yaml \ - libyaml-cpp-dev \ - libedit-dev \ - ninja-build \ +RUN sudo apt update \ + && sudo /scripts/install-deps.sh -y \ && sudo rm -rf /var/lib/apt/lists/* # Add configured user to image with sudo access: # -RUN \ - if test "$USER" != "flux"; then \ - sudo groupadd -g $UID $USER \ - && sudo useradd -g $USER -u $UID -d /home/$USER -m $USER \ - && sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ - && sudo adduser $USER sudo ; \ - fi +RUN /scripts/add_docker_user.sh USER $USER WORKDIR /home/$USER