diff --git a/ansible/playbooks/README.md b/ansible/playbooks/README.md index 12ddd783..8a4a235e 100644 --- a/ansible/playbooks/README.md +++ b/ansible/playbooks/README.md @@ -42,8 +42,10 @@ Before running the playbook, make sure the following settings are correct, and o | prefix_required_space | Minimal amount of disk space that is required for the Gentoo Prefix bootstrap | | prefix_snapshot_url | Directory (served over http(s)) containing snapshot files | | prefix_snapshot_version | Date (`YYYYMMDD`) of the Portage snapshot file for the Prefix installation | -| prefix_python_targets | String consisting of [Gentoo Python targets](https://wiki.gentoo.org/wiki/Project:Python/PYTHON_TARGETS) Python targets used for the Prefix installation | | prefix_user_defined_trusted_dirs | List of paths to the user defined trusted dirs for glibc | +| prefix_mask_packages | Contents of a [package.mask file](https://wiki.gentoo.org/wiki//etc/portage/package.mask) that should be used during the bootstrap | +| prefix_use_builtin_bootstrap | Use the container's built-in bootstrap script? | +| prefix_custom_bootstrap_script | Dictionary with the `local` source and `remote` destination of the bootstrap script | | prefix_singularity_command | Singularity command for launching the container with the bootstrap script | | prefix_source | Singularity container path used for the Prefix installtion | | prefix_source_options | Arguments to be passed to the Prefix bootstrap script | @@ -75,3 +77,7 @@ ip-or-hostname-of-your-stratum0 eessi_host_arch=x86_64 eessi_host_os=linux The `eessi_host_arch` corresponds to the architecture of the machine that executes the playbook and for which this compatibility layer has to be built, e.g. `x86_64`, `aarch64`, or `ppc64le`. Similarly, `eessi_host_os` should refer to the operating system of the machine, and should be set to either `linux` or `macos`. + +Note that, by default, the bootstrap script from the root directory of this git repository will be used, unless you set `prefix_use_builtin_bootstrap` to `yes` +(in which case the bootstrap script included in the Prefix build container will be used). +A different bootstrap script can be used by changing `prefix_custom_bootstrap_script.local` to a local path. diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index a5bd0895..866ee26b 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -1,6 +1,6 @@ # Defaults file for the compatibility layer role. --- -eessi_version: "2021.06" +eessi_version: "2021.12" custom_overlays: - name: eessi @@ -18,17 +18,30 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/{{ eessi_version }}/compat/{{ # How to build the prefix prefix_required_space: 15 GB prefix_snapshot_url: https://eessi-gentoo-snapshot.s3-eu-west-1.amazonaws.com -prefix_snapshot_version: 20210607 +prefix_snapshot_version: 20211120 prefix_user_defined_trusted_dirs: - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib" -prefix_singularity_command: "singularity run -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}" +prefix_mask_packages: | + # avoid glibc 2.34, as it's causing issues with the bootstrap, and it's not compatible with CUDA 11. + # see https://github.com/EESSI/compatibility-layer/issues/137 + https://bugs.gentoo.org/824482 + >=sys-libs/glibc-2.34 + # avoid libgcrypt 1.9.4 due to compiler errros on ppc64le, + # see https://github.com/EESSI/compatibility-layer/issues/134 + https://bugs.gentoo.org/825722 + =dev-libs/libgcrypt-1.9.4 +prefix_use_builtin_bootstrap: no +prefix_custom_bootstrap_script: + local: "{{ playbook_dir }}/../../bootstrap-prefix.sh" + remote: /tmp/bootstrap-prefix.sh +prefix_singularity_command: "singularity exec -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}" prefix_source: "docker://ghcr.io/eessi/bootstrap-prefix:centos8" prefix_source_options: "{{ gentoo_prefix_path }} noninteractive" prefix_install: >- SINGULARITYENV_USE_CPU_CORES={{ ansible_processor_vcpus }} SINGULARITYENV_CUSTOM_SNAPSHOT_URL="{{ prefix_snapshot_url }}" SINGULARITYENV_CUSTOM_SNAPSHOT_VERSION="{{ prefix_snapshot_version }}" - {{ prefix_singularity_command }} {{ prefix_source }} {{ prefix_source_options }} + {{ prefix_singularity_command }} {{ prefix_source }} + {{ prefix_use_builtin_bootstrap | ternary('/usr/local/bin/bootstrap-prefix.sh', prefix_custom_bootstrap_script.remote) }} + {{ prefix_source_options }} # Logging eessi_log_dir: "/tmp/eessi-logs" diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml index 531ee308..8bf3d77e 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml @@ -61,6 +61,33 @@ tags: - build_prefix +- name: "Copy custom bootstrap script" + copy: + src: "{{ prefix_custom_bootstrap_script.local }}" + dest: "{{ prefix_custom_bootstrap_script.remote }}" + mode: 0755 + when: not prefix_use_builtin_bootstrap + tags: + - build_prefix + +- name: "Create $EPREFIX/etc/portage directory" + file: + path: "{{ gentoo_prefix_path }}/etc/portage" + state: directory + mode: 0755 + when: prefix_mask_packages is defined and prefix_mask_packages | length > 0 + tags: + - build_prefix + +- name: "Mask packages for the bootstrap" + copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask" + content: "{{ prefix_mask_packages }}" + mode: 0644 + when: prefix_mask_packages is defined and prefix_mask_packages | length > 0 + tags: + - build_prefix + - name: "Give {{ username_on_host.stdout }} recursive user and group ownership of {{ gentoo_prefix_path }}" file: dest: "{{ gentoo_prefix_path }}" @@ -81,8 +108,12 @@ stat: path: "{{ gentoo_prefix_path }}/startprefix" register: startprefix + tags: + - build_prefix - name: "Fail if startprefix script has not been created" fail: msg: "The resulting Gentoo Prefix installation does not have a startprefix script. Something went wrong!" when: not startprefix.stat.exists + tags: + - build_prefix diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml index 0650d73d..0f1ab4f9 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml @@ -1,15 +1,12 @@ # Make sure that glibc is always compiled with a user-defined-trusted-dirs option --- -- name: Find all strings in libc library - command: "strings {{ gentoo_prefix_path }}/usr/lib64/libc.a" - register: libc_strings +- name: Check the EXTRA_EMAKE flags of glibc + command: "equery has --package glibc EXTRA_EMAKE" + changed_when: false + ignore_errors: yes + register: glibc_extra_emake when: eessi_host_os == "linux" -- name: Find user defined trusted dirs in libc strings output - set_fact: match='{{ libc_strings.stdout | regex_search("\n" + item + "/?\n") | default('', True) | string | length>0 }}' - with_items: "{{ prefix_user_defined_trusted_dirs }}" - register: trusted_dirs_in_libc - - name: (Re)install glibc with the user-defined-trusted-dirs option portage: package: sys-libs/glibc @@ -20,7 +17,7 @@ EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}" when: - eessi_host_os == "linux" - - trusted_dirs_in_libc.results | selectattr('ansible_facts.match', 'equalto', False) | list | length>0 + - glibc_extra_emake.stdout != "user-defined-trusted-dirs=" + ":".join(prefix_user_defined_trusted_dirs) - name: Create portage env directory file: diff --git a/bootstrap-prefix.sh b/bootstrap-prefix.sh index d7204e6a..1776e0a0 100755 --- a/bootstrap-prefix.sh +++ b/bootstrap-prefix.sh @@ -105,7 +105,7 @@ efetch() { configure_cflags() { export CPPFLAGS="-I${ROOT}/tmp/usr/include" - + case ${CHOST} in *-darwin*) export LDFLAGS="-Wl,-search_paths_first -L${ROOT}/tmp/usr/lib" @@ -203,9 +203,9 @@ configure_toolchain() { ;; *"Apple clang version "*|*"Apple LLVM version "*) # recent binutils-apple are hard to build (C++11 - # features, and cmake buildsystem) so avoid going + # features, and cmake build system) so avoid going # there, the system ld is good enough to bring us to - # stage3, after which system set will take care of + # stage3, after which the @system set will take care of # the rest linker=sys-devel/native-cctools ;; @@ -319,8 +319,15 @@ bootstrap_setup() { echo "CONFIG_SHELL=\"${ROOT}/bin/bash\"" echo "DISTDIR=\"${DISTDIR:-${ROOT}/var/cache/distfiles}\"" if is-rap ; then - echo "# sandbox does not work well on Prefix, bug 490246" + echo "# sandbox does not work well on Prefix, bug #490246" echo 'FEATURES="${FEATURES} -usersandbox -sandbox"' + # bug #759424 + [[ -n ${STABLE_PREFIX} ]] && \ + echo 'ACCEPT_KEYWORDS="${ARCH} -~${ARCH}"' + else + echo "# last mirror is for Prefix specific distfiles, you" + echo "# might experience fetch failures if you remove it" + echo "GENTOO_MIRRORS=\"${GENTOO_MIRRORS} ${DISTFILES_PFX}\"" fi if [[ ${FS_INSENSITIVE} == 1 ]] ; then echo @@ -337,10 +344,24 @@ bootstrap_setup() { fi if is-rap ; then - [[ -f ${ROOT}/etc/passwd ]] || getent passwd > "${ROOT}"/etc/passwd || \ - ln -sf {,"${ROOT}"}/etc/passwd - [[ -f ${ROOT}/etc/group ]] || getent group > "${ROOT}"/etc/group || \ - ln -sf {,"${ROOT}"}/etc/group + if [[ ! -f ${ROOT}/etc/passwd ]]; then + if grep -q $(id -un) /etc/passwd; then + ln -sf {,"${ROOT}"}/etc/passwd + else + getent passwd > "${ROOT}"/etc/passwd + # add user if it's not in /etc/passwd, bug #766417 + getent passwd $(id -un) >> "${ROOT}"/etc/passwd + fi + fi + if [[ ! -f ${ROOT}/etc/group ]]; then + if grep -q $(id -gn) /etc/group; then + ln -sf {,"${ROOT}"}/etc/group + else + getent group > "${ROOT}"/etc/group + # add group if it's not in /etc/group, bug #766417 + getent group $(id -gn) >> "${ROOT}"/etc/group + fi + fi [[ -f ${ROOT}/etc/resolv.conf ]] || ln -s {,"${ROOT}"}/etc/resolv.conf [[ -f ${ROOT}/etc/hosts ]] || cp {,"${ROOT}"}/etc/hosts local profile_linux=default/linux/ARCH/17.0/prefix/$(profile-kernel) @@ -361,15 +382,31 @@ bootstrap_setup() { rev=${CHOST##*darwin} profile="prefix/darwin/macos/10.$((rev - 4))/x64" ;; - x86_64-apple-darwin2[0123456789]) + x86_64-apple-darwin20) # Big Sur is 11.0 rev=${CHOST##*darwin} profile="prefix/darwin/macos/11.$((rev - 20))/x64" ;; - arm64-apple-darwin2[0123456789]) + x86_64-apple-darwin2[123456789]) + # Monterey is 12.0 + rev=${CHOST##*darwin} + profile="prefix/darwin/macos/12.$((rev - 21))/x64" + ;; + arm64-apple-darwin20) rev=${CHOST##*darwin} profile="prefix/darwin/macos/11.$((rev - 20))/arm64" ;; + # TODO: Come up with something better for macOS 11+ + x86_64-apple-darwin2[123456789]) + # Monterey is 12.0 + rev=${CHOST##*darwin} + profile="prefix/darwin/macos/12.$((rev - 21))/x64" + ;; + arm64-apple-darwin2[123456789]) + # Monterey is 12.0 + rev=${CHOST##*darwin} + profile="prefix/darwin/macos/12.$((rev - 21))/arm64" + ;; i*86-pc-linux-gnu) profile=${profile_linux/ARCH/x86} ;; @@ -410,7 +447,7 @@ bootstrap_setup() { x86_64-pc-cygwin*) profile="prefix/windows/cygwin/x64" ;; - *) + *) eerror "UNKNOWN ARCH: You need to set up a make.profile symlink to a" eerror "profile in ${PORTDIR} for your CHOST ${CHOST}" exit 1 @@ -440,7 +477,7 @@ bootstrap_setup() { profile=${PROFILE_BASE:-prefix}/${profile#prefix/}${PROFILE_VARIANT:+/${PROFILE_VARIANT}} if [[ -n ${profile} && ! -e ${ROOT}/etc/portage/make.profile ]] ; then local fullprofile="${PORTDIR}/profiles/${profile}" - + ln -s "${fullprofile}" "${ROOT}"/etc/portage/make.profile einfo "Your profile is set to ${fullprofile}." fi @@ -540,7 +577,7 @@ do_tree() { bootstrap_tree() { # RAP uses the latest gentoo main repo snapshot to bootstrap. is-rap && LATEST_TREE_YES=1 - local PV="20210213" + local PV="20211105" if is-rap ; then do_tree "${CUSTOM_SNAPSHOT_URL:-$SNAPSHOT_URL}" portage-${CUSTOM_SNAPSHOT_VERSION:-latest}.tar.bz2 else @@ -607,12 +644,12 @@ bootstrap_portage() { # STABLE_PV that is known to work. Intended for power users only. ## It is critical that STABLE_PV is the lastest (non-masked) version that is ## included in the snapshot for bootstrap_tree. - STABLE_PV="3.0.12.0.2" - [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.12.0.2" + STABLE_PV="3.0.21" + [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.21" PV="${TESTING_PV:-${STABLE_PV}}" A=prefix-portage-${PV}.tar.bz2 einfo "Bootstrapping ${A%.tar.*}" - + efetch ${DISTFILES_URL}/${A} || return 1 einfo "Unpacking ${A%.tar.*}" @@ -661,7 +698,7 @@ bootstrap_portage() { cd "${ROOT}" rm -Rf ${ptmp} >& /dev/null - # Some people will skip the tree() step and hence var/log is not created + # Some people will skip the tree() step and hence var/log is not created # As such, portage complains.. mkdir -p "${ROOT}"/tmp/var/log @@ -942,9 +979,9 @@ bootstrap_gnu() { einfo "${A%.tar.*} successfully bootstrapped" } -PYTHONMAJMIN=3.8 # keep this number in line with PV below for stage1,2 +PYTHONMAJMIN=3.9 # keep this number in line with PV below for stage1,2 bootstrap_python() { - PV=3.8.6 + PV=3.9.6 A=Python-${PV}.tar.xz einfo "Bootstrapping ${A%.tar.*}" @@ -1010,8 +1047,8 @@ bootstrap_python() { -e 's/KQUEUE/KQUEUE_DISABLED/' \ configure # fixup thread id detection - efetch "http://dev.gentoo.org/~grobian/distfiles/python-3.8.6-darwin9.patch" - patch -p1 < "${DISTDIR}"/python-3.8.6-darwin9.patch + efetch "https://dev.gentoo.org/~sam/distfiles/dev-lang/python/python-3.9.6-darwin9_pthreadid.patch" + patch -p1 < "${DISTDIR}"/python-3.9.6-darwin9_pthreadid.patch ;; (arm64-*-darwin*) # Teach Python a new trick (arm64) @@ -1153,7 +1190,7 @@ bootstrap_cmake_core() { emake install || return 1 # we need sysroot crap to build cmake itself, but it makes trouble - # lateron, so kill it in the installed version + # later on, so kill it in the installed version ver=${A%-*} ; ver=${ver%.*} sed -i -e '/cmake_gnu_set_sysroot_flag/d' \ "${ROOT}"/tmp/usr/share/${ver}/Modules/Platform/Apple-GNU-*.cmake || die @@ -1365,7 +1402,7 @@ bootstrap_stage_host_gentoo() { einfo "are bootstrapping prefix-rpath. Do nothing." return 0 fi - + if [[ ! -L ${ROOT}/tmp ]] ; then if [[ -e ${ROOT}/tmp ]] ; then einfo "${ROOT}/tmp exists and is not a symlink to ${HOST_GENTOO_EROOT}" @@ -1602,6 +1639,7 @@ do_emerge_pkgs() { -python -qmanifest -qtegrity -readline + -sanitize bootstrap clang internal-glib @@ -2002,6 +2040,7 @@ bootstrap_stage3() { pkgs=( sys-apps/attr sys-libs/libcap + sys-libs/libxcrypt ) BOOTSTRAP_RAP=yes \ USE="${USE} -pam" \ @@ -2041,7 +2080,7 @@ bootstrap_stage3() { # in addition, avoid collisions rm -Rf "${ROOT}"/tmp/usr/lib/python${PYTHONMAJMIN}/site-packages/clang - # try to get ourself out of the mudd, bug #575324 + # try to get ourself out of the mud, bug #575324 EXTRA_ECONF="--disable-compiler-version-checks $(rapx '--disable-lto --disable-bootstrap')" \ GCC_MAKE_TARGET=$(rapx all) \ MYCMAKEARGS="-DCMAKE_USE_SYSTEM_LIBRARY_LIBUV=OFF" \ @@ -2129,10 +2168,10 @@ bootstrap_stage3() { # Avoid glib compiling for Cocoa libs if it finds them, since we're # still with an old llvm that may not understand the system headers # very well on Darwin (-DGNUSTEP_BASE_VERSION hack) - einfo "running emerge -u system" - estatus "stage3: emerge -u system" + einfo "running emerge -uDNv system" + estatus "stage3: emerge -uDNv system" CPPFLAGS="-DGNUSTEP_BASE_VERSION" \ - CFLAGS= CXXFLAGS= emerge --color n -u -v system || return 1 + CFLAGS= CXXFLAGS= emerge --color n -uDNv system || return 1 # remove anything that we don't need (compilers most likely) einfo "running emerge --depclean" @@ -2694,6 +2733,38 @@ EOF esac fi + # The experimental support for Stable Prefix. + # When expanding this to other CHOSTs, don't forget to update + # make.conf generation in bootstrap_setup(). + # TODO: Consider at some point removing the ~ARCH override from + # profiles/features/prefix/standalone/make.defaults. + # https://bugs.gentoo.org/759424 + if is-rap ; then + if [[ "${CHOST}" == x86_64-pc-linux-gnu ]]; then + cat <