From da41dfec80281cb2d8b3d5e48b171fa089da0e67 Mon Sep 17 00:00:00 2001
From: James Le Cuirot <jlecuirot@microsoft.com>
Date: Fri, 5 Jul 2024 10:11:34 +0100
Subject: [PATCH] Don't use repo snapshots for stage1 by updating seed the new
 way

This is what upstream Gentoo does. They would previously update the
entire seed, but this took a long time. Our seeds are much bigger, so we
kept repo snapshots to build stage1 against these instead. The new
method of only rebuilding packages with changed sub-slots is a good
compromise and removes the need to write stage1 hooks that selectively
catch the repository up.

This also avoids some conflicts by adding the `--ignore-world` option.
Gentoo seeds have nothing in @world. We have much more, but none of that
is needed for stage1.

This continues to exclude cross-*-cros-linux-gnu/* as that is not needed
for stage1. It now also excludes dev-lang/rust, because it is never a
DEPEND, so it would not break other packages in this way. It may fail to
run due to a sub-slot change in one of its own dependencies, but it is
also unlikely to be needed in stage1 and it is not configured to use the
system LLVM. If needs be, we could improve the behaviour of Portage's
@changed-subslot to respect `--with-bdeps`.

In my testing, it was unable to handle an SDK from 17 months ago, but
one from 7 months ago did work. In practise, we will always use a much
more recent one, which is far more likely to work.

Signed-off-by: James Le Cuirot <jlecuirot@microsoft.com>
---
 bootstrap_sdk                                 | 90 -------------------
 build_library/build_image_util.sh             |  3 +-
 build_library/catalyst.sh                     |  5 +-
 build_library/catalyst_sdk.sh                 |  9 --
 .../0000-bump-baselayout-coreos-overlay.sh    | 30 -------
 .../0000-glibc-crypt-portage-stable.sh        | 19 ----
 .../0001-update-profile-coreos-overlay.sh     | 25 ------
 .../0002-glibc-crypt-coreos-overlay.sh        | 53 -----------
 .../coreos/stage1_hooks/README.md             | 18 ----
 9 files changed, 4 insertions(+), 248 deletions(-)
 delete mode 100755 sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-bump-baselayout-coreos-overlay.sh
 delete mode 100755 sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-glibc-crypt-portage-stable.sh
 delete mode 100755 sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0001-update-profile-coreos-overlay.sh
 delete mode 100755 sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0002-glibc-crypt-coreos-overlay.sh
 delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/README.md

diff --git a/bootstrap_sdk b/bootstrap_sdk
index bf094e28a79..8a36665b64d 100755
--- a/bootstrap_sdk
+++ b/bootstrap_sdk
@@ -59,12 +59,6 @@ TYPE="flatcar-sdk"
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
 
-DEFINE_string stage1_portage_path "" \
-  "Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
-DEFINE_string stage1_overlay_path "" \
-  "Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
-
-
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
@@ -142,90 +136,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 #  by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
 #  outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
 
-stage_repo() {
-    local repo=${1}
-    local path=${2}
-    local dest=${3}
-    local update_seed_file=${4}
-
-    mkdir "$dest/$repo"
-
-    if [ -z "$path" ]; then
-        case "$repo" in
-            portage-stable) path=gentoo ;;
-            *) path=${repo} ;;
-        esac
-        cp -R --reflink=auto "/var/gentoo/repos/${path}/"* "$dest/${repo}/"
-        info "Using local SDK's ebuild repo '$repo' in stage 1."
-    else
-        cp -R --reflink=auto "${path}/"* "$dest/${repo}/"
-        info "Using custom path '$path' for ebuild repo '$repo' in stage 1."
-        info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
-    fi
-    (
-        set -euo pipefail
-        local repo_var hook name
-
-        # FLAGS_coreos_overlay for coreos-overlay
-        repo_var="FLAGS_${repo//-/_}"
-        shopt -s nullglob
-        for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${repo}.sh"; do
-            name=${hook##*/}
-            name=${name%"-${repo}.sh"}
-            info "Invoking stage1 ${repo} hook ${name} on ${dest}/${repo}"
-            "${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
-        done
-    )
-}
-
-build_stage1() {
-    # First, write out the default catalyst configuration files
-    write_configs
-
-    # Prepare local copies of both the "known-good" portage-stable and the
-    #  "known-good" coreos-overlay ebuild repos
-    local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
-    info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
-    rm -rf "$stage1_repos"
-    mkdir "$stage1_repos"
-
-    # If the file exists and is not empty, seed will be updated.
-    # Stage1 hooks may decide that the seed SDK needs updating.
-    local update_seed_file="${TEMPDIR}/update_seed"
-
-    # prepare ebuild repos for stage 1, either from the local SDK (default)
-    #  or from custom paths specified via command line flags
-    stage_repo "portage-stable" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
-    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"
-
-    # take the "portage directory" (portage-stable copy) snapshot
-    build_snapshot "$stage1_repos/portage-stable" "${FLAGS_version}-stage1"
-
-    # Update the stage 1 spec to use the "known-good" portage-stable snapshot
-    #  and coreos-overlay copy repository versions from above.
-    sed -i -e "s/^snapshot_treeish:.*/snapshot_treeish: $FLAGS_version-stage1/" \
-           -e "s,^repos:.*,repos: $stage1_repos/coreos-overlay," \
-        "$TEMPDIR/stage1.spec"
-
-    # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
-    if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
-        sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
-        echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
-            >>"$TEMPDIR/stage1.spec"
-    fi
-
-    rm -f "${update_seed_file}"
-
-    # Finally, build stage 1
-    build_stage stage1
-}
-
-if [[ "$STAGES" =~ stage1 ]]; then
-    build_stage1
-    STAGES="${STAGES/stage1/}"
-    SEED="${TYPE}/stage1-${ARCH}-latest"
-fi
-
 catalyst_build
 
 if [[ "$STAGES" =~ stage4 ]]; then
diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh
index 4abef8f0ef4..285ed519708 100755
--- a/build_library/build_image_util.sh
+++ b/build_library/build_image_util.sh
@@ -365,7 +365,7 @@ get_metadata() {
             local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
             if [ -n "${mirror}" ]; then
                 # Take only first mirror, those not working should be removed
-                local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
                 v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
             fi
             new_val+="${v} "
@@ -491,7 +491,6 @@ EOF
     local license_dirs=(
         "/mnt/host/source/src/third_party/coreos-overlay/licenses/"
         "/mnt/host/source/src/third_party/portage-stable/"
-        "/var/gentoo/repos/gentoo/licenses/"
         "none"
     )
     for license_file in ${license_list}; do
diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh
index ef08aa5f8b6..fab4a947c1e 100644
--- a/build_library/catalyst.sh
+++ b/build_library/catalyst.sh
@@ -104,7 +104,8 @@ catalyst_stage1() {
 cat <<EOF
 # stage1 packages aren't published, save in tmp
 pkgcache_path: ${TEMPDIR}/stage1-${ARCH}-packages
-update_seed: no
+update_seed: yes
+update_seed_command: --exclude cross-*-cros-linux-gnu/* --exclude dev-lang/rust --ignore-world y --ignore-built-slot-operator-deps y @changed-subslot
 EOF
 catalyst_stage_default 1
 }
@@ -225,7 +226,7 @@ write_configs() {
     info "Configuring Portage..."
     cp -r "${BUILD_LIBRARY_DIR}"/portage/ "${TEMPDIR}/"
 
-    ln -sfT '/var/gentoo/repos/coreos-overlay/coreos/user-patches' \
+    ln -sfT '/mnt/host/source/src/third_party/coreos-overlay/coreos/user-patches' \
         "${TEMPDIR}"/portage/patches
 }
 
diff --git a/build_library/catalyst_sdk.sh b/build_library/catalyst_sdk.sh
index ff3f3d4b1d4..742b04f86c9 100644
--- a/build_library/catalyst_sdk.sh
+++ b/build_library/catalyst_sdk.sh
@@ -20,12 +20,3 @@ for cross_chost in $(get_chost_list); do
     PKGDIR="$(portageq envvar PKGDIR)/crossdev" \
         install_cross_rust "${cross_chost}" ${clst_myemergeopts}
 done
-
-echo "Saving snapshot of repos for future SDK bootstraps"
-# Copy portage-stable and coreos-overlay, which are under
-# /mnt/host/source/src/third_party, into a local directory because they are
-# removed before archiving and we want to keep snapshots. These snapshots are
-# used by stage 1 in future bootstraps.
-mkdir -p /var/gentoo/repos/{gentoo,coreos-overlay}
-cp -R /mnt/host/source/src/third_party/portage-stable/* /var/gentoo/repos/gentoo/
-cp -R /mnt/host/source/src/third_party/coreos-overlay/* /var/gentoo/repos/coreos-overlay/
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-bump-baselayout-coreos-overlay.sh b/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-bump-baselayout-coreos-overlay.sh
deleted file mode 100755
index 9910b1cfeff..00000000000
--- a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-bump-baselayout-coreos-overlay.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-stage1_repo="${1}"
-new_repo="${2}"
-
-good_version="3.6.8-r10"
-stage1_version=''
-
-for f in "${stage1_repo}/sys-apps/baselayout/baselayout-"*'.ebuild'; do
-    f="${f##*/}"
-    if [[ "${f}" = *9999* ]]; then continue; fi
-    f="${f%.ebuild}"
-    f="${f#baselayout-}"
-    stage1_version="${f}"
-done
-
-if [[ -z "${stage1_version}" ]]; then exit 1; fi
-
-older_version=$(printf '%s\n' "${stage1_version}" "${good_version}" | sort -V | head -n 1)
-
-if [[ "${older_version}" = "${good_version}" ]]; then
-    # Stage1 version is equal or newer than the good version, nothing
-    # to do.
-    exit 0
-fi
-
-rm -rf "${stage1_repo}/sys-apps/baselayout"
-cp -a "${new_repo}/sys-apps/baselayout" "${stage1_repo}/sys-apps/baselayout"
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-glibc-crypt-portage-stable.sh b/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-glibc-crypt-portage-stable.sh
deleted file mode 100755
index b5838185a10..00000000000
--- a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0000-glibc-crypt-portage-stable.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-set -x
-set -euo pipefail
-
-stage1_repo=${1}
-new_repo=${2}
-update_seed_file=${3}
-
-cat=sys-libs
-pkg=libxcrypt
-
-if [[ -d "${stage1_repo}/${cat}/${pkg}" ]]; then
-    # libxcrypt is already a part of portage-stable, nothing to do
-    exit 0
-fi
-
-mkdir -p "${stage1_repo}/${cat}"
-cp -a "${new_repo}/${cat}/${pkg}" "${stage1_repo}/${cat}/${pkg}"
-echo x >"${update_seed_file}"
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0001-update-profile-coreos-overlay.sh b/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0001-update-profile-coreos-overlay.sh
deleted file mode 100755
index e9a01036aed..00000000000
--- a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0001-update-profile-coreos-overlay.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-set -x
-set -euo pipefail
-
-stage1_repo="${1}"
-new_repo="${2}"
-parent_file='profiles/coreos/amd64/parent'
-old_parent_line='portage-stable:default/linux/amd64/17.0/no-multilib/hardened'
-stage1_parent="${stage1_repo}/${parent_file}"
-new_parent="${new_repo}/${parent_file}"
-
-if [[ ! -e "${new_parent}" ]]; then
-    echo "no file '${parent_file}' in new repo, nothing to do"
-    exit 0
-fi
-
-if [[ ! -e "${stage1_parent}" ]]; then
-    echo "no file '${parent_file}' in stage1 repo, nothing to do"
-    exit 0
-fi
-
-if grep --quiet --fixed-strings --line-regexp --regexp="${old_parent_line}" -- "${stage1_parent}"; then
-    rm -f "${stage1_parent}"
-    cp -a "${new_parent}" "${stage1_parent}"
-fi
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0002-glibc-crypt-coreos-overlay.sh b/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0002-glibc-crypt-coreos-overlay.sh
deleted file mode 100755
index e8891c76a33..00000000000
--- a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/0002-glibc-crypt-coreos-overlay.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-set -x
-set -euo pipefail
-
-stage1_repo=${1}
-new_repo=${2}
-update_seed_file=${3}
-
-base_profile_dir='profiles/coreos/base'
-
-declare -A fixups_old=(
-    ['package.mask']='>=virtual/libcrypt-2'
-    ['package.unmask']='=virtual/libcrypt-1-r1'
-    ['package.use.force']='sys-libs/glibc crypt'
-    ['package.use.mask']='sys-libs/glibc -crypt'
-)
-
-declare -A fixups_new=(
-    ['package.mask']='>=virtual/libcrypt-2'
-    ['package.unmask']='<virtual/libcrypt-2'
-    ['package.use.force']='sys-libs/glibc crypt'
-    ['package.use.mask']='sys-libs/glibc -crypt'
-)
-
-for var_name in fixups_old fixups_new; do
-    declare -n fixups="${var_name}"
-
-    skip=''
-    for f in "${!fixups[@]}"; do
-        l=${fixups["${f}"]}
-        ff="${stage1_repo}/${base_profile_dir}/${f}"
-        if ! grep --quiet --fixed-strings --line-regexp --regexp="${l}" -- "${ff}"; then
-            # fixup not applicable, try next one
-            skip=x
-            break
-        fi
-    done
-
-    if [[ -n ${skip} ]]; then
-        unset -n fixups
-        continue
-    fi
-
-    for f in "${!fixups[@]}"; do
-        l=${fixups["${f}"]}
-        ff="${stage1_repo}/${base_profile_dir}/${f}"
-        ffb="${ff}.bak"
-        mv "${ff}" "${ffb}"
-        grep --invert-match --fixed-strings --line-regexp --regexp="${l}" -- "${ffb}" >"${ff}"
-    done
-    echo x >"${update_seed_file}"
-    exit 0
-done
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/README.md b/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/README.md
deleted file mode 100644
index d31163fe6c2..00000000000
--- a/sdk_container/src/third_party/coreos-overlay/coreos/stage1_hooks/README.md
+++ /dev/null
@@ -1,18 +0,0 @@
-The scripts in this directory are called by the SDK bootstrapping
-script when setting up the portage-stable and coreos-overlay repos for
-the stage1 build. The scripts are invoked with two arguments - a path
-to the stage1 repository, and a path to the current repository. The
-difference between the two is that the stage1 repository is a copy of
-a repository saved in the seed SDK (thus it's going to be an older
-version of the repository), whereas the current repository is a
-repository that will be a base of the new SDK. The idea here is that
-something in the stage1 repository may be too old, thus it should be
-replaced with its equivalent from the current repository.
-
-For more information about the bootstrap process, please see the
-`bootstrap_sdk` script in [the scripts
-repository](https://github.com/flatcar/scripts).
-
-The script for portage-stable should end with `-portage-stable.sh`,
-and the script for coreos-overlay with '-coreos-overlay.sh`. For
-example: `0000-replace-ROOTPATH-coreos-overlay.sh`.