Merge pull request #2115 from flatcar/chewi/catalyst-4

Upgrade to Catalyst 4 and don't use repo snapshots for stage1
flatcar · Jul 15, 2024 · 7c23476 · 7c23476
2 parents 354a0f2 + 186f311
commit 7c23476
Show file tree

Hide file tree

Showing 43 changed files with 580 additions and 423 deletions.
diff --git a/bootstrap_sdk b/bootstrap_sdk
@@ -4,30 +4,19 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
-# This uses Gentoo's catalyst for very thoroughly building images from
-# scratch. Using images based on this will eliminate some of the hackery
-# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
-# 
+# This uses Gentoo's catalyst for very thoroughly building images from scratch.
 # For reference the procedure it performs is this:
-# 
-# 1. snapshot: Grab a snapshot of the portage-stable repo from
-#     the current SDK's /var/lib/gentoo/repos/gentoo.
-#     Alternatively, check out a git ref specified via --portage-ref.
 #
-# 2. stage1: Using a "seed" tarball as a build environment, build a
-#     minimal root file system into a clean directory using ROOT=...
-#     and USE=-* The restricted USE flags are key be small and avoid
-#     circular dependencies.
+# 1. seed: Take a recent SDK, dev container, or custom tarball as a seed to
+#     build stage 1 with. Before proceeding, update relevant packages that have
+#     changed sub-slot to avoid missing library issues later in the build.
+#
+# 2. stage1: Using the above seed tarball as a build environment, build a
+#     minimal root file system into a clean directory using ROOT=... and USE=-*
+#     The restricted USE flags are key be small and avoid circular dependencies.
 #     NOTE that stage1 LACKS PROPER STAGE ISOLATION. Binaries produced in stage1
-#      will be linked against the SEED SDK libraries, NOT against libraries
-#      built in stage 1. See "stage_repo()" documentation further below for more.
-#     This stage uses:
-#     - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
-#        or a custom path via --stage1_portage_path command line option
-#     - coreos-overlay  from the SDK's /var/lib/gentoo/repos/coreos-overlay
-#        or a custom path via --stage1_overlay_path command line option
-#     Command line option refs need caution though, since
-#     stage1 must not contain updated ebuilds (see build_stage1 below).
+#     will be linked against the SEED SDK libraries, NOT against libraries built
+#     in stage 1.
 #
 # 3. stage2: Run portage-stable/scripts/bootstrap.sh
 #     This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
@@ -59,24 +48,17 @@ TYPE="flatcar-sdk"
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
 
-DEFINE_string stage1_portage_path "" \
-  "Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
-DEFINE_string stage1_overlay_path "" \
-  "Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
-
-
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
-target: stage4
 pkgcache_path: $BINPKGS
 stage4/packages: coreos-devel/sdk-depends
 stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
 stage4/root_overlay: ${ROOT_OVERLAY}
-stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
+stage4/empty: /root /var/cache/edb
 stage4/rm: /etc/machine-id /etc/resolv.conf
 EOF
-catalyst_stage_default
+catalyst_stage_default 4
 }
 
 # Switch to HTTP because early boostrap stages do not have SSL support.
@@ -118,7 +100,7 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 #
 # No package updates must happen in stage 1, so we use the portage-stable and
 #  coreos-overlay paths included with the current SDK (from the SDK chroot's
-#  /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
+#  /var/gentoo/repos/). "Current SDK" refers to the SDK we entered with
 #  'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
 #
 # Using ebuilds from the above mentioned sources will ensure that stage 1 builds
@@ -143,95 +125,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
 #  by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
 #  outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
 
-stage_repo() {
-    local repo=${1}
-    local path=${2}
-    local dest=${3}
-    local update_seed_file=${4}
-    local gitname="$repo"
-
-    if [ "$gitname" = "gentoo" ] ; then
-        gitname="portage-stable"
-    fi
-
-    if [ -z "$path" ]; then
-        cp -R "/var/gentoo/repos/${repo}" "$dest"
-        info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
-    else
-        mkdir "$dest/$repo"
-        cp -R "${path}/"* "$dest/${repo}/"
-        info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
-        info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
-    fi
-    (
-        set -euo pipefail
-        local repo_var hook name
-
-        # FLAGS_coreos_overlay for gitname coreos-overlay
-        repo_var="FLAGS_${gitname//-/_}"
-        shopt -s nullglob
-        for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
-            name=${hook##*/}
-            name=${name%"-${gitname}.sh"}
-            info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
-            "${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
-        done
-    )
-}
-
-build_stage1() {
-    # First, write out the default 4-stage catalyst configuration files
-    write_configs
-
-    # Prepare local copies of both the "known-good" portage-stable and the
-    #  "known-good" coreos-overlay ebuild repos
-    local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
-    info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
-    rm -rf "$stage1_repos"
-    mkdir "$stage1_repos"
-
-    # If the file exists and is not empty, seed will be updated.
-    # Stage1 hooks may decide that the seed SDK needs updating.
-    local update_seed_file="${TEMPDIR}/update_seed"
-
-    # prepare ebuild repos for stage 1, either from the local SDK (default)
-    #  or from custom paths specified via command line flags
-    stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
-    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"
-
-    # Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
-    #  This requires us to create a custom catalyst config to point it to the
-    #  repo copy we just created, for snapshotting.
-    catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
-    sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
-         "$TEMPDIR/catalyst-stage1.conf"
-    # take the "portage directory" (portage-stable copy) snapshot
-    build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1"
-
-    # Update the stage 1 spec to use the "known-good" portage-stable snapshot
-    #  and coreos-overlay copy repository versions from above.
-    sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
-           -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
-        "$TEMPDIR/stage1.spec"
-
-    # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
-    if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
-        sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
-        echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
-            >>"$TEMPDIR/stage1.spec"
-    fi
-    rm -f "${update_seed_file}"
-
-    # Finally, build stage 1
-    build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
-}
-
-if [[ "$STAGES" =~ stage1 ]]; then
-    build_stage1
-    STAGES="${STAGES/stage1/}"
-    SEED="${TYPE}/stage1-${ARCH}-latest"
-fi
-
 catalyst_build
 
 if [[ "$STAGES" =~ stage4 ]]; then

diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh
@@ -365,7 +365,7 @@ get_metadata() {
             local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
             if [ -n "${mirror}" ]; then
                 # Take only first mirror, those not working should be removed
-                local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
+                local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
                 v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
             fi
             new_val+="${v} "
@@ -490,8 +490,7 @@ EOF
     license_list="$(jq -r '.[] | "\(.licenses | .[])"' "${json_input}" | sort | uniq)"
     local license_dirs=(
         "/mnt/host/source/src/third_party/coreos-overlay/licenses/"
-        "/mnt/host/source/src/third_party/portage-stable/"
-        "/var/gentoo/repos/gentoo/licenses/"
+        "/mnt/host/source/src/third_party/portage-stable/licenses/"
         "none"
     )
     for license_file in ${license_list}; do