Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to Catalyst 4 and don't use repo snapshots for stage1 #2115

Merged
merged 8 commits into from
Jul 15, 2024
Merged

Upgrade to Catalyst 4 and don't use repo snapshots for stage1 #2115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
15cfc5b
app-arch/pixz: Import from Gentoo
chewi Jul 8, 2024
82c6aac
sys-fs/squashfs-tools-ng: Import from Gentoo
chewi Jul 8, 2024
6d96f2a
dev-util/catalyst: Sync with Gentoo
chewi Jul 8, 2024
8d8be98
Drop Python path override hack in profile.bashrc
chewi Jun 28, 2024
1d7d53f
Upgrade to Catalyst 4
chewi Jul 8, 2024
872ea6d
Don't use repo snapshots for stage1 by updating seed the new way
chewi Jul 5, 2024
a783cc2
Fix up SDK repo configuration to use new coreos-overlay name on startup
chewi Jul 11, 2024
186f311
Upgrade to (at least) Catalyst 4 before trying to run Catalyst
chewi Jul 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 13 additions & 120 deletions bootstrap_sdk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,19 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This uses Gentoo's catalyst for very thoroughly building images from
# scratch. Using images based on this will eliminate some of the hackery
# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
#
# This uses Gentoo's catalyst for very thoroughly building images from scratch.
# For reference the procedure it performs is this:
#
# 1. snapshot: Grab a snapshot of the portage-stable repo from
# the current SDK's /var/lib/gentoo/repos/gentoo.
# Alternatively, check out a git ref specified via --portage-ref.
#
# 2. stage1: Using a "seed" tarball as a build environment, build a
# minimal root file system into a clean directory using ROOT=...
# and USE=-* The restricted USE flags are key be small and avoid
# circular dependencies.
# 1. seed: Take a recent SDK, dev container, or custom tarball as a seed to
# build stage 1 with. Before proceeding, update relevant packages that have
# changed sub-slot to avoid missing library issues later in the build.
#
# 2. stage1: Using the above seed tarball as a build environment, build a
# minimal root file system into a clean directory using ROOT=... and USE=-*
# The restricted USE flags are key be small and avoid circular dependencies.
# NOTE that stage1 LACKS PROPER STAGE ISOLATION. Binaries produced in stage1
# will be linked against the SEED SDK libraries, NOT against libraries
# built in stage 1. See "stage_repo()" documentation further below for more.
# This stage uses:
# - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
# or a custom path via --stage1_portage_path command line option
# - coreos-overlay from the SDK's /var/lib/gentoo/repos/coreos-overlay
# or a custom path via --stage1_overlay_path command line option
# Command line option refs need caution though, since
# stage1 must not contain updated ebuilds (see build_stage1 below).
# will be linked against the SEED SDK libraries, NOT against libraries built
# in stage 1.
#
# 3. stage2: Run portage-stable/scripts/bootstrap.sh
# This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
Expand Down Expand Up @@ -59,24 +48,17 @@ TYPE="flatcar-sdk"
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1


DEFINE_string stage1_portage_path "" \
"Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
DEFINE_string stage1_overlay_path "" \
"Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"


## Define the stage4 config template
catalyst_stage4() {
cat <<EOF
target: stage4
pkgcache_path: $BINPKGS
stage4/packages: coreos-devel/sdk-depends
stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
stage4/root_overlay: ${ROOT_OVERLAY}
stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
stage4/empty: /root /var/cache/edb
stage4/rm: /etc/machine-id /etc/resolv.conf
EOF
catalyst_stage_default
catalyst_stage_default 4
}

# Switch to HTTP because early boostrap stages do not have SSL support.
Expand Down Expand Up @@ -118,7 +100,7 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
#
# No package updates must happen in stage 1, so we use the portage-stable and
# coreos-overlay paths included with the current SDK (from the SDK chroot's
# /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
# /var/gentoo/repos/). "Current SDK" refers to the SDK we entered with
# 'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
#
# Using ebuilds from the above mentioned sources will ensure that stage 1 builds
Expand All @@ -143,95 +125,6 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
# by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
# outdated "seed tarball" libraries which have been updated to newer versions in stage 1.

stage_repo() {
local repo=${1}
local path=${2}
local dest=${3}
local update_seed_file=${4}
local gitname="$repo"

if [ "$gitname" = "gentoo" ] ; then
gitname="portage-stable"
fi

if [ -z "$path" ]; then
cp -R "/var/gentoo/repos/${repo}" "$dest"
info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
else
mkdir "$dest/$repo"
cp -R "${path}/"* "$dest/${repo}/"
info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
fi
(
set -euo pipefail
local repo_var hook name

# FLAGS_coreos_overlay for gitname coreos-overlay
repo_var="FLAGS_${gitname//-/_}"
shopt -s nullglob
for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
name=${hook##*/}
name=${name%"-${gitname}.sh"}
info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
"${hook}" "${dest}/${repo}" "${!repo_var}" "${update_seed_file}"
done
)
}

build_stage1() {
# First, write out the default 4-stage catalyst configuration files
write_configs

# Prepare local copies of both the "known-good" portage-stable and the
# "known-good" coreos-overlay ebuild repos
local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
rm -rf "$stage1_repos"
mkdir "$stage1_repos"

# If the file exists and is not empty, seed will be updated.
# Stage1 hooks may decide that the seed SDK needs updating.
local update_seed_file="${TEMPDIR}/update_seed"

# prepare ebuild repos for stage 1, either from the local SDK (default)
# or from custom paths specified via command line flags
stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos" "${update_seed_file}"
stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" "${update_seed_file}"

# Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
# This requires us to create a custom catalyst config to point it to the
# repo copy we just created, for snapshotting.
catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
"$TEMPDIR/catalyst-stage1.conf"
# take the "portage directory" (portage-stable copy) snapshot
build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1"

# Update the stage 1 spec to use the "known-good" portage-stable snapshot
# and coreos-overlay copy repository versions from above.
sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
-e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
"$TEMPDIR/stage1.spec"

# If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
if [[ -n ${FLAGS_stage1_portage_path} ]] || [[ -n ${FLAGS_stage1_overlay_path} ]] || [[ -s ${update_seed_file} ]]; then
sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
>>"$TEMPDIR/stage1.spec"
fi
rm -f "${update_seed_file}"

# Finally, build stage 1
build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
}

if [[ "$STAGES" =~ stage1 ]]; then
build_stage1
STAGES="${STAGES/stage1/}"
SEED="${TYPE}/stage1-${ARCH}-latest"
fi

catalyst_build

if [[ "$STAGES" =~ stage4 ]]; then
Expand Down
5 changes: 2 additions & 3 deletions build_library/build_image_util.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ get_metadata() {
local mirror="$(echo "${v}" | grep mirror:// | cut -d '/' -f 3)"
if [ -n "${mirror}" ]; then
# Take only first mirror, those not working should be removed
local location="$(grep "^${mirror}"$'\t' /var/gentoo/repos/gentoo/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
local location="$(grep "^${mirror}"$'\t' /mnt/host/source/src/third_party/portage-stable/profiles/thirdpartymirrors | cut -d $'\t' -f 2- | cut -d ' ' -f 1 | tr -d $'\t')"
v="$(echo "${v}" | sed "s#mirror://${mirror}/#${location}#g")"
fi
new_val+="${v} "
Expand Down Expand Up @@ -490,8 +490,7 @@ EOF
license_list="$(jq -r '.[] | "\(.licenses | .[])"' "${json_input}" | sort | uniq)"
local license_dirs=(
"/mnt/host/source/src/third_party/coreos-overlay/licenses/"
"/mnt/host/source/src/third_party/portage-stable/"
"/var/gentoo/repos/gentoo/licenses/"
"/mnt/host/source/src/third_party/portage-stable/licenses/"
"none"
)
for license_file in ${license_list}; do
Expand Down
Loading
Loading