diff --git a/bootstrap_sdk b/bootstrap_sdk index fccd6e039cf..bf094e28a79 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -7,9 +7,9 @@ # This uses Gentoo's catalyst for very thoroughly building images from # scratch. Using images based on this will eliminate some of the hackery # in make_chroot.sh for building up the sdk from a stock stage3 tarball. -# +# # For reference the procedure it performs is this: -# +# # 1. snapshot: Grab a snapshot of the portage-stable repo from # the current SDK's /var/lib/gentoo/repos/gentoo. # Alternatively, check out a git ref specified via --portage-ref. @@ -68,15 +68,14 @@ DEFINE_string stage1_overlay_path "" \ ## Define the stage4 config template catalyst_stage4() { cat < "$TEMPDIR/catalyst-stage1.conf" - sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \ - "$TEMPDIR/catalyst-stage1.conf" # take the "portage directory" (portage-stable copy) snapshot - build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1" + build_snapshot "$stage1_repos/portage-stable" "${FLAGS_version}-stage1" # Update the stage 1 spec to use the "known-good" portage-stable snapshot # and coreos-overlay copy repository versions from above. - sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \ - -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \ + sed -i -e "s/^snapshot_treeish:.*/snapshot_treeish: $FLAGS_version-stage1/" \ + -e "s,^repos:.*,repos: $stage1_repos/coreos-overlay," \ "$TEMPDIR/stage1.spec" # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK @@ -220,10 +213,11 @@ build_stage1() { echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \ >>"$TEMPDIR/stage1.spec" fi + rm -f "${update_seed_file}" # Finally, build stage 1 - build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf" + build_stage stage1 } if [[ "$STAGES" =~ stage1 ]]; then diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index ea1f6a97654..ef08aa5f8b6 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -55,17 +55,15 @@ DEFINE_boolean debug ${FLAGS_FALSE} "Enable verbose output from catalyst." catalyst_conf() { cat < "$TEMPDIR/catalyst.conf" info " catalystrc" catalystrc > "$TEMPDIR/catalystrc" - info " portage/repos.conf/coreos.conf" - repos_conf > "$TEMPDIR/portage/repos.conf/coreos.conf" info " stage1.spec" catalyst_stage1 > "$TEMPDIR/stage1.spec" - info " stage2.spec" - catalyst_stage2 > "$TEMPDIR/stage2.spec" - info " stage3.spec" - catalyst_stage3 > "$TEMPDIR/stage3.spec" - info " stage4.spec" - catalyst_stage4 > "$TEMPDIR/stage4.spec" - info "Putting a symlink to user patches..." - ln -sfT '/var/gentoo/repos/local/coreos/user-patches' \ - "$TEMPDIR/portage/patches" + + info "Configuring Portage..." + cp -r "${BUILD_LIBRARY_DIR}"/portage/ "${TEMPDIR}/" + + ln -sfT '/var/gentoo/repos/coreos-overlay/coreos/user-patches' \ + "${TEMPDIR}"/portage/patches } build_stage() { - local stage srcpath catalyst_conf target_tarball + local stage catalyst_conf target_tarball stage="$1" - srcpath="$2" - catalyst_conf="$3" + catalyst_conf="$TEMPDIR/catalyst.conf" target_tarball="${stage}-${ARCH}-${FLAGS_version}.tar.bz2" - [ -z "$catalyst_conf" ] && catalyst_conf="$TEMPDIR/catalyst.conf" - if [[ -f "$BUILDS/${target_tarball}" && $FLAGS_rebuild == $FLAGS_FALSE ]] then info "Skipping $stage, $target_tarball already exists." @@ -270,8 +247,7 @@ build_stage() { "${DEBUG[@]}" \ --verbose \ --config "$TEMPDIR/catalyst.conf" \ - --file "$TEMPDIR/${stage}.spec" \ - --cli "source_subpath=$srcpath" + --file "$TEMPDIR/${stage}.spec" # Catalyst does not clean up after itself... rm -rf "$TEMPDIR/$stage-${ARCH}-${FLAGS_version}" ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \ @@ -280,46 +256,19 @@ build_stage() { } build_snapshot() { - local catalyst_conf snapshot snapshots_dir snapshot_base snapshot_path + local repo_dir snapshot snapshots_dir snapshot_path - catalyst_conf=${1:-"${TEMPDIR}/catalyst.conf"} + repo_dir=${1:-"${FLAGS_portage_stable}"} snapshot=${2:-"${FLAGS_version}"} snapshots_dir="${CATALYST_ROOT}/snapshots" - snapshot_base="${snapshots_dir}/gentoo-${snapshot}" - snapshot_path="${snapshot_base}.tar.bz2" - if [[ -f "${snapshot_path}" && $FLAGS_rebuild == $FLAGS_FALSE ]] + snapshot_path="${snapshots_dir}/portage-stable-${snapshot}.sqfs" + if [[ -f ${snapshot_path} && $FLAGS_rebuild == $FLAGS_FALSE ]] then info "Skipping snapshot, ${snapshot_path} exists" else info "Creating snapshot ${snapshot_path}" - catalyst \ - "${DEBUG[@]}" \ - --verbose \ - --config "${catalyst_conf}" \ - --snapshot "${snapshot}" - fi - local f - local to_remove=() - # This will expand to at least our just built snapshot tarball, so - # no nullglob is needed here. - for f in "${snapshot_base}".*; do - case "${f}" in - "${snapshot_path}") - # Our snapshot, keep it as is. - : - ;; - *.CONTENTS|*.CONTENTS.gz|*.DIGESTS) - # These can stay, catalyst is not bothered by those. - : - ;; - *) - to_remove+=("${f}") - ;; - esac - done - if [[ ${#to_remove[@]} -gt 0 ]]; then - info "$(printf '%s\n' 'Found spurious files in snapshots directory that may confuse Catalyst, removing them:' "${to_remove[@]}")" - rm -rf "${to_remove[@]}" + mkdir -p "${snapshot_path%/*}" + tar -c -C "${repo_dir}" . | tar2sqfs "${snapshot_path}" -q -f -j1 -c gzip fi } @@ -335,7 +284,7 @@ catalyst_build() { used_seed=0 if [[ "$STAGES" =~ stage1 ]]; then - build_stage stage1 "$SEED" + build_stage stage1 used_seed=1 fi @@ -343,7 +292,9 @@ catalyst_build() { if [[ $used_seed -eq 1 ]]; then SEED="${TYPE}/stage1-${ARCH}-latest" fi - build_stage stage2 "$SEED" + info " stage2.spec" + catalyst_stage2 > "$TEMPDIR/stage2.spec" + build_stage stage2 used_seed=1 fi @@ -351,7 +302,9 @@ catalyst_build() { if [[ $used_seed -eq 1 ]]; then SEED="${TYPE}/stage2-${ARCH}-latest" fi - build_stage stage3 "$SEED" + info " stage3.spec" + catalyst_stage3 > "$TEMPDIR/stage3.spec" + build_stage stage3 used_seed=1 fi @@ -359,10 +312,12 @@ catalyst_build() { if [[ $used_seed -eq 1 ]]; then SEED="${TYPE}/stage3-${ARCH}-latest" fi - build_stage stage4 "$SEED" + info " stage4.spec" + catalyst_stage4 > "$TEMPDIR/stage4.spec" + build_stage stage4 used_seed=1 fi # Cleanup snapshots, we don't use them - rm -rf "$CATALYST_ROOT/snapshots/gentoo-${FLAGS_version}.tar.bz2"* + rm -rf "$CATALYST_ROOT/snapshots/${FLAGS_portage_stable##*/}-${FLAGS_version}.sqfs"* } diff --git a/build_library/catalyst_sdk.sh b/build_library/catalyst_sdk.sh index e5fb29a188b..ff3f3d4b1d4 100644 --- a/build_library/catalyst_sdk.sh +++ b/build_library/catalyst_sdk.sh @@ -11,7 +11,7 @@ echo "Setting the default Python interpreter" eselect python update echo "Building cross toolchain for the SDK." -configure_crossdev_overlay / /tmp/crossdev +configure_crossdev_overlay / /usr/local/portage/crossdev for cross_chost in $(get_chost_list); do echo "Building cross toolchain for ${cross_chost}" @@ -21,10 +21,11 @@ for cross_chost in $(get_chost_list); do install_cross_rust "${cross_chost}" ${clst_myemergeopts} done -echo "Saving snapshot of coreos-overlay repo for future SDK bootstraps" -# Copy coreos-overlay, which is in /var/gentoo/repos/local/, into a -# local directory. /var/gentoo/repos/local/ is removed before archiving -# and we want to keep a snapshot. This snapshot is used - alongside -# /var/gentoo/repos/gentoo - by stage 1 of future bootstraps. -mkdir -p /var/gentoo/repos/coreos-overlay -cp -R /var/gentoo/repos/local/* /var/gentoo/repos/coreos-overlay +echo "Saving snapshot of repos for future SDK bootstraps" +# Copy portage-stable and coreos-overlay, which are under +# /mnt/host/source/src/third_party, into a local directory because they are +# removed before archiving and we want to keep snapshots. These snapshots are +# used by stage 1 in future bootstraps. +mkdir -p /var/gentoo/repos/{gentoo,coreos-overlay} +cp -R /mnt/host/source/src/third_party/portage-stable/* /var/gentoo/repos/gentoo/ +cp -R /mnt/host/source/src/third_party/coreos-overlay/* /var/gentoo/repos/coreos-overlay/ diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index 0c041b3dd34..92d6e932136 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -37,7 +37,7 @@ build_target_toolchain() { run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}" } -configure_crossdev_overlay / /tmp/crossdev +configure_crossdev_overlay / /usr/local/portage/crossdev for board in $(get_board_list); do echo "Building native toolchain for ${board}" diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index a0c07477cd1..3d702c32eb1 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -42,22 +42,24 @@ PORTAGE_BINHOST="$(get_binhost_url "${binhost}" "${update_group}" 'pkgs') $(get_binhost_url "${binhost}" "${update_group}" 'toolchain')" EOF - sudo_clobber "${root_fs_dir}/etc/portage/repos.conf/coreos.conf" < /dev/null "${sudo[@]}" tee "${root}${location}/metadata/layout.conf" > /dev/null <