bootstrap_sdk

#!/bin/bash
#
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This uses Gentoo's catalyst for very thoroughly building images from
# scratch. Using images based on this will eliminate some of the hackery
# in make_chroot.sh for building up the sdk from a stock stage3 tarball.
# 
# For reference the procedure it performs is this:
# 
# 1. snapshot: Grab a snapshot of the portage-stable repo from
#     the current SDK's /var/lib/gentoo/repos/gentoo.
#     Alternatively, check out a git ref specified via --portage-ref.
#
# 2. stage1: Using a "seed" tarball as a build environment, build a
#     minimal root file system into a clean directory using ROOT=...
#     and USE=-* The restricted USE flags are key be small and avoid
#     circular dependencies.
#     NOTE that stage1 LACKS PROPER STAGE ISOLATION. Binaries produced in stage1
#      will be linked against the SEED SDK libraries, NOT against libraries
#      built in stage 1. See "stage_repo()" documentation further below for more.
#     This stage uses:
#     - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
#        or a custom path via --stage1_portage_path command line option
#     - coreos-overlay  from the SDK's /var/lib/gentoo/repos/coreos-overlay
#        or a custom path via --stage1_overlay_path command line option
#     Command line option refs need caution though, since
#     stage1 must not contain updated ebuilds (see build_stage1 below).
#
# 3. stage2: Run portage-stable/scripts/bootstrap.sh
#     This rebuilds the toolchain using Gentoo bootstrapping, ensuring it's not linked
#     to or otherwise influenced by whatever was in the "seed" tarball.
#     The toolchain rebuild may contain updated package ebuilds from
#     third_party/(portage-stable|coreos-overlay).
#     This and all following stages use portage-stable and coreos-overlay
#     from third_party/... (see 1.)
#
# 4. stage3: Run emerge -e system to rebuild everything using the fresh updated
#     toolchain from 3., using the normal USE flags provided by the profile. This
#     will also pull in assorted base system packages that weren't included
#     in the minimal environment stage1 created.
#
# 5. stage4: Install any extra packages or other desired tweaks. For the
#     sdk we just install all the packages normally make_chroot.sh does.
#
# Usage: bootstrap_sdk [stage1 stage2 etc]
# By default all four stages will be built using the latest stage4 as a seed.

SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1

TYPE="flatcar-sdk"

. "${BUILD_LIBRARY_DIR}/catalyst.sh" || exit 1

# include upload options
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1


DEFINE_string stage1_portage_path "" \
  "Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
DEFINE_string stage1_overlay_path "" \
  "Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"


## Define the stage4 config template
catalyst_stage4() {
cat <<EOF
target: stage4
pkgcache_path: $BINPKGS
stage4/packages: coreos-devel/sdk-depends
stage4/fsscript: ${BUILD_LIBRARY_DIR}/catalyst_sdk.sh
stage4/root_overlay: ${ROOT_OVERLAY}
stage4/empty: /etc/portage/repos.conf /root /usr/portage /var/cache/edb
stage4/rm: /etc/machine-id /etc/resolv.conf
EOF
catalyst_stage_default
}

# Switch to HTTP because early boostrap stages do not have SSL support.
GENTOO_MIRRORS=$(portageq envvar GENTOO_MIRRORS)
GENTOO_MIRRORS="${GENTOO_MIRRORS//https:\/\//http://}"
export GENTOO_MIRRORS

catalyst_init "$@"
check_gsutil_opts
ROOT_OVERLAY=${TEMPDIR}/stage4_overlay

if [[ "$STAGES" =~ stage4 ]]; then
    info "Setting release to ${FLATCAR_VERSION}"
    rm -rf "${ROOT_OVERLAY}"
    # need to setup the lib->lib64 symlink correctly
    libdir=$(get_sdk_libdir)
    mkdir -p "${ROOT_OVERLAY}/usr/${libdir}"
    if [[ "${libdir}" != lib ]]; then
        if [[ "$(get_sdk_symlink_lib)" == "yes" ]]; then
            ln -s "${libdir}" "${ROOT_OVERLAY}/usr/lib"
        else
            mkdir -p "${ROOT_OVERLAY}/usr/lib"
        fi
    fi
    "${BUILD_LIBRARY_DIR}/set_lsb_release" \
        --root "${ROOT_OVERLAY}"
fi

# toolchain_util.sh is required by catalyst_sdk.sh
# To copy it, we need to create /tmp with the right permissions as it will be
# used in the exported chroot.
mkdir -p "${ROOT_OVERLAY}/tmp"
chmod 1777 "${ROOT_OVERLAY}/tmp"
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"


# Stage 1 uses "known-good" ebuilds (from both coreos-overlay and portage-stable)
#  to build a minimal toolchain (USE="-*") for stage 2.
#
# No package updates must happen in stage 1, so we use the portage-stable and
#  coreos-overlay paths included with the current SDK (from the SDK chroot's
#  /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
#  'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
#
# Using ebuilds from the above mentioned sources will ensure that stage 1 builds
#  a minimal stage 2 from known-good ebuild versions - the same ebuild versions
#  that were used to build the very SDK we run ./bootstrap_sdk in.
#
# DANGER ZONE
#
# Stage 1 lacks proper isolation and will link all packages built for
#  stage 2 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries
#  installed into the FS root of the stage 2 seed ("/tmp/stage1root" in the catalyst chroot).
#  This is why we must prevent any updated package ebuilds to "leak" into stage 1, hence we use
#  "known good" ebuild repo versions outlined above.
#
# In special circumstances it may be required to circumvent this and use custom paths
#  for either (or both) portage and overlay. The command line options
#  --stage1-portage-path and --stage1-overlay-path may be used to specify
#  a repo path known to work for stage1. In that case the stage1 seed (i.e. the seed SDK)
#  will be updated prior to starting to build stage 2.
#  NOTE that this should never be used to introduce library updates in stage 1. All binaries
#  produced in stage 1 are linked against libraries in the seed tarball, NOT libraries produced
#  by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
#  outdated "seed tarball" libraries which have been updated to newer versions in stage 1.

stage_repo() {
    local repo="$1"
    local path="$2"
    local dest="$3"
    local gitname="$repo"

    if [ "$gitname" = "gentoo" ] ; then
        gitname="portage-stable"
    fi

    if [ -z "$path" ]; then
        cp -R "/var/gentoo/repos/${repo}" "$dest"
        info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
    else
        mkdir "$dest/$repo"
        cp -R "${path}/"* "$dest/${repo}/"
        info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
        info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
    fi
    (
        set -euo pipefail
        local repo_var hook name

        # FLAGS_coreos_overlay for gitname coreos-overlay
        repo_var="FLAGS_${gitname//-/_}"
        shopt -s nullglob
        for hook in "${FLAGS_coreos_overlay}/coreos/stage1_hooks/"*"-${gitname}.sh"; do
            name=${hook##*/}
            name=${name%"-${gitname}.sh"}
            info "Invoking stage1 ${gitname} hook ${name} on ${dest}/${repo}"
            "${hook}" "${dest}/${repo}" "${!repo_var}"
        done
    )
}

build_stage1() {
    # First, write out the default 4-stage catalyst configuration files
    write_configs

    # Prepare local copies of both the "known-good" portage-stable and the
    #  "known-good" coreos-overlay ebuild repos
    local stage1_repos="$TEMPDIR/stage1-ebuild-repos"
    info "Creating stage 1 ebuild repos and stage 1 snapshot in '$stage1_repos'"
    rm -rf "$stage1_repos"
    mkdir "$stage1_repos"

    # prepare ebuild repos for stage 1, either from the local SDK (default)
    #  or from custom paths specified via command line flags
    stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos"
    stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos"

    # Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
    #  This requires us to create a custom catalyst config to point it to the
    #  repo copy we just created, for snapshotting.
    catalyst_conf > "$TEMPDIR/catalyst-stage1.conf"
    sed -i "s:^portdir.*:portdir=\"$stage1_repos/gentoo\":" \
         "$TEMPDIR/catalyst-stage1.conf"
    # take the "portage directory" (portage-stable copy) snapshot
    build_snapshot "${TEMPDIR}/catalyst-stage1.conf" "${FLAGS_version}-stage1"

    # Update the stage 1 spec to use the "known-good" portage-stable snapshot
    #  and coreos-overlay copy repository versions from above.
    sed -i -e "s/^snapshot:.*/snapshot: $FLAGS_version-stage1/" \
           -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
        "$TEMPDIR/stage1.spec"

    # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
    if [ -n "${FLAGS_stage1_portage_path}" -o -n "${FLAGS_stage1_overlay_path}" ] ; then
        sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
        echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
            >>"$TEMPDIR/stage1.spec"
    fi

    # Finally, build stage 1
    build_stage stage1 "$SEED" "$TEMPDIR/catalyst-stage1.conf"
}

if [[ "$STAGES" =~ stage1 ]]; then
    build_stage1
    STAGES="${STAGES/stage1/}"
    SEED="${TYPE}/stage1-${ARCH}-latest"
fi

catalyst_build

if [[ "$STAGES" =~ stage4 ]]; then
    info "Build complete! Changing output name to something more sensible."
    build_name="stage4-${ARCH}-${FLAGS_version}.tar.bz2"
    release_name="${TYPE}-${ARCH}-${FLAGS_version}.tar.bz2"
    build_image="${BUILDS}/${build_name}"
    release_image="${BUILDS}/${release_name}"
    build_contents="${build_image}.CONTENTS.gz"
    release_contents="${release_image}.CONTENTS.gz"
    build_digests="${build_image}.DIGESTS"
    release_digests="${release_image}.DIGESTS"
    ln -f "${build_image}" "${release_image}"
    ln -f "${build_contents}" "${release_contents}"
    sed -e "s/${build_name}/${release_name}/" \
        "${build_digests}" > "${release_digests}"

    # Validate we didn't break the DIGESTS with sed
    verify_digests "${release_image}" "${release_contents}"

    info "SDK ready: ${release_image}"

    def_upload_path="${UPLOAD_ROOT}/sdk/${ARCH}/${FLAGS_version}"
    sign_and_upload_files "tarball" "${def_upload_path}" "" \
        "${release_image}" "${release_contents}" "${release_digests}"
    sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \
        "${BINPKGS}"/*

    if [ -d "${BINPKGS}/crossdev" ]; then
        # Upload the SDK toolchain packages
        sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
            "toolchain/" "${BINPKGS}/crossdev"/*
    fi
fi

command_completed