build_torcx_store

#!/bin/bash

# Copyright (c) 2017 The CoreOS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

. "$(dirname "$0")/common.sh" || exit 1

# Script must run inside the chroot
assert_inside_chroot

assert_not_root_user

# Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \
  "The board to build packages for."
DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
  "Directory in which to place torcx stores and manifests (named by board/version)"

# include upload options
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1

FLAGS_HELP="usage: $(basename $0) [flags] [images]

This script builds a collection of torcx images to be installed into a torcx
store.  By default, all supported images are built, but a list of images can be
given as command arguments.  Note that their order matters, since the version
specified last will get the default reference symlink.
"
show_help_if_requested "$@"

# The following options are advanced options, only available to those willing
# to read the source code. They are not shown in help output, since they are
# not needed for the typical developer workflow.
DEFINE_integer build_attempt 1 \
  "The build attempt for this image build."
DEFINE_string group developer \
  "The update group."
DEFINE_string version '' \
  "Overrides version number in name to this version."

# Parse command line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

# Only now can we die on error.  shflags functions leak non-zero error codes,
# so will die prematurely if 'switch_to_strict_mode' is specified before now.
switch_to_strict_mode

# Initialize upload options
check_gsutil_opts

# Define BUILD_DIR and set_build_symlinks.
. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1

TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}"

# Print the first level of runtime dependencies for a torcx meta-package.
function torcx_dependencies() (
        pkg=${1:?}
        ebuild=$(equery-${BOARD} w "${pkg}")
        function inherit() { : ; }
        . "${ebuild}"
        echo ${RDEPEND}
)

# Build and install a package configured as part of a torcx image.
function torcx_build() (
        pkg=${2:?}
        tmproot=${1:?}

        export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib
        export PKGDIR="${tmproot}/var/lib/portage/pkgs"

        # Allow the meta-package to install bashrc to customize the builds.
        [ -s "${tmproot}/etc/portage/bashrc" ] &&
        . "${tmproot}/etc/portage/bashrc"

        # Build binary packages using dev files in the board root.
        emerge-${BOARD} \
            --buildpkg \
            --buildpkgonly \
            --nodeps \
            --oneshot \
            --quiet \
            --root-deps=rdeps \
            "${pkg}"

        # Install the binary packages in the temporary torcx image root.
        emerge-${BOARD} \
            --nodeps \
            --oneshot \
            --quiet \
            --root="${tmproot}" \
            --root-deps=rdeps \
            --sysroot="${tmproot}" \
            --usepkgonly \
            "${pkg}"
)

# Create a torcx image from the given meta-package.
function torcx_package() {
        local pkg="app-torcx/${1##*/}"
        local name=${pkg%-[0-9]*}
        local version=${pkg:${#name}+1}
        local manifest_path="${2}"
        local type="${3}"
        local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default
        local pkg_cas_file pkg_cas_root
        local pkg_locations=()
        local name=${name##*/}
        local version=${version%%-r*}

        # Set up the base package layout to dump everything into /bin and /lib.
        # tmproot is what the packages are installed into.
        # A subset of the files from tmproot are then moved into tmppkgroot,
        # which is then archived and uploaded.
        tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
        tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d)
        trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT RETURN
        sudo chmod 0755 "${tmproot}" "${tmppkgroot}"
        sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr}
        sudo ln -fns ../bin "${tmproot}/usr/bin"
        sudo ln -fns ../lib "${tmproot}/usr/lib"
        sudo ln -fns lib "${tmproot}/usr/lib64"
        sudo ln -fns bin "${tmproot}/usr/sbin"
        sudo ln -fns lib "${tmproot}/lib64"
        sudo ln -fns bin "${tmproot}/sbin"

        # Install the meta-package and its direct dependencies.
        for deppkg in "=${pkg}" $(torcx_dependencies "${pkg}")
        do
                torcx_build "${tmproot}" "${deppkg}"
        done

        # by convention, the first dependency in a torcx package is the primary
        # source package
        rdepends=($(torcx_dependencies "${pkg}"))
        source_pkg="${rdepends[0]#=}"

        # Pluck out shared libraries and SONAME links.
        sudo mv "${tmproot}"/{lib,tmplib}
        sudo rm -fr "${tmproot}/tmplib/debug"
        sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete
        sudo mkdir -p "${tmproot}/lib"
        sudo find "${tmproot}/tmplib" -name 'lib*.so*' \
            -exec mv -t "${tmproot}/lib/" {} +

        # Rewrite any units for transparent activation from the torcx root.
        if [ -e "${tmproot}/tmplib/systemd/system" ]
        then
                sudo mkdir -p "${tmproot}/lib/systemd"
                sudo mv "${tmproot}/tmplib/systemd/system" \
                    "${tmproot}/lib/systemd/"
                sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \
                    -e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \
                    -e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \
                    -e "/^\[Service]/aEnvironment=TORCX_IMAGEDIR=/${name}" \
                    -e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \
                    -e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} +
        fi

        # Network configuration can be installed unmodified.
        if [ -e "${tmproot}/tmplib/systemd/network" ]
        then
                sudo mkdir -p "${tmproot}/lib/systemd"
                sudo mv "${tmproot}/tmplib/systemd/network" \
                    "${tmproot}/lib/systemd/"
        fi

        # Rewrite RPATHs to use the real $ORIGIN value.
        find -H "${tmproot}"/{bin,lib} -type f |
        while read file
        do
                rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) &&
                test "${rpath#/ORIGIN/}" != "${rpath}" &&
                sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}"
                :  # Set $? to 0 or the pipeline fails and -e quits.
        done

        # Move anything we plan to package to its root.
        sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}"
        if [ -e "${tmproot}/usr/share" ]
        then
                sudo mkdir "${tmppkgroot}/usr"
                sudo mv "${tmproot}/usr/share" "${tmppkgroot}/usr/"
        fi

        tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz"
        tar --force-local -C "${tmppkgroot}" -czf "${tmpfile}" .
        sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}')

        # TODO(euank): this opaque digest, if it were reproducible, could save
        # users from having to download things that haven't changed.
        # For now, use the sha512sum of the final image.
        # Ideally we should move to something more like a casync digest or tarsum.
        # The reason this is currently not being done is because to do that we
        # *MUST* ensure that a given pair of (digest, sha512sum) referenced in
        # a previous torcx package remains correct.
        # Because this code, as written, clobbers existing things with the same
        # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime)
        # that property doesn't hold.
        # To switch this back to a reprodicble digest, we *must* never clobber
        # existing objects (and thus re-use their sha512sum here).
        digest="${sha512sum}"

        pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}"
        pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz"
        mkdir -p "${pkg_cas_root}"
        mv "${tmpfile}" "${pkg_cas_file}"

        update_default=false
        if [[ "${type}" == "default" ]]; then
                update_default=true
                pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz")
        fi
        if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
                pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
        fi
        torcx_manifest::add_pkg "${manifest_path}" \
            "${name}" \
            "${version}" \
            "sha512-${sha512sum}" \
            "${digest}" \
            "${source_pkg}" \
            "${update_default}" \
            "${pkg_locations[@]}"

        trap - EXIT
}

# This list defines every torcx image that goes into the vendor store for the
# current branch's release version.  Note that the default reference symlink
# for each package will point at the last version specified.  This can handle
# swapping default package versions for different OS releases by reordering.
DEFAULT_IMAGES=(
        =app-torcx/docker-1.12
        =app-torcx/docker-18.06
)

# This list contains extra images which will be uploaded and included in the
# generated manifest, but won't be included in the vendor store.
EXTRA_IMAGES=(
	=app-torcx/docker-17.03
)

mkdir -p "${BUILD_DIR}"
manifest_path="${BUILD_DIR}/torcx_manifest.json"
torcx_manifest::create_empty "${manifest_path}"
for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done
for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done

set_build_symlinks latest "${FLAGS_group}-latest"

# Upload the pkgs referenced by this manifest
for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do
        for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do
            # no need to sign; the manifest includes their shasum and is signed.
            upload_files \
                'torcx pkg' \
                "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \
                "" \
                "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz
      done
done

# Upload the manifest
# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than
# 'TORCX_UPLOAD_ROOT'.
# For non-release builds, those two locations will be the same, so it usually
# won't matter.
# However, for release builds, torcx packages may be uploaded directly to their
# final location, while the manifest still has to go through build bucket in
# order to get signed.
sign_and_upload_files \
    'torcx manifest' \
    "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${COREOS_VERSION}" \
    "" \
    "${manifest_path}"

# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab