cmd-build

#!/usr/bin/env bash
set -euo pipefail

dn=$(dirname "$0")
# shellcheck source=src/cmdlib.sh
. "${dn}"/cmdlib.sh

print_help() {
    cat 1>&2 <<'EOF'
Usage: coreos-assembler build --help
       coreos-assembler build [--force] [--skip-prune] [TARGETS]

  Build OSTree and image artifacts from previously fetched packages.
  The TARGETS argument is a list of artifact types to build; if unspecified it
  defaults to "qemu".

  Valid targets:

    - ostree
    - qemu

  Note that all image targets also require the "ostree" target.
EOF
}

# Parse options
FORCE=
SKIP_PRUNE=0
rc=0
options=$(getopt --options hf --longoptions help,force,force-nocache,skip-prune -- "$@") || rc=$?
[ $rc -eq 0 ] || {
    print_help
    exit 1
}
eval set -- "$options"
while true; do
    case "$1" in
        -h | --help)
            print_help
            exit 0
            ;;
        -f | --force | --force-nocache)
            FORCE="--force-nocache"
            ;;
        --skip-prune)
            SKIP_PRUNE=1
            ;;
        --)
            shift
            break
            ;;
        -*)
            fatal "$0: unrecognized option: $1"
            exit 1
            ;;
        *)
            break
            ;;
    esac
    shift
done

if [ $# -eq 0 ]; then
    set -- qemu
fi

build_qemu=
for target in "$@"; do
    case $target in
        ostree) ;;
        qemu) build_qemu=1;;
        *) fatal "Unrecognized target: $target";;
    esac
done

export LIBGUESTFS_BACKEND=direct

prepare_build

ostree --version
rpm-ostree --version

previous_build=
if [ -L "${workdir}"/builds/latest ]; then
    previous_build=$(readlink "${workdir}"/builds/latest)
    previous_builddir="${workdir}/builds/${previous_build}"
    echo "Previous build: ${previous_build}"
fi

previous_commit=
if [ -n "${previous_build}" ]; then
    previous_commit=$(jq -r '.["ostree-commit"]' < "${previous_builddir}/meta.json")
fi
echo "Previous commit: ${previous_commit:-none}"

if [ -n "${previous_commit}" ]; then
    # If we don't have the previous commit (or it's partial), then try to
    # re-import it; this saves us recompression time later on since it's likely
    # a lot of the new objects in this build will be the same.
    commitpath=${tmprepo}/objects/${previous_commit::2}/${previous_commit:2}.commit
    commitpartial=${tmprepo}/state/${previous_commit}.commitpartial
    if [ ! -f "${commitpath}" ] || [ -f "${commitpartial}" ]; then
        if [ -f "${previous_builddir}/ostree-commit.tar" ]; then
            mkdir tmp/prev_repo
            tar -C tmp/prev_repo -xf "${previous_builddir}/ostree-commit.tar"
            ostree pull-local --repo="${tmprepo}" tmp/prev_repo "${previous_commit}"
            rm -rf tmp/prev_repo
        else
            # ok, just fallback to importing the commit object only
            mkdir -p "$(dirname "${commitpath}")"
            cp "${previous_builddir}/ostree-commit-object" "${commitpath}"
            touch "${commitpartial}"
        fi
    fi

    # and point the ref to it if there isn't one already (in which case it might be newer, but e.g. Anaconda failed)
    if ! ostree rev-parse --repo="${tmprepo}" "${ref}" &>/dev/null; then
        ostree refs --repo="${tmprepo}" --create "${ref}" "${previous_commit}"
    fi
fi

# Calculate image input checksum now and gather previous image build variables if any
ks_path="${configdir}"/image.ks
if [ -f "${ks_path}" ]; then
    fatal "Kickstart support was removed; migrate to image.yaml"
fi
image_input="${configdir}"/image.yaml
if ! [ -f "${image_input}" ]; then
    fatal "Failed to find ${image_input}"
fi
image_config_checksum=$(< "${image_input}" sha256sum_str)
if [ -n "${previous_build}" ]; then
    previous_image_input_checksum=$(jq -r '.["coreos-assembler.image-input-checksum"]' < "${previous_builddir}/meta.json")
    previous_image_genver=$(jq -r '.["coreos-assembler.image-genver"]' < "${previous_builddir}/meta.json")
fi
echo "Image Config checksum: ${image_config_checksum}"

# Generate metadata that's *input* to the ostree commit
config_gitrev=$(cd "${configdir}" && git describe --tags --always --abbrev=42)
config_dirty=false
if ! git -C "${configdir}" diff --quiet --exit-code; then
    config_dirty=true
fi
commitmeta_input_json=${PWD}/tmp/commit-metadata-input.json
cat > "${commitmeta_input_json}" <<EOF
{
  "coreos-assembler.config-gitrev": "${config_gitrev}",
  "coreos-assembler.config-dirty": "${config_dirty}"
}
EOF

prepare_git_artifacts "${configdir_gitrepo}" "${PWD}/coreos-assembler-config.tar.gz" "${PWD}/coreos-assembler-config-git.json"

lock_arg=
if [ -f "${manifest_lock}" ]; then
    lock_arg="--ex-lockfile=${manifest_lock}"
    echo "Building from lockfile ${manifest_lock}"
    sleep 1
fi

# These need to be absolute paths right now for rpm-ostree
composejson=${PWD}/tmp/compose.json
# Put this under tmprepo so it gets automatically chown'ed if needed
lockfile_out=${tmprepo}/tmp/manifest-lock.generated.json
# --cache-only is here since `fetch` is a separate verb.
# shellcheck disable=SC2086
runcompose --cache-only ${FORCE} --add-metadata-from-json "${commitmeta_input_json}" \
           --write-composejson-to "${composejson}" \
           --ex-write-lockfile-to "${lockfile_out}" \
           ${lock_arg}
# Very special handling for --write-composejson-to as rpm-ostree doesn't
# write it if the commit didn't change.
if [ -f "${changed_stamp}" ]; then
    commit=$(jq -r '.["ostree-commit"]' < "${composejson}")
    # Clean up prior versions
    rm -f "${workdir}"/tmp/compose-*.json
    # Save this in case the image build fails
    cp -a --reflink=auto "${composejson}" "${workdir}"/tmp/compose-"${commit}".json
else
    # Pick up from tmprepo; that's what rpm-ostree is comparing against. It may
    # be the same as previous_commit, or newer if a previous build failed image
    # creation.
    commit=$(ostree rev-parse --repo="${tmprepo}" "${ref}")
    image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str)
    # Note we may not actually have a previous build in the case of
    # successfully composing an ostree but failing the image on the
    # first build.
    if [ -n "${previous_build}" ] && [ "${image_input_checksum}" = "${previous_image_input_checksum}" ]; then
        echo "No changes in image inputs."
        exit 0
    fi

    # Grab the previous treecompose JSON (local developer case: treecompose succeeded but
    # image build failed) if possible, otherwise grab the previous build
    cached_previous_composejson=${workdir}/tmp/compose-${commit}.json
    if [ -f "${cached_previous_composejson}" ]; then
        echo "Resuming partial build from: ${commit}"
        cp -a --reflink=auto "${cached_previous_composejson}" "${composejson}"
    else
        if [ -z "${previous_build}" ]; then
            # This can happen if building the OSTree worked on the first time,
            # but image creation failed, and then tmp/ was nuked before trying a
            # second time. Just recommend re-running with --force.
            fatal "compose tree had no changes, but no previous build or cached data; try rerunning with --force"
        fi
        echo "Commit ${commit} unchanged; reusing previous build's rpm-ostree metadata"
        # This will have all of the data from the previous build, but we'll
        # overwrite things.
        cp -a --reflink=auto "${previous_builddir}"/meta.json "${composejson}"
    fi
fi

if [ -n "${previous_build}" ]; then
    # do it once for the terminal
    rpm-ostree --repo="${tmprepo}" db diff "${previous_commit}" "${commit}"
    # and once more for the metadata, but only keep the pkgdiff key
    rpm-ostree --repo="${tmprepo}" db diff --format=json \
            "${previous_commit}" "${commit}" | \
        jq '{pkgdiff: .pkgdiff}' > tmp/diff.json
else
    echo '{}' > tmp/diff.json
fi

image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str)
echo "New image input checksum: ${image_input_checksum}"
version=$(ostree --repo="${tmprepo}" show --print-metadata-key=version "${commit}" | sed -e "s,',,g")
if [ "${previous_commit}" = "${commit}" ] && [ -n "${previous_image_genver:-}" ]; then
    image_genver=$((previous_image_genver + 1))
    buildid="${version}"-"${image_genver}"
else
    image_genver=0
    buildid="${version}"
fi
echo "New build ID: ${buildid}"

imageprefix="${name}"-"${buildid}"
# Make these two verbose
set -x
mkdir -p tmp/anaconda
img_base=tmp/${imageprefix}-base.qcow2

# forgive me for this sin
checksum_location=$(find /usr/lib/coreos-assembler-anaconda/ -name '*CHECKSUM' | head -1)

# Hack to force qemu to default to config drive based install on s390x
# See these PRs for relevant discussion:
#   Long term feature request          - https://github.com/coreos/ignition/issues/825
#   Short term workaround in ignition  - https://github.com/coreos/ignition/pull/905
#   Related/Alternative Hack to dracut - https://github.com/coreos/ignition-dracut/pull/145
#   Multi-Arch 4.2 CoreOS workaround   - https://github.com/coreos/coreos-assembler/pull/1004
platformid=qemu
if [ "$arch" == "s390x" ]; then
  platformid=openstack
fi

if [ -n "${build_qemu}" ]; then
    img_qemu=${imageprefix}-qemu.qcow2
    run_virtinstall "${tmprepo}" "${ref}" "${PWD}"/"${img_base}" --variant=cloud
    /usr/lib/coreos-assembler/gf-platformid "$(pwd)"/"${img_base}" "$(pwd)"/"${img_qemu}" ${platformid}
fi

"${dn}"/write-commit-object "${tmprepo}" "${commit}" "$(pwd)"

build_timestamp=$(date -u +$RFC3339)
vm_iso_checksum=$(awk '/SHA256.*iso/{print$NF}' "${checksum_location}")

src_location="container"
if [ ! -f /lib/coreos-assembler/.clean ]; then
    info "This version of coreos-assembler is running code from outside the container."
    src_location="bind mount"
fi

# The base metadata, plus locations for code sources.
# If the following condition is true, then /lib/coreos-assembler has been bind
# mounted in and is using a different build tree.
#
# notice need to backslash escape double quotes in summary since the
# summary could have double quotes: https://github.com/coreos/coreos-assembler/issues/327
#
# shellcheck disable=SC2046 disable=SC2086
cat > tmp/meta.json <<EOF
{
 "buildid": "${buildid}",
 "name": "${name}",
 "summary": "${summary//\"/\\\"}",
 "coreos-assembler.build-timestamp": "${build_timestamp}",
 "coreos-assembler.image-config-checksum": "${image_config_checksum}",
 "coreos-assembler.image-genver": "${image_genver}",
 "coreos-assembler.image-input-checksum": "${image_input_checksum}",
 "coreos-assembler.code-source": "${src_location}",
 "coreos-assembler.vm-iso-checksum": "${vm_iso_checksum}",
 "coreos-assembler.container-config-git": $(jq -M '.git' ${PWD}/coreos-assembler-config-git.json)
}
EOF

if [ -n "${build_qemu}" ]; then
    cat > tmp/images.json <<EOF
{
    "images": {
        "qemu": {
            "path": "${img_qemu}",
            "sha256": "$(sha256sum_str < "${img_qemu}")",
            "size": $(stat -c '%s' "${img_qemu}")
        }
    }
}
EOF
else
    echo '{ "images": {} }' > tmp/images.json
fi

# And the build information about our container, if we are executing
# from a container.
if [ -d /cosa ]; then
    cat > tmp/cosa-image.json <<EOF
{ "coreos-assembler.container-image-git": $(jq -M '.git' /cosa/coreos-assembler-git.json) }
EOF
else
    echo '{}' > tmp/cosa-image.json
fi

# Merge all the JSON; note that we want ${composejson} first
# since we may be overriding data from a previous build.
cat "${composejson}" tmp/meta.json tmp/diff.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json

# Filter out `ref` if it's temporary
if [ -n "${ref_is_temp}" ]; then
    jq 'del(.ref)' < meta.json > meta.json.new
    mv meta.json{.new,}
fi

# Move lockfile into build dir
mv "${lockfile_out}" .

# And add the commit metadata itself, which includes notably the rpmdb pkglist
# in a format that'd be easy to generate diffs out of for higher level tools
"${dn}"/commitmeta_to_json "${tmprepo}" "${commit}" > commitmeta.json

# And create the ostree repo tarball containing the commit
if [ "${commit}" == "${previous_commit}" ]; then
    cp -a --reflink=auto "${previous_builddir}/ostree-commit.tar" .
else
    ostree init --repo=repo --mode=archive
    ostree pull-local --repo=repo "${tmprepo}" "${ref}"
    if [ -n "${ref_is_temp}" ]; then
        ostree refs --repo=repo --delete "${ref}"
    fi
    # Don't compress; archive repos are already compressed individually and we'd
    # gain ~20M at best. We could probably have better gains if we compress the
    # whole repo in bare/bare-user mode, but that's a different story...
    tar -cf ostree-commit.tar -C repo .
    rm -rf repo
fi

# Clean up our temporary data
saved_build_tmpdir="${workdir}/tmp/last-build-tmp"
rm -rf "${saved_build_tmpdir}"
mv -T tmp "${saved_build_tmpdir}"
ostree prune --repo="${tmprepo}" --refs-only
# Back to the toplevel build directory, so we can rename this one
cd "${workdir}"/builds
# We create a .build-commit file to note that we're in the
# middle of a "commit".  This may be useful in the future
# for having things be transactional.  If for example we
# were interrupted between the rename() and linkat() below,
# things would be inconsistent and future builds would fail
# on the `mv`.
touch .build-commit
mv -T "${tmp_builddir}" "${buildid}"
# Replace the latest link
ln -Tsfr "${buildid}" latest
# Update builds.json
# the variables passed to `prune_builds` end up single quoted and
# python treats them as literals, so we workaround this by duplicating
# the command  ¯\_(ツ)_/¯
if [ "${SKIP_PRUNE}" == 1 ]; then
    "${dn}"/prune_builds --workdir "${workdir}" --insert-only "${buildid}"
else
    "${dn}"/prune_builds --workdir "${workdir}"
fi
rm .build-commit