Move OSTree commit into build directory

Rather than keeping OSTree data separately in the toplevel `repo/`, make
it part of the build directory. This solves a bunch of issues and makes
things conceptually clearer.

See discussions in:
coreos#159

src/cmd-build

@@ -79,15 +79,37 @@ if [ -L "${workdir}"/builds/latest ]; then
 fi
 
 previous_commit=
-if [ -n "${ref:-}" ]; then
-    previous_commit=$(ostree --repo="${workdir}"/repo rev-parse "${ref}" 2>/dev/null || true)
-fi
-# If the ref was unset or missing, look at the previous build
-if [ -z "${previous_commit}" ] && [ -n "${previous_build}" ]; then
+if [ -n "${previous_build}" ]; then
     previous_commit=$(jq -r '.["ostree-commit"]' < "${previous_builddir}/meta.json")
 fi
 echo "Previous commit: ${previous_commit:-none}"
 
+if [ -n "${previous_commit}" ]; then
+    # If we don't have the previous commit (or it's partial), then try to
+    # re-import it; this saves us recompression time later on since it's likely
+    # a lot of the new objects in this build will be the same.
+    commitpath=${tmprepo}/objects/${previous_commit::2}/${previous_commit:2}.commit
+    commitpartial=${tmprepo}/state/${previous_commit}.commitpartial
+    if [ ! -f "${commitpath}" ] || [ -f "${commitpartial}" ]; then
+        if [ -f "${previous_builddir}/ostree-commit.tar" ]; then
+            mkdir tmp/prev_repo
+            tar -C tmp/prev_repo -xf "${previous_builddir}/ostree-commit.tar"
+            ostree pull-local --repo="${tmprepo}" tmp/prev_repo "${previous_commit}"
+            rm -rf tmp/prev_repo
+        else
+            # ok, just fallback to importing the commit object only
+            mkdir -p "$(dirname "${commitpath}")"
+            cp "${previous_builddir}/ostree-commit-object" "${commitpath}"
+            touch "${commitpartial}"
+        fi
+    fi
+
+    # and point the ref to it if there isn't one already (in which case it might be newer, but e.g. Anaconda failed)
+    if ! ostree rev-parse --repo="${tmprepo}" "${ref}" &>/dev/null; then
+        ostree refs --repo="${tmprepo}" --create "${ref}" "${previous_commit}"
+    fi
+fi
+
 # Calculate image input checksum now and gather previous image build variables if any
 ks_path="${configdir}"/image.ks
 if [ -f "${ks_path}" ]; then
@@ -125,8 +147,6 @@ composejson=${PWD}/tmp/compose.json
 # --cache-only is here since `fetch` is a separate verb.
 runcompose --cache-only ${FORCE} --add-metadata-from-json "${commitmeta_input_json}" \
            --write-composejson-to "${composejson}"
-# Always update the summary, since we used to do so
-ostree --repo="${workdir}/repo" summary -u
 # Very special handling for --write-composejson-to as rpm-ostree doesn't
 # write it if the commit didn't change.
 if [ -f "${changed_stamp}" ]; then
@@ -136,7 +156,10 @@ if [ -f "${changed_stamp}" ]; then
     # Save this in case the image build fails
     cp -a --reflink=auto "${composejson}" "${workdir}"/tmp/compose-"${commit}".json
 else
-    commit=${previous_commit}
+    # Pick up from tmprepo; that's what rpm-ostree is comparing against. It may
+    # be the same as previous_commit, or newer if a previous build failed image
+    # creation.
+    commit=$(ostree rev-parse --repo="${tmprepo}" "${ref}")
     image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str)
     # Note we may not actually have a previous build in the case of
     # successfully composing an ostree but failing the image on the
@@ -148,7 +171,7 @@ else
 
     # Grab the previous treecompose JSON (local developer case: treecompose succeeded but
     # image build failed) if possible, otherwise grab the previous build
-    cached_previous_composejson="${workdir}"/tmp/compose-"${previous_commit}".json
+    cached_previous_composejson=${workdir}/tmp/compose-${commit}.json
     if [ -f "${cached_previous_composejson}" ]; then
         echo "Resuming partial build from: ${commit}"
         cp -a --reflink=auto "${cached_previous_composejson}" "${composejson}"
@@ -167,12 +190,17 @@ else
 fi
 
 if [ -n "${previous_build}" ]; then
-    rpm-ostree --repo="${workdir}"/repo db diff "${previous_commit}" "${commit}"
+    # do it once for the terminal
+    rpm-ostree --repo="${tmprepo}" db diff "${previous_commit}" "${commit}"
+    # and once more for the metadata, but only keep the pkgdiff key
+    rpm-ostree --repo="${tmprepo}" db diff --format=json \
+            "${previous_commit}" "${commit}" | \
+        jq '{pkgdiff: .pkgdiff}' > tmp/diff.json
 fi
 
 image_input_checksum=$( (echo "${commit}" && echo "${image_config_checksum}") | sha256sum_str)
 echo "New image input checksum: ${image_input_checksum}"
-version=$(ostree --repo="${workdir}"/repo show --print-metadata-key=version "${commit}" | sed -e "s,',,g")
+version=$(ostree --repo="${tmprepo}" show --print-metadata-key=version "${commit}" | sed -e "s,',,g")
 if [ "${previous_commit}" = "${commit}" ] && [ -n "${previous_image_genver:-}" ]; then
     image_genver=$((previous_image_genver + 1))
     buildid="${version}"-"${image_genver}"
@@ -192,10 +220,10 @@ img_base=tmp/${imageprefix}-base.qcow2
 checksum_location=$(find /usr/lib/coreos-assembler-anaconda/ -name '*CHECKSUM' | head -1)
 
 img_qemu=${imageprefix}-qemu.qcow2
-run_virtinstall "${workdir}/repo" "${ref}" "${PWD}"/"${img_base}" --variant=cloud
+run_virtinstall "${tmprepo}" "${ref}" "${PWD}"/"${img_base}" --variant=cloud
 /usr/lib/coreos-assembler/gf-platformid "$(pwd)"/"${img_base}" "$(pwd)"/"${img_qemu}" qemu
 
-"${dn}"/write-commit-object "${workdir}/repo" "${commit}" "$(pwd)"
+"${dn}"/write-commit-object "${tmprepo}" "${commit}" "$(pwd)"
 
 build_timestamp=$(date -u +$RFC3339)
 vm_iso_checksum=$(awk '/SHA256.*iso/{print$NF}' "${checksum_location}")
@@ -253,7 +281,7 @@ fi
 
 # Merge all the JSON; note that we want ${composejson} first
 # since we may be overriding data from a previous build.
-cat "${composejson}" tmp/meta.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json
+cat "${composejson}" tmp/meta.json tmp/diff.json tmp/images.json tmp/cosa-image.json "${commitmeta_input_json}" | jq -s add > meta.json
 
 # Filter out `ref` if it's temporary
 if [ -n "${ref_is_temp}" ]; then
@@ -263,12 +291,29 @@ fi
 
 # And add the commit metadata itself, which includes notably the rpmdb pkglist
 # in a format that'd be easy to generate diffs out of for higher level tools
-"${dn}"/commitmeta_to_json "${workdir}/repo" "${commit}" > commitmeta.json
+"${dn}"/commitmeta_to_json "${tmprepo}" "${commit}" > commitmeta.json
+
+# And create the ostree repo tarball containing the commit
+if [ "${commit}" == "${previous_commit}" ]; then
+    cp -a --reflink=auto "${previous_builddir}/ostree-commit.tar" .
+else
+    ostree init --repo=repo --mode=archive
+    ostree pull-local --repo=repo "${tmprepo}" "${ref}"
+    if [ -n "${ref_is_temp}" ]; then
+        ostree refs --repo=repo --delete "${ref}"
+    fi
+    # Don't compress; archive repos are already compressed individually and we'd
+    # gain ~20M at best. We could probably have better gains if we compress the
+    # whole repo in bare/bare-user mode, but that's a different story...
+    tar -cf ostree-commit.tar -C repo .
+    rm -rf repo
+fi
 
 # Clean up our temporary data
 saved_build_tmpdir="${workdir}/tmp/last-build-tmp"
 rm -rf "${saved_build_tmpdir}"
 mv -T tmp "${saved_build_tmpdir}"
+ostree prune --repo="${tmprepo}" --refs-only
 # Back to the toplevel build directory, so we can rename this one
 cd "${workdir}"/builds
 # We create a .build-commit file to note that we're in the

src/cmd-buildextend-installer

@@ -14,6 +14,7 @@ import tempfile
 
 sys.path.insert(0, '/usr/lib/coreos-assembler')
 from cmdlib import run_verbose, write_json, sha256sum_file
+from cmdlib import import_ostree_commit
 
 # Parse args and dispatch
 parser = argparse.ArgumentParser()
@@ -36,6 +37,11 @@ buildmeta_path = os.path.join(builddir, 'meta.json')
 with open(buildmeta_path) as f:
     buildmeta = json.load(f)
 
+# Grab the commit hash for this build
+buildmeta_commit = buildmeta['ostree-commit']
+
+repo = os.path.join(workdir, 'tmp/repo')
+
 # Don't run if it's already been done, unless forced
 if 'iso' in buildmeta['images'] and not args.force:
     print(f"Installer has already been built for {args.build}. Skipping.")
@@ -68,20 +74,17 @@ def generate_iso():
 
     tmpisofile = os.path.join(tmpdir, iso_name)
 
-    # Grab the commit hash for this build
-    buildmeta_commit = buildmeta['ostree-commit']
-
     # Find the directory under `/usr/lib/modules/<kver>` where the
     # kernel/initrd live. It will be the 2nd entity output by
     # `ostree ls <commit> /usr/lib/modules`
-    process = run_verbose(['/usr/bin/ostree', '--repo=./repo', 'ls',
+    process = run_verbose(['/usr/bin/ostree', 'ls', '--repo', repo,
                            '--nul-filenames-only', f"{buildmeta_commit}",
                            '/usr/lib/modules'], capture_output=True)
     moduledir = process.stdout.decode().split('\0')[1]
 
     # copy those files out of the ostree into the iso root dir
     for file in ['initramfs.img', 'vmlinuz']:
-        run_verbose(['/usr/bin/ostree', '--repo=./repo', 'checkout',
+        run_verbose(['/usr/bin/ostree', 'checkout', '--repo', repo,
                      '--user-mode', '--subpath', os.path.join(moduledir, file),
                      f"{buildmeta_commit}", tmpisoimages])
         # initramfs isn't world readable by default so let's open up perms
@@ -199,5 +202,8 @@ def generate_iso():
     print(f"Updated: {buildmeta_path}")
 
 
+commit_tar = os.path.join(builddir, 'ostree-commit.tar')
+import_ostree_commit(repo, buildmeta_commit, commit_tar)
+
 # Do it!
 generate_iso()

src/cmd-buildextend-metal

@@ -98,6 +98,25 @@ if [ "${ref}" = "null" ]; then
     ref="tmpref-${name}"
     ref_is_temp=1
 fi
+commit=$(json_key ostree-commit)
+
+ostree_repo=${tmprepo}
+rev_parsed=$(ostree rev-parse --repo="${ostree_repo}" "${ref}" 2>/dev/null || :)
+if [ "${rev_parsed}" != "${commit}" ]; then
+    # Probably an older commit or tmp/ was wiped. Let's extract it to a separate
+    # temporary repo (not to be confused with ${tmprepo}...) so we can feed it
+    # as a ref (if not temp) to Anaconda.
+    echo "Cache for build ${build} is gone"
+    echo "Importing commit ${commit} into temporary OSTree repo"
+    mkdir -p tmp/repo
+    tar -C tmp/repo -xf "${builddir}/ostree-commit.tar"
+    ostree_repo=$PWD/tmp/repo
+    if [ -n "${ref_is_temp}" ]; then
+        # this gets promptly "dereferenced" back in run_virtinstall, but it
+        # keeps the code below simple so it can work in both temp/not temp cases
+        ostree refs --repo="${ostree_repo}" "${commit}" --create "${ref}"
+    fi # otherwise, the repo already has a ref, so no need to create
+fi
 
 # for anaconda logs
 mkdir -p tmp/anaconda
@@ -110,7 +129,7 @@ for itype in ${BIOS:+metal-bios} ${UEFI:+metal-uefi}; do
     fi
 
     path=${PWD}/${img}
-    run_virtinstall "${workdir}/repo" "${ref}" "${path}.tmp" --variant="${itype}"
+    run_virtinstall "${ostree_repo}" "${ref}" "${path}.tmp" --variant="${itype}"
     /usr/lib/coreos-assembler/gf-platformid "${path}"{.tmp,} metal
     rm -f "${path}".tmp
 

src/cmd-clean

@@ -53,5 +53,4 @@ prepare_build
 cd "${workdir:?}"
 # Note we don't prune the cache.qcow2 or the objects
 # in the repo.  If you want that, just rm -rf them.
-rm -rf repo/refs/heads/* builds/* tmp/*
-ostree --repo=repo summary -u
+rm -rf builds/* tmp/*

src/cmd-init

@@ -116,4 +116,3 @@ mkdir -p cache
 mkdir -p builds
 mkdir -p tmp
 mkdir -p overrides/rpm
-ostree --repo=repo init --mode=archive

src/cmdlib.py

@@ -98,6 +98,7 @@ def fatal(msg):
     print('fatal: {}'.format(msg), file=sys.stderr)
     raise SystemExit(1)
 
+
 def info(msg):
     """
     Prints info messages.
@@ -137,3 +138,22 @@ def rm_allow_noent(path):
         os.unlink(path)
     except FileNotFoundError:
         pass
+
+
+def import_ostree_commit(repo, commit, tarfile):
+    # create repo in case e.g. tmp/ was cleared out; idempotent
+    subprocess.check_call(['ostree', 'init', '--repo', repo, '--mode=archive'])
+
+    # in the common case where we're operating on a recent build, the OSTree
+    # commit should already be in the tmprepo
+    commitpartial = os.path.join(repo, f'state/{commit}.commitpartial')
+    if (subprocess.call(['ostree', 'show', '--repo', repo, commit],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL) == 0
+            and not os.path.isfile(commitpartial)):
+        return
+
+    with tempfile.TemporaryDirectory(dir=f'{repo}/tmp') as d:
+        subprocess.check_call(['tar', '-C', d, '-xf', tarfile])
+        subprocess.check_call(['ostree', 'pull-local', '--repo', repo,
+                               d, commit])

src/cmdlib.sh

@@ -115,8 +115,8 @@ preflight() {
 
 prepare_build() {
     preflight
-    if ! [ -d repo ]; then
-        fatal "No $(pwd)/repo found; did you run coreos-assembler init?"
+    if ! [ -d builds ]; then
+        fatal "No $(pwd)/builds found; did you run coreos-assembler init?"
     elif ! has_privileges; then
         if [ ! -f cache/cache.qcow2 ]; then
             qemu-img create -f qcow2 cache/cache.qcow2 10G
@@ -135,14 +135,25 @@ prepare_build() {
 
     echo "Using manifest: ${manifest}"
 
+    tmprepo=${workdir}/tmp/repo
+    if [ ! -d "${tmprepo}" ]; then
+        # backcompat: just move the toplevel repo/
+        if [ -d "${workdir}/repo" ]; then
+            mv -T "${workdir}/repo" "${tmprepo}"
+            rm -f "${tmprepo}/summary"
+        else
+            ostree init --repo="${tmprepo}" --mode=archive
+        fi
+    fi
+
     configdir_gitrepo=${configdir}
     if [ -e "${workdir}/src/config-git" ]; then
         configdir_gitrepo="${workdir}/src/config-git"
     fi
     export configdir_gitrepo
 
     manifest_tmp_json=${workdir}/tmp/manifest.json
-    rpm-ostree compose tree --repo=repo --print-only "${manifest}" > "${manifest_tmp_json}"
+    rpm-ostree compose tree --repo="${tmprepo}" --print-only "${manifest}" > "${manifest_tmp_json}"
 
     # Abuse the rojig/name as the name of the VM images
     # Also grab rojig summary for image upload descriptions
@@ -230,16 +241,16 @@ EOF
     rm -f "${changed_stamp}"
 
     # shellcheck disable=SC2086
-    set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree --repo="${workdir}"/repo \
+    set - ${COSA_RPMOSTREE_GDB:-} rpm-ostree compose tree --repo="${tmprepo}" \
             --cachedir="${workdir}"/cache --touch-if-changed "${changed_stamp}" \
-            --unified-core "${manifest}" ${COSA_RPMOSTREE_ARGS:-} "$@"
+            --unified-core --no-parent "${manifest}" ${COSA_RPMOSTREE_ARGS:-} "$@"
 
     echo "Running: $*"
 
     # this is the heart of the privs vs no privs dual path
     if has_privileges; then
         sudo -E "$@"
-        sudo chown -R -h "${USER}":"${USER}" "${workdir}"/repo
+        sudo chown -R -h "${USER}":"${USER}" "${tmprepo}"
     else
         runvm "$@"
     fi

src/prune_builds

@@ -174,20 +174,3 @@ if skip_pruning:
 for build in builds_to_delete:
     print(f"Pruning build {build.id}")
     shutil.rmtree(os.path.join(builds_dir, build.id))
-
-# and finally prune OSTree repos
-print(f"Pruning repo")
-repo = os.path.join(args.workdir, 'repo')
-
-# For now, we just use the --keep-younger-than CLI here. Doing this more
-# accurately would require enhancing the `prune` CLI (or just use the API
-# directly?). Or we could also manually figure out the depth of the oldest
-# build and then use that as the arg to `--depth`.
-oldest_ostree_t = new_builds[-1].ostree_timestamp
-# In a quick test, it seems like --keep-younger-than=x actually keeps commits
-# with timestamps exactly equal to x, but the name is a bit tricky so let's
-# be safe and just pick a time 1h before that so we're doubly sure. It might
-# keep a few other older commits, but meh...
-younger_than = rfc3339_time(oldest_ostree_t - timedelta(hours=1))
-subprocess.run(["ostree", "prune", "--repo", repo, "--refs-only",
-               f"--keep-younger-than={younger_than}"], check=True)