From 66807dfc04374f7621d9a3e2fa91cc938c7f468c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 6 Feb 2020 22:21:27 +0000 Subject: [PATCH] upload-oscontainer: New command This code lives today in the RHCOS pipeline (internal gitlab); to be honest, I have no idea why only the "core" oscontainer bits ended up in cosa upstream and the script lived downstream. Let's fix that, moving all of the logic here. Prep for further work. Make this work a bit better too by automatically copying in `~/.docker/config.json` into the supermin appliance, which is something the RHCOS pipeline does. And yes I hope to clean this all up more in the future but for now let's get the code out of the darkness and into the light. --- src/cmd-oscontainer | 12 ++++- src/cmd-upload-oscontainer | 96 ++++++++++++++++++++++++++++++++++++++ src/coreos-assembler | 2 +- 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100755 src/cmd-upload-oscontainer diff --git a/src/cmd-oscontainer b/src/cmd-oscontainer index 897bee2751..3e952ece51 100755 --- a/src/cmd-oscontainer +++ b/src/cmd-oscontainer @@ -19,8 +19,18 @@ if [ "$#" -eq 0 ]; then fi if has_privileges; then + # Deal with nested containerization + if [ -n "${container:-}" ] && test "$(id -u)" != 0; then + exec buildah unshare "${osc}" "$@" + fi exec "${osc}" "$@" else info "Required privileges not detected; running via supermin appliance" - runvm -- "${osc}" --workdir /host/container-work "$@" + if [ -f ~/.docker/config.json ]; then + info "Using ~/.docker/config.json" + cp ~/.docker/config.json "${workdir}/tmp/docker-auth.json" + else + info "warning: No ~/.docker/config.json found, push will likely fail" + fi + runvm -- env PYTHONUNBUFFERED=1 "${osc}" --authfile="${workdir}/tmp/docker-auth.json" --workdir /host/container-work "$@" fi diff --git a/src/cmd-upload-oscontainer b/src/cmd-upload-oscontainer new file mode 100755 index 0000000000..c04b2ef704 --- /dev/null +++ b/src/cmd-upload-oscontainer @@ -0,0 +1,96 @@ +#!/usr/bin/python3 -u +# Upload an oscontainer. This is a wrapper for +# `cosa oscontainer` that just for historical reasons +# used to live downstream in the redhat-coreos pipeline. +# In the future we should just have one `cosa oscontainer` +# command. + +import argparse +import json +import os +import shutil +import subprocess +import sys + +cosa_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, cosa_dir) + +from cosalib import cmdlib + +parser = argparse.ArgumentParser() +parser.add_argument("--name", help="oscontainer name", + action='store', required=True) +parser.add_argument("--from", help="Base image", default='scratch', + dest='from_image') +parser.add_argument("--add-directory", help="Copy in all content from referenced directory DIR", + metavar='DIR', action='append', default=[]) + +args = parser.parse_args() + +with open('builds/builds.json') as f: + builds = json.load(f)['builds'] +if len(builds) == 0: + cmdlib.fatal("No builds found") +latest_build = builds[0]['id'] +arch = cmdlib.get_basearch() +latest_build_path = f"builds/{latest_build}/{arch}" + +metapath = f"{latest_build_path}/meta.json" +with open(metapath) as f: + meta = json.load(f) + +print("Preparing to upload oscontainer for build: {}".format(latest_build)) + +osc_workdir = "{}/tmp/oscontainer-work".format(os.getcwd()) + +tmprepo = "{}/tmp/repo".format(os.getcwd()) +# if tmprepo is not a directory, but is unexpectedly a file, +# just nuke it +if not os.path.isdir(tmprepo) and os.path.exists(tmprepo): + os.remove(tmprepo) + +# if tmprepo is not a directory and not a file, recreate from +# the tarfile +if not os.path.exists(tmprepo): + os.makedirs(tmprepo, exist_ok=True) + ostree_commit_tar = 'ostree-commit.tar' + if 'ostree' in meta['images']: + ostree_commit_tar = meta['images']['ostree']['path'] + subprocess.check_call(['tar', '-xf', + f'{latest_build_path}/{ostree_commit_tar}', + '-C', tmprepo]) + + +# The build ID is the container tag +osc_name_and_tag = "{}:{}".format(args.name, latest_build) + +# TODO: Use labels for the build hash and avoid pulling the oscontainer +# every time we want to poll. +# TODO: Remove --from +# We have removed the use of `--workdir` from this script because +# `coreos-assembler` assumes the working diretory in the unprivileged case +# See: https://github.com/coreos/coreos-assembler/pull/360 +inspect_out = os.getcwd() + '/oscontainer.json' +cosa_argv = ['coreos-assembler', 'oscontainer', 'build', f"--from={args.from_image}"] +for d in args.add_directory: + cosa_argv.append(f"--add-directory={d}") +subprocess.check_call(cosa_argv + + ['--inspect-out=' + inspect_out, + '--push', tmprepo, + meta['ostree-commit'], + osc_name_and_tag]) + +with open(inspect_out) as f: + osc_inspect = json.load(f) +osc_digest = osc_inspect['Digest'] + +# Inject the oscontainer with SHA256 into the build metadata +meta['oscontainer'] = {'image': args.name, + 'digest': osc_digest} +metapath_new = f"{metapath}.new" +with open(metapath_new, 'w') as f: + json.dump(meta, f, sort_keys=True) +os.rename(metapath_new, metapath) + +# Cleanup workdirs +shutil.rmtree(osc_workdir, ignore_errors=True) diff --git a/src/coreos-assembler b/src/coreos-assembler index 61c0c4d82c..a8eda665e8 100755 --- a/src/coreos-assembler +++ b/src/coreos-assembler @@ -42,7 +42,7 @@ cmd=${1:-} # commands we'd expect to use in the local dev path build_commands="init fetch build run prune clean list" # commands more likely to be used in a prod pipeline only -advanced_build_commands="buildprep buildupload oscontainer" +advanced_build_commands="buildprep buildupload oscontainer upload-oscontainer" buildextend_commands="aws azure digitalocean gcp ibmcloud installer live metal metal4k openstack qemu vmware vultr exoscale" utility_commands="tag sign compress koji-upload kola aws-replicate remote-prune" other_commands="shell meta"