diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index e669dcea6e..3de30404cc 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -42,6 +42,8 @@ coreos.pod([image: 'registry.fedoraproject.org/fedora:31', runAsUser: 0, kvm: tr cosa_cmd("buildextend-openstack") cosa_cmd("buildextend-vmware") cosa_cmd("compress") + // quick schema validation" + cosa_cmd("meta --get name") cosa_cmd("buildupload --dry-run s3 --acl=public-read my-nonexistent-bucket/my/prefix") } } diff --git a/Makefile b/Makefile index a21587c19c..ebdef7b86b 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,9 @@ flake8: # find src -maxdepth 1 -name "*.py" | xargs flake8 --ignore=$(PYIGNORE) unittest: - PYTHONPATH=`pwd`/src python3 -m pytest tests/ + COSA_TEST_META_JSON=`pwd`/fixtures/rhcos.json \ + COSA_META_SCHEMA=`pwd`/src/schema/v1.json \ + PYTHONPATH=`pwd`/src python3 -m pytest tests/ clean: rm -f ${src_checked} ${tests_checked} ${cwd_checked} @@ -60,6 +62,8 @@ install: cp -df -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler $$(find src/ -maxdepth 1 -type l) install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/cosalib $$(find src/cosalib/ -maxdepth 1 -type f) + install -d $(DESTDIR)$(PREFIX)/lib/coreos-assembler/schema + install -D -t $(DESTDIR)$(PREFIX)/lib/coreos-assembler/schema $$(find src/schema/ -maxdepth 1 -type f) install -d $(DESTDIR)$(PREFIX)/bin ln -sf ../lib/coreos-assembler/coreos-assembler $(DESTDIR)$(PREFIX)/bin/ ln -sf ../lib/coreos-assembler/cp-reflink $(DESTDIR)$(PREFIX)/bin/ diff --git a/fixtures/rhcos.json b/fixtures/rhcos.json new file mode 100644 index 0000000000..a84614a390 --- /dev/null +++ b/fixtures/rhcos.json @@ -0,0 +1,160 @@ +{ + "build-url": "https://example.org/job/rhcos-rhcos-4.4/5/", + "buildid": "44devel.81.202001151926.0", + "coreos-assembler.basearch": "x86_64", + "coreos-assembler.build-timestamp": "2020-01-15T19:32:19Z", + "coreos-assembler.code-source": "container", + "coreos-assembler.config-dirty": "true", + "coreos-assembler.config-gitrev": "v3.1-728-g742edc307e58f35824d906958b6493510e12b593", + "coreos-assembler.container-config-git": { + "branch": "HEAD", + "commit": "742edc307e58f35824d906958b6493510e12b593", + "dirty": "true", + "origin": "https://example.org/coreos/redhat-coreos.git" + }, + "coreos-assembler.container-image-git": { + "branch": "HEAD", + "commit": "e41cbf0422adbb468911734b0516ebf4e7f977f4", + "dirty": "false", + "origin": "git@github.com:coreos/coreos-assembler.git" + }, + "coreos-assembler.image-config-checksum": "f15f5b25cf138a7683e3d200c53ece2091bf71d31332135da87892ab72ff4ee3", + "coreos-assembler.image-genver": 0, + "coreos-assembler.image-input-checksum": "59b0904f91aafcf55a66075b731476f802c9d60f17b0c670fb5c43d26333b876", + "images": { + "ostree": { + "path": "rhcos-44devel.81.202001151926.0-ostree.x86_64.tar", + "sha256": "43375fa20ea1ff31ce83110b604cfab518da5b6b5a6c9c4c6cde3449d862c530", + "size": 814407680 + }, + "qemu": { + "path": "rhcos-44devel.81.202001151926.0-qemu.x86_64.qcow2.gz", + "sha256": "32d6716fb5df55457870298e24bdbf695d8c9127458d8fbfb0f7820e860901d5", + "size": 893759709, + "uncompressed-sha256": "72775ed71e40fce806a5a76bee73b22155f9a2d2ef1a9a9ea9a1a12f5bbbf3ac", + "uncompressed-size": 2476539904 + }, + "metal": { + "path": "rhcos-44devel.81.202001151926.0-metal.x86_64.raw.gz", + "sha256": "2899aa904e7646a10f25dae6ecc2c1099673b3fd39c122265d2d5faa5b9a7595", + "size": 894265661, + "uncompressed-sha256": "0f135871fe452b66f28383f5882aa5544fdb700755a68fbbb4b3dc42e3c5896e", + "uncompressed-size": 3824156672 + }, + "iso": { + "path": "rhcos-44devel.81.202001151926.0-installer.x86_64.iso", + "sha256": "770d6372dc469c2704b0ffff8b3c7f655115ddda8505d64789457ae450d79a54" + }, + "kernel": { + "path": "rhcos-44devel.81.202001151926.0-installer-kernel-x86_64", + "sha256": "7ace7ebdb828e1dc4d242b2fb8a360e7b97da7748d2fde4ffa3bd30232c04865" + }, + "initramfs": { + "path": "rhcos-44devel.81.202001151926.0-installer-initramfs.x86_64.img", + "sha256": "6cf23a16b8da57d35a0b066dfca7c2a2815654a0e84e65ba463a8ae45031a05b" + }, + "openstack": { + "path": "rhcos-44devel.81.202001151926.0-openstack.x86_64.qcow2.gz", + "sha256": "72adb012bda15edc9d73052a534c1e0622c0671312691df9688f89bc4cf80f98", + "size": 893754996, + "uncompressed-sha256": "f43e20128d46e33c3a3a05985848647be11d51087cf2c24f6aee9310b4c53868", + "uncompressed-size": 2476605440 + }, + "vmware": { + "path": "rhcos-44devel.81.202001151926.0-vmware.x86_64.ova", + "sha256": "13980e76ca5dfe17cdbf1cae8d0e725805a99177136ac74fd95755b5eb61771c", + "size": 927324160 + }, + "aliyun": { + "path": "rhcos-44devel.81.202001151926.0-aliyun.x86_64.qcow2.gz", + "sha256": "1503360deb4ff46d662284ef00cc7c0e4cb96a6a0588ec10fb562238f1f1cf46", + "size": 893754990, + "uncompressed-sha256": "5abd1223cfd0283c0be7a2a29bf0326b37ed14b6771916c17806dc9dae22bdf1", + "uncompressed-size": 2476605440 + }, + "aws": { + "path": "rhcos-44devel.81.202001151926.0-aws.x86_64.vmdk.gz", + "sha256": "96481a13c5a4cc25eb718abf1e032cf68c15444d7d4fbf98fe7ab01e72d12ee6", + "size": 908650544, + "uncompressed-sha256": "7d9216eeb942df24a46c320667222c2f7677290d631c2dace2874a5b8be4e833", + "uncompressed-size": 927316480 + }, + "azure": { + "path": "rhcos-44devel.81.202001151926.0-azure.x86_64.vhd.gz", + "sha256": "73ac7c8ab0e7d08fb991425d184ab363b96c5604597ace73625ed2734e9ff43f", + "size": 893094276, + "uncompressed-sha256": "910e3a0a9c59eea1dd9303414fa987377f301cd519c52573cdf993793711f1d8", + "uncompressed-size": 2521427456 + }, + "gcp": { + "path": "rhcos-44devel.81.202001151926.0-gcp.x86_64.tar.gz", + "sha256": "842e04f40889ca05da54f1a8acd263f273dd72fce8e004264739f38eaf46f11d", + "size": 892692916 + } + }, + "name": "rhcos", + "oscontainer": { + "digest": "sha256:f8d18011913e87ae59d3781f3d07819267d43401ab444fb3b5794a5eb392b915", + "image": "registry.svc.ci.openshift.org/rhcos-devel/machine-os-content" + }, + "ostree-commit": "9665ab0cfd4a995cf70f1a3bb678d3515a03f7d3b5bb87d723ba06c26f0daa6e", + "ostree-content-bytes-written": 156269945, + "ostree-content-checksum": "e02647edba305ad68e2c7c5bb3a2c7765eb4ea6aadd1ebf8e538e459ebf99ed7", + "ostree-n-cache-hits": 19185, + "ostree-n-content-total": 3688, + "ostree-n-content-written": 1210, + "ostree-n-metadata-total": 9225, + "ostree-n-metadata-written": 3015, + "ostree-timestamp": "2020-01-15T19:31:31Z", + "ostree-version": "44devel.81.202001151926.0", + "pkgdiff": [ + [ + "machine-config-daemon", + 2, + { + "NewPackage": [ + "machine-config-daemon", + "4.4.0-202001151823.git.1.7a12db8.el8", + "x86_64" + ], + "PreviousPackage": [ + "machine-config-daemon", + "4.4.0-202001151223.git.1.ca1f2c2.el8", + "x86_64" + ] + } + ] + ], + "rpm-ostree-inputhash": "13de3656ed8f55f8b8bafeab7a2320496c247cf533063e3d3daa63a95592f1ac", + "summary": "OpenShift 4", + "aliyun": [ + { + "name": "us-west-1", + "id": "m-rj000000aaaa111azzzz" + }, + { + "name": "cn-shanghai", + "id": "m-rj000000aaaa111azzzz" + } + ], + "amis": [ + { + "name": "us-east-1", + "hvm": "ami-012345678abcdef00", + "snapshot": "snap-012345678abcdef00" + }, + { + "name": "eu-north-1", + "hvm": "ami-012345678abcdef00", + "snapshot": "snap-012345678abcdef00" + } + ], + "azure": { + "image": "rhcos-44devel.81.202001151926.0-azure.x86_64.vhd", + "url": "https://example.org/imagebucket/rhcos-44devel.81.202001151926.0-azure.x86_64.vhd" + }, + "gcp": { + "image": "rhcos-44devel-81-202001151926-0", + "url": "https://example.org/rhcos-devel/devel/rhcos/44devel.81.202001151926.0.tar.gz" + } +} diff --git a/src/cmd-meta b/src/cmd-meta index 5478acc507..3872f0c35f 100755 --- a/src/cmd-meta +++ b/src/cmd-meta @@ -7,7 +7,8 @@ import argparse import os import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +COSA_PATH = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, COSA_PATH) from cosalib.meta import GenericBuildMeta as Meta @@ -15,6 +16,12 @@ def new_cli(): parser = argparse.ArgumentParser() parser.add_argument('--workdir', default=os.getcwd()) parser.add_argument('--build', default='latest') + parser.add_argument('--skip-validation', + help='do not validate meta.json', + action='store_true') + parser.add_argument('--schema', help='location of meta.json schema', + default=os.environ.get("COSA_META_SCHEMA", + f'{COSA_PATH}/schema/v1.json')) sub_parser = parser.add_mutually_exclusive_group(required=True) sub_parser.add_argument('--get', help='get a field', action='append') sub_parser.add_argument('--set', help='set a field', action='append') @@ -26,7 +33,10 @@ def new_cli(): metavar='IMAGETYPE') args = parser.parse_args() - meta = Meta(args.workdir, args.build) + schema = args.schema + if args.skip_validation: + schema = None + meta = Meta(args.workdir, args.build, schema=schema) # support the coreos-assembler.* keys def pather(val): diff --git a/src/coreos-assembler b/src/coreos-assembler index ae9d4fc173..e8d1775754 100755 --- a/src/coreos-assembler +++ b/src/coreos-assembler @@ -73,6 +73,8 @@ if [ -z "${cmd}" ]; then fi shift +export COSA_META_SCHEMA="/usr/lib/coreos-assembler/schema/v1.json" + target=/usr/lib/coreos-assembler/cmd-${cmd} if test -x "${target}"; then exec "${target}" "$@" diff --git a/src/cosalib/meta.py b/src/cosalib/meta.py index 86e146ac37..ec4e7c0e6d 100644 --- a/src/cosalib/meta.py +++ b/src/cosalib/meta.py @@ -1,4 +1,5 @@ import json +import jsonschema import os.path from cosalib.builds import Builds @@ -7,12 +8,25 @@ write_json) +SCHEMA_PATH = os.environ.get("COSA_META_SCHEMA", + "/usr/lib/coreos-assembler/schema/v1.json") + + +class COSAInvalidMeta(Exception): + """ + Raised when meta.json does not validate + """ + def __str__(self): + return f"meta.json is or would be invalid: {': '.join(self.args)}" + + class GenericBuildMeta(dict): """ GenericBuildMeta interacts with a builds meta.json """ - def __init__(self, workdir=None, build='latest'): + def __init__(self, workdir=None, build='latest', + schema=SCHEMA_PATH): builds = Builds(workdir) if build != "latest": if not builds.has(build): @@ -20,6 +34,15 @@ def __init__(self, workdir=None, build='latest'): else: build = builds.get_latest() + # Load the schema + self._validator = None + self._schema_path = schema + if schema: + with open(schema, 'r') as data: + self._validator = jsonschema.Draft7Validator( + json.loads(data.read()) + ) + self._meta_path = os.path.join( builds.get_build_dir(build), 'meta.json') self.read() @@ -28,6 +51,15 @@ def __init__(self, workdir=None, build='latest'): def path(self): return self._meta_path + def validate(self): + """ + validate ensures that the meta structure matches the schema + expected. + """ + if not self._validator: + return + self._validator.validate(dict(self)) + def read(self): """ Read the meta.json file into this object instance. @@ -36,11 +68,13 @@ def read(self): self.clear() # Load the file self.update(load_json(self._meta_path)) + self.validate() def write(self): """ Write out the dict to the meta path. """ + self.validate() write_json(self._meta_path, dict(self)) def get(self, *args): @@ -71,6 +105,9 @@ def get(self, *args): except KeyError: return default + def dict(self): + return dict(self) + def set(self, pathing, value): """ Sets key path to a value. diff --git a/src/deps.txt b/src/deps.txt index 883a4ddb50..a188e275ac 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -64,3 +64,6 @@ fedora-messaging # For debugging running processes in the pipelines strace + +# Used to validate the meta.json schema +python3-jsonschema diff --git a/src/schema/v1.json b/src/schema/v1.json new file mode 100644 index 0000000000..61423aec0d --- /dev/null +++ b/src/schema/v1.json @@ -0,0 +1,1193 @@ +{ + "definitions": {}, + "$schema":"http://json-schema.org/draft-07/schema#", + "$id":"http://github.com/coreos/coreos-assembler/blob/master/schema/v1.json", + "type":"object", + "title":"CoreOS v1 meta schema", + "required": [ + "buildid", + "coreos-assembler.basearch", + "coreos-assembler.build-timestamp", + "coreos-assembler.code-source", + "coreos-assembler.config-dirty", + "coreos-assembler.config-gitrev", + "coreos-assembler.container-config-git", + "coreos-assembler.container-image-git", + "coreos-assembler.image-config-checksum", + "coreos-assembler.image-genver", + "coreos-assembler.image-input-checksum", + "images", + "name", + "ostree-commit", + "ostree-content-bytes-written", + "ostree-content-checksum", + "ostree-n-cache-hits", + "ostree-n-content-total", + "ostree-n-content-written", + "ostree-n-metadata-total", + "ostree-n-metadata-written", + "ostree-timestamp", + "ostree-version", + "rpm-ostree-inputhash", + "summary" + ], + "optional": [ + "aliyun", + "amis", + "azure", + "build-url", + "gcp", + "oscontainer", + "pkgdiff" + ], + "properties": { + "build-url": { + "$id":"#/properties/build-url", + "type":"string", + "title":"The Build-url", + "default":"", + "examples": [ + "https://jenkins-host.example.com/job/rhcos/5/" + ], + "pattern":"^(.*)$" + }, + "buildid": { + "$id":"#/properties/buildid", + "type":"string", + "title":"The Buildid", + "default":"", + "examples": [ + "31-202001010000-0" + ], + "pattern":"^(.*)$" + }, + "coreos-assembler.basearch": { + "$id":"#/properties/coreos-assembler.basearch", + "type":"string", + "title":"Architecture Schema", + "default":"", + "examples": [ + "x86_64" + ], + "pattern":"^(.*)$" + }, + "coreos-assembler.build-timestamp": { + "$id":"#/properties/coreos-assembler.build-timestamp", + "type":"string", + "title":"Build Time Stamp schema", + "default":"", + "examples": [ + "2020-01-15T19:32:19Z" + ], + "pattern":"^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\\.[0-9]+)?(Z)?$" + }, + "coreos-assembler.code-source": { + "$id":"#/properties/coreos-assembler.code-source", + "type":"string", + "title":"CoreOS Source", + "default":"", + "examples": [ + "container" + ], + "pattern":"^(.*)$" + }, + "coreos-assembler.config-dirty": { + "$id":"#/properties/coreos-assembler.config-dirty", + "type":"string", + "title":"Indication if the source has been overrriden", + "default":"", + "examples": [ + "true" + ], + "pattern":"^(.*)$" + }, + "coreos-assembler.config-gitrev": { + "$id":"#/properties/coreos-assembler.config-gitrev", + "type":"string", + "title":"Config GitRev for Build", + "default":"", + "examples": [ + "v3.1-728-g742edc307e58f35824d906958b6493510e12b593" + ], + "pattern":"^(.*)$" + }, + "coreos-assembler.container-config-git": { + "$id":"#/properties/coreos-assembler.container-config-git", + "type":"object", + "title":"COSAcontainer-config-git", + "required": [ + "branch", + "commit", + "dirty", + "origin" + ], + "properties": { + "branch": { + "$id":"#/properties/coreos-assembler.container-config-git/properties/branch", + "type":"string", + "title":"Git branch for COSA", + "default":"", + "examples": [ + "HEAD" + ], + "pattern":"^(.*)$" + }, + "commit": { + "$id":"#/properties/coreos-assembler.container-config-git/properties/commit", + "type":"string", + "title":"COSA git ref", + "default":"", + "examples": [ + "742edc307e58f35824d906958b6493510e12b593" + ], + "pattern":"\\b[0-9a-f]{5,40}\\b" + }, + "dirty": { + "$id":"#/properties/coreos-assembler.container-config-git/properties/dirty", + "type":"string", + "title":"Indicates if COSA checkout is dirty", + "default":"", + "examples": [ + "true" + ], + "pattern":"^(.*)$" + }, + "origin": { + "$id":"#/properties/coreos-assembler.container-config-git/properties/origin", + "type":"string", + "title":"Build config origin", + "default":"", + "examples": [ + "https://github.com/coreos/fedora-coreos-config" + ], + "pattern":"^(.*)$" + } + } + }, + "coreos-assembler.container-image-git": { + "$id":"#/properties/coreos-assembler.container-image-git", + "type":"object", + "title":"COSA Container Image Git Source", + "required": [ + "branch", + "commit", + "dirty", + "origin" + ], + "properties": { + "branch": { + "$id":"#/properties/coreos-assembler.container-image-git/properties/branch", + "type":"string", + "title":"COSA branch", + "default":"", + "examples": [ + "HEAD" + ], + "pattern":"^(.*)$" + }, + "commit": { + "$id":"#/properties/coreos-assembler.container-image-git/properties/commit", + "type":"string", + "title":"COSA Commit", + "default":"", + "examples": [ + "e41cbf0422adbb468911734b0516ebf4e7f977f4" + ], + "pattern":"\\b[0-9a-f]{5,40}\\b" + }, + "dirty": { + "$id":"#/properties/coreos-assembler.container-image-git/properties/dirty", + "type":"string", + "title":"Branch state is dirty", + "default":"", + "examples": [ + "false" + ], + "pattern":"^(.*)$" + }, + "origin": { + "$id":"#/properties/coreos-assembler.container-image-git/properties/origin", + "type":"string", + "title":"Origin URL of the COSA", + "default":"", + "examples": [ + "git@github.com:coreos/coreos-assembler.git" + ], + "pattern":"^(.*)$" + } + } + }, + "coreos-assembler.image-config-checksum": { + "$id":"#/properties/coreos-assembler.image-config-checksum", + "type":"string", + "title":"COSA image checksum", + "default":"", + "examples": [ + "f15f5b25cf138a7683e3d200c53ece2091bf71d31332135da87892ab72ff4ee3" + ], + "pattern":"[A-Fa-f0-9]{64}$" + }, + "coreos-assembler.image-genver": { + "$id":"#/properties/coreos-assembler.image-genver", + "type":"integer", + "title":"COSA Image Version", + "default": 0, + "examples": [ + 0 + ] + }, + "coreos-assembler.image-input-checksum": { + "$id":"#/properties/coreos-assembler.image-input-checksum", + "type":"string", + "title":"Image input checksum", + "default":"", + "examples": [ + "59b0904f91aafcf55a66075b731476f802c9d60f17b0c670fb5c43d26333b876" + ], + "pattern":"^(.*)$" + }, + "images": { + "$id":"#/properties/images", + "type":"object", + "title":"Build Artifacts", + "required": [ + "ostree" + ], + "optional": [ + "aliyun", + "aws", + "azure", + "gcp", + "initramfs", + "iso", + "kernel", + "metal", + "openstack", + "qemu", + "vmware" + ], + "properties": { + "ostree": { + "$id":"#/properties/images/properties/ostree", + "type":"object", + "title":"OSTree", + "required": [ + "path", + "sha256", + "size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/ostree/properties/path", + "type":"string", + "title":"Relative Path of OSTree Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-ostree.x86_64.tar" + ], + "pattern":"(.*).tar(|.xz|.gz)" + }, + "sha256": { + "$id":"#/properties/images/properties/ostree/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "43375fa20ea1ff31ce83110b604cfab518da5b6b5a6c9c4c6cde3449d862c530" + ], + "pattern":"[A-Fa-f0-9]{64}$" + }, + "size": { + "$id":"#/properties/images/properties/ostree/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 814407680 + ] + } + } + }, + "qemu": { + "$id":"#/properties/images/properties/qemu", + "type":"object", + "title":"The Qemu", + "required": [ + "path", + "sha256", + "size" + ], + "optional": [ + "uncompressed-sha256", + "uncompressed-size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/qemu/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-qemu.x86_64.qcow2.gz" + ], + "pattern":"^(.*).qcow2(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/qemu/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "32d6716fb5df55457870298e24bdbf695d8c9127458d8fbfb0f7820e860901d5" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/qemu/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 893759709 + ] + }, + "uncompressed-sha256": { + "$id":"#/properties/images/properties/qemu/properties/uncompressed-sha256", + "type":"string", + "title":"The Uncompressed-sha256", + "default":"", + "examples": [ + "72775ed71e40fce806a5a76bee73b22155f9a2d2ef1a9a9ea9a1a12f5bbbf3ac" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "uncompressed-size": { + "$id":"#/properties/images/properties/qemu/properties/uncompressed-size", + "type":"integer", + "title":"The Uncompressed-size", + "default": 0, + "examples": [ + 2476539904 + ] + } + } + }, + "metal": { + "$id":"#/properties/images/properties/metal", + "type":"object", + "title":"The Metal", + "required": [ + "path", + "sha256", + "size" + ], + "optional": [ + "uncompressed-sha256", + "uncompressed-size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/metal/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-metal.x86_64.raw.gz" + ], + "pattern":"^(.*).raw(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/metal/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "2899aa904e7646a10f25dae6ecc2c1099673b3fd39c122265d2d5faa5b9a7595" + ], + "pattern":"^(.*)$" + }, + "size": { + "$id":"#/properties/images/properties/metal/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 894265661 + ] + }, + "uncompressed-sha256": { + "$id":"#/properties/images/properties/metal/properties/uncompressed-sha256", + "type":"string", + "title":"The Uncompressed-sha256", + "default":"", + "examples": [ + "0f135871fe452b66f28383f5882aa5544fdb700755a68fbbb4b3dc42e3c5896e" + ], + "pattern": "[A-Fa-f0-9]{64}" + }, + "uncompressed-size": { + "$id":"#/properties/images/properties/metal/properties/uncompressed-size", + "type":"integer", + "title":"The Uncompressed-size", + "default": 0, + "examples": [ + 3824156672 + ] + } + } + }, + "iso": { + "$id":"#/properties/images/properties/iso", + "type":"object", + "title":"The Iso", + "required": [ + "path", + "sha256" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/iso/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-installer.x86_64.iso" + ], + "pattern":"^(.*).iso(|.gz|xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/iso/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "770d6372dc469c2704b0ffff8b3c7f655115ddda8505d64789457ae450d79a54" + ], + "pattern": "[A-Fa-f0-9]{64}" + } + } + }, + "kernel": { + "$id":"#/properties/images/properties/kernel", + "type":"object", + "title":"The Kernel", + "required": [ + "path", + "sha256" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/kernel/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-installer-kernel-x86_64" + ], + "pattern":"^(.*)-kernel-(.*)$" + }, + "sha256": { + "$id":"#/properties/images/properties/kernel/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "7ace7ebdb828e1dc4d242b2fb8a360e7b97da7748d2fde4ffa3bd30232c04865" + ], + "pattern":"[A-Fa-f0-9]{64}" + } + } + }, + "initramfs": { + "$id":"#/properties/images/properties/initramfs", + "type":"object", + "title":"The Initramfs", + "required": [ + "path", + "sha256" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/initramfs/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-installer-initramfs.x86_64.img" + ], + "pattern":"^(.*).img$" + }, + "sha256": { + "$id":"#/properties/images/properties/initramfs/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "6cf23a16b8da57d35a0b066dfca7c2a2815654a0e84e65ba463a8ae45031a05b" + ], + "pattern":"^(.*)$" + } + } + }, + "openstack": { + "$id":"#/properties/images/properties/openstack", + "type":"object", + "title":"The Openstack", + "required": [ + "path", + "sha256", + "size" + ], + "optional": [ + "uncompressed-sha256", + "uncompressed-size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/openstack/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-openstack.x86_64.qcow2.gz" + ], + "pattern":"^(.*).qcow2(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/openstack/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "72adb012bda15edc9d73052a534c1e0622c0671312691df9688f89bc4cf80f98" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/openstack/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 893754996 + ] + }, + "uncompressed-sha256": { + "$id":"#/properties/images/properties/openstack/properties/uncompressed-sha256", + "type":"string", + "title":"The Uncompressed-sha256", + "default":"", + "examples": [ + "f43e20128d46e33c3a3a05985848647be11d51087cf2c24f6aee9310b4c53868" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "uncompressed-size": { + "$id":"#/properties/images/properties/openstack/properties/uncompressed-size", + "type":"integer", + "title":"The Uncompressed-size", + "default": 0, + "examples": [ + 2476605440 + ] + } + } + }, + "vmware": { + "$id":"#/properties/images/properties/vmware", + "type":"object", + "title":"The Vmware", + "required": [ + "path", + "sha256", + "size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/vmware/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-vmware.x86_64.ova" + ], + "pattern":"^(.*).ova$" + }, + "sha256": { + "$id":"#/properties/images/properties/vmware/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "13980e76ca5dfe17cdbf1cae8d0e725805a99177136ac74fd95755b5eb61771c" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/vmware/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 927324160 + ] + } + } + }, + "aliyun": { + "$id":"#/properties/images/properties/aliyun", + "type":"object", + "title":"The Aliyun", + "required": [ + "path", + "sha256", + "size" + ], + "optional": [ + "uncompressed-sha256", + "uncompressed-size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/aliyun/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-aliyun.x86_64.qcow2.gz" + ], + "pattern":"^(.*).qcow2(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/aliyun/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "1503360deb4ff46d662284ef00cc7c0e4cb96a6a0588ec10fb562238f1f1cf46" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/aliyun/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 893754990 + ] + }, + "uncompressed-sha256": { + "$id":"#/properties/images/properties/aliyun/properties/uncompressed-sha256", + "type":"string", + "title":"The Uncompressed-sha256", + "default":"", + "examples": [ + "5abd1223cfd0283c0be7a2a29bf0326b37ed14b6771916c17806dc9dae22bdf1" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "uncompressed-size": { + "$id":"#/properties/images/properties/aliyun/properties/uncompressed-size", + "type":"integer", + "title":"The Uncompressed-size", + "default": 0, + "examples": [ + 2476605440 + ] + } + } + }, + "aws": { + "$id":"#/properties/images/properties/aws", + "type":"object", + "title":"The Aws", + "required": [ + "path", + "sha256", + "size" + ], + "optional": [ + "uncompressed-sha256", + "uncompressed-size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/aws/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-aws.x86_64.vmdk.gz" + ], + "pattern":"^(.*).vmdk(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/aws/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "96481a13c5a4cc25eb718abf1e032cf68c15444d7d4fbf98fe7ab01e72d12ee6" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/aws/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 908650544 + ] + }, + "uncompressed-sha256": { + "$id":"#/properties/images/properties/aws/properties/uncompressed-sha256", + "type":"string", + "title":"The Uncompressed-sha256", + "default":"", + "examples": [ + "7d9216eeb942df24a46c320667222c2f7677290d631c2dace2874a5b8be4e833" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "uncompressed-size": { + "$id":"#/properties/images/properties/aws/properties/uncompressed-size", + "type":"integer", + "title":"The Uncompressed-size", + "default": 0, + "examples": [ + 927316480 + ] + } + } + }, + "azure": { + "$id":"#/properties/images/properties/azure", + "type":"object", + "title":"The Azure", + "required": [ + "path", + "sha256", + "size" + ], + "optional": [ + "uncompressed-sha256", + "uncompressed-size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/azure/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-azure.x86_64.vhd.gz" + ], + "pattern":"^(.*).vhd(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/azure/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "73ac7c8ab0e7d08fb991425d184ab363b96c5604597ace73625ed2734e9ff43f" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/azure/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 893094276 + ] + }, + "uncompressed-sha256": { + "$id":"#/properties/images/properties/azure/properties/uncompressed-sha256", + "type":"string", + "title":"The Uncompressed-sha256", + "default":"", + "examples": [ + "910e3a0a9c59eea1dd9303414fa987377f301cd519c52573cdf993793711f1d8" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "uncompressed-size": { + "$id":"#/properties/images/properties/azure/properties/uncompressed-size", + "type":"integer", + "title":"The Uncompressed-size", + "default": 0, + "examples": [ + 2521427456 + ] + } + } + }, + "gcp": { + "$id":"#/properties/images/properties/gcp", + "type":"object", + "title":"The Gcp", + "required": [ + "path", + "sha256", + "size" + ], + "properties": { + "path": { + "$id":"#/properties/images/properties/gcp/properties/path", + "type":"string", + "title":"Relative Path of Artifact", + "default":"", + "examples": [ + "fcos-31-202001010000-0-gcp.x86_64.tar.gz" + ], + "pattern":"(.*).tar(|.gz|.xz)$" + }, + "sha256": { + "$id":"#/properties/images/properties/gcp/properties/sha256", + "type":"string", + "title":"Artifact SHA256 Checksum", + "default":"", + "examples": [ + "842e04f40889ca05da54f1a8acd263f273dd72fce8e004264739f38eaf46f11d" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "size": { + "$id":"#/properties/images/properties/gcp/properties/size", + "type":"integer", + "title":"Size in bytes", + "default": 0, + "examples": [ + 892692916 + ] + } + } + } + } + }, + "name": { + "$id":"#/properties/name", + "type":"string", + "title":"The Name", + "default":"", + "examples": [ + "rhcos" + ], + "pattern":"^(.*)$" + }, + "oscontainer": { + "$id":"#/properties/oscontainer", + "type":"object", + "title":"The Oscontainer", + "required": [ + "digest", + "image" + ], + "properties": { + "digest": { + "$id":"#/properties/oscontainer/properties/digest", + "type":"string", + "title":"The Digest", + "default":"", + "examples": [ + "sha256:f8d18011913e87ae59d3781f3d07819267d43401ab444fb3b5794a5eb392b915" + ], + "pattern":"sha256:[A-Fa-f0-9]{64}" + }, + "image": { + "$id":"#/properties/oscontainer/properties/image", + "type":"string", + "title":"The Image", + "default":"", + "examples": [ + "registry.example.org/devel/machine-os-content" + ], + "pattern":"^(.*)$" + } + } + }, + "ostree-commit": { + "$id":"#/properties/ostree-commit", + "type":"string", + "title":"ostree-commit", + "default":"", + "examples": [ + "9665ab0cfd4a995cf70f1a3bb678d3515a03f7d3b5bb87d723ba06c26f0daa6e" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "ostree-content-bytes-written": { + "$id":"#/properties/ostree-content-bytes-written", + "type":"integer", + "title":"ostree-content-bytes-written", + "default": 0, + "examples": [ + 156269945 + ] + }, + "ostree-content-checksum": { + "$id":"#/properties/ostree-content-checksum", + "type":"string", + "title":"ostree-content-checksum", + "default":"", + "examples": [ + "e02647edba305ad68e2c7c5bb3a2c7765eb4ea6aadd1ebf8e538e459ebf99ed7" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "ostree-n-cache-hits": { + "$id":"#/properties/ostree-n-cache-hits", + "type":"integer", + "title":"ostree-n-cache-hits", + "default": 0, + "examples": [ + 19185 + ] + }, + "ostree-n-content-total": { + "$id":"#/properties/ostree-n-content-total", + "type":"integer", + "title":"ostree-n-content-total", + "default": 0, + "examples": [ + 3688 + ] + }, + "ostree-n-content-written": { + "$id":"#/properties/ostree-n-content-written", + "type":"integer", + "title":"ostree-n-content-written", + "default": 0, + "examples": [ + 1210 + ] + }, + "ostree-n-metadata-total": { + "$id":"#/properties/ostree-n-metadata-total", + "type":"integer", + "title":"ostree-n-metadata-total", + "default": 0, + "examples": [ + 9225 + ] + }, + "ostree-n-metadata-written": { + "$id":"#/properties/ostree-n-metadata-written", + "type":"integer", + "title":"ostree-n-metadata-written", + "default": 0, + "examples": [ + 3015 + ] + }, + "ostree-timestamp": { + "$id":"#/properties/ostree-timestamp", + "type":"string", + "title":"ostree timestamp", + "default":"", + "examples": [ + "2020-01-15T19:31:31Z" + ], + "pattern":"^(.*)$" + }, + "ostree-version": { + "$id":"#/properties/ostree-version", + "type":"string", + "title":"ostree version", + "default":"", + "examples": [ + "31-202001010000-0" + ], + "pattern":"^(.*)$" + }, + "pkgdiff": { + "$id":"#/properties/pkgdiff", + "type":"array", + "title":"pkgdiff between builds", + "items": { + "$id":"#/properties/pkgdiff/items", + "type":"array", + "title":"Package Set differences", + "items": { + "$id":"#/properties/pkgdiff/items/items", + "title":"The Items", + "default":"", + "examples": [ + "foo-bar", + 2 + ], + "pattern":"^(.*)$" + } + } + }, + "rpm-ostree-inputhash": { + "$id":"#/properties/rpm-ostree-inputhash", + "type":"string", + "title":"input has of the rpm-ostree", + "default":"", + "examples": [ + "13de3656ed8f55f8b8bafeab7a2320496c247cf533063e3d3daa63a95592f1ac" + ], + "pattern":"[A-Fa-f0-9]{64}" + }, + "summary": { + "$id":"#/properties/summary", + "type":"string", + "title":"Build Summary", + "default":"", + "examples": [ + "FCOS 31" + ], + "pattern":"^(.*)$" + }, + "aliyun": { + "$id":"#/properties/aliyun", + "type":"array", + "title":"Alibaba/Aliyun Uploads", + "items": { + "$id":"#/properties/aliyun/items", + "type":"object", + "title":"Image ID", + "required": [ + "name", + "id" + ], + "properties": { + "name": { + "$id":"#/properties/aliyun/items/properties/name", + "type":"string", + "title":"Region Name", + "default":"", + "examples": [ + "us-west-1" + ], + "pattern":"^(.*)$" + }, + "id": { + "$id":"#/properties/aliyun/items/properties/id", + "type":"string", + "title":"The Id", + "default":"", + "examples": [ + "m-rj9d477fe8uxzai8d8zf" + ], + "pattern":"^(.*)$" + } + } + } + }, + "amis": { + "$id":"#/properties/amis", + "type":"array", + "title":"Amazon AWS AMIs", + "items": { + "$id":"#/properties/amis/items", + "type":"object", + "title":"The Items", + "required": [ + "name", + "hvm", + "snapshot" + ], + "properties": { + "name": { + "$id":"#/properties/amis/items/properties/name", + "type":"string", + "title":"Region Name", + "default":"", + "examples": [ + "us-east-1" + ], + "pattern":"^(.*)$" + }, + "hvm": { + "$id":"#/properties/amis/items/properties/hvm", + "type":"string", + "title":"HVM AMI ID", + "default":"", + "examples": [ + "ami-0de107f30d290b9a1" + ], + "pattern":"^ami-(\\w{17})$" + }, + "snapshot": { + "$id":"#/properties/amis/items/properties/snapshot", + "type":"string", + "title":"The Snapshot", + "default":"", + "examples": [ + "snap-006453e8b55eddb0e" + ], + "pattern":"^snap-(\\w{17})$" + } + } + } + }, + "azure": { + "$id":"#/properties/azure", + "type":"object", + "title":"Azure Upload", + "required": [ + "image", + "url" + ], + "properties": { + "image": { + "$id":"#/properties/azure/properties/image", + "type":"string", + "title":"The Image", + "default":"", + "examples": [ + "fcos-31-202001010000-0-azure.x86_64.vhd" + ], + "pattern":"^(.*)$" + }, + "url": { + "$id":"#/properties/azure/properties/url", + "type":"string", + "title":"URL for Artifact", + "default":"", + "examples": [ + "https://example.com/imagebucket/fcos-31-202001010000-0-azure.x86_64.vhd" + ], + "pattern":"^(.*)$" + } + } + }, + "gcp": { + "$id":"#/properties/gcp", + "type":"object", + "title":"GCP Object Storage Location", + "required": [ + "image", + "url" + ], + "properties": { + "image": { + "$id":"#/properties/gcp/properties/image", + "type":"string", + "title":"The Image", + "default":"", + "examples": [ + "fcos-31-202001010000-0" + ], + "pattern":"^(.*)$" + }, + "url": { + "$id":"#/properties/gcp/properties/url", + "type":"string", + "title":"URL for Artifact", + "default":"", + "examples": [ + "https://example.com/devel/devel/rhcos/fcos-31-202001010000-0.tar.gz" + ], + "pattern":"(.*).tar.gz$" + } + } + } + } +} diff --git a/tests/test_cosalib_meta.py b/tests/test_cosalib_meta.py index c56e8f01ad..000d934eaa 100644 --- a/tests/test_cosalib_meta.py +++ b/tests/test_cosalib_meta.py @@ -9,9 +9,15 @@ from cosalib import meta from cosalib.cmdlib import get_basearch +from jsonschema import ValidationError +TEST_META = os.environ.get( + "COSA_TEST_META_JSON", "/usr/lib/coreos-assembler/fixtures/rhcos.json") +TEST_SCHEMA = os.environ.get( + "COSA_META_SCHEMA", "/usr/lib/coreos-assembler/cosalib/schema/v1.json") -def _create_test_files(tmpdir): + +def _create_test_files(tmpdir, meta_data=None): """ Creates test data for each run. """ @@ -28,12 +34,14 @@ def _create_test_files(tmpdir): "timestamp": "2019-01-1T15:19:45Z" } - data = { - 'test': 'data', - 'a': { - 'b': 'c', + if meta_data is None: + meta_data = { + 'test': 'data', + 'a': { + 'b': 'c', + } } - } + buildsdir = os.path.join(tmpdir, 'builds') os.makedirs(buildsdir, exist_ok=True) with open(os.path.join(buildsdir, 'builds.json'), 'w') as f: @@ -42,17 +50,17 @@ def _create_test_files(tmpdir): tmpdir, 'builds', '1.2.3', get_basearch()) os.makedirs(metadir, exist_ok=True) with open(os.path.join(metadir, 'meta.json'), 'w') as f: - f.write(json.dumps(data)) + f.write(json.dumps(meta_data)) return tmpdir def test_init(tmpdir): - m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3') + m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3', schema=None) assert m['test'] is not None def test_get(tmpdir): - m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3') + m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3', schema=None) assert m.get('test') == 'data' assert m.get('nope', 'default') == 'default' assert m.get(['a', 'b']) == 'c' @@ -63,7 +71,7 @@ def test_set(tmpdir): """ Verify setting works as expected. """ - m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3') + m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3', schema=None) m.set('test', 'changed') m.write() assert m.get('test') == 'changed' @@ -80,5 +88,40 @@ def test_str(tmpdir): Verifies the string representation is exactly the same as the instance dict. """ - m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3') + m = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3', schema=None) assert dict(m) == json.loads(str(m)) + + +def test_valid_schema(tmpdir): + """ + Verifies that schema testing is enforced and checked against a known-good + meta.json. + """ + m = None + with open(TEST_META, 'r') as valid_data: + td = json.load(valid_data) + m = meta.GenericBuildMeta(_create_test_files(tmpdir, meta_data=td), + '1.2.3') + + schema = None + with open(TEST_SCHEMA, 'r') as schema_data: + schema = json.load(schema_data) + + for k, v in m.items(): + if k not in schema['required']: + continue + + _ = m.pop(k) + with pytest.raises(ValidationError): + m.write() + m.set(k, v) + m.write() + + +def test_invalid_schema(tmpdir): + """ + Verifies that schema testing is enforced and checked against a known-good + meta.json. + """ + with pytest.raises(ValidationError): + _ = meta.GenericBuildMeta(_create_test_files(tmpdir), '1.2.3')