From 2c996fc38d9ae4865f383d8ea88ac92be8f859d0 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Wed, 5 Apr 2023 15:45:53 +0100 Subject: [PATCH] tests: add sanity tests for examples Signed-off-by: Justin Chadwell --- .github/workflows/e2e.yml | 13 +- .gitignore | 1 + Makefile | 5 +- README.md | 4 + cmd/check/main.go | 147 +++++++++++++++++++++ examples/alpine/checks/sbom-base.spdx.json | 26 ++++ examples/alpine/checks/sbom.spdx.json | 4 + examples/centos/checks/sbom-base.spdx.json | 26 ++++ examples/centos/checks/sbom.spdx.json | 4 + examples/scratch/checks/sbom.spdx.json | 4 + examples/ubuntu/checks/sbom-base.spdx.json | 26 ++++ examples/ubuntu/checks/sbom.spdx.json | 4 + hack/check-example.sh | 33 +++++ 13 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 .gitignore create mode 100644 cmd/check/main.go create mode 100644 examples/alpine/checks/sbom-base.spdx.json create mode 100644 examples/alpine/checks/sbom.spdx.json create mode 100644 examples/centos/checks/sbom-base.spdx.json create mode 100644 examples/centos/checks/sbom.spdx.json create mode 100644 examples/scratch/checks/sbom.spdx.json create mode 100644 examples/ubuntu/checks/sbom-base.spdx.json create mode 100644 examples/ubuntu/checks/sbom.spdx.json create mode 100755 hack/check-example.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d7eedae7..94bfd5e5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -71,24 +71,21 @@ jobs: push: true - name: Test - uses: docker/build-push-action@v4 - with: - context: ./examples/${{ matrix.example }} - sbom: generator=${{ env.IMAGE_LOCAL }} - outputs: /tmp/buildx-build + run: | + ./hack/check-example.sh ${{ env.IMAGE_LOCAL }} ${{ matrix.example }} - name: Check output folder run: | - tree /tmp/buildx-build + tree ./examples/${{ matrix.example }}/build - name: Print SBOM if: matrix.example != 'scratch' run: | - jq . /tmp/buildx-build/sbom-base.spdx.json + jq . ./examples/${{ matrix.example }}/build/sbom-base.spdx.json - name: Upload output folder uses: actions/upload-artifact@v3 with: name: e2e-${{ matrix.example }} - path: /tmp/buildx-build/* + path: ./examples/${{ matrix.example }}/build/* if-no-files-found: error diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6c031103 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/examples/*/build/ \ No newline at end of file diff --git a/Makefile b/Makefile index c77594c0..9b7607eb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ -.PHONY: all dev +.PHONY: all dev examples all: dev: IMAGE_LOCAL=$(IMAGE) docker buildx bake --push + +examples: + ./hack/check-example.sh $(IMAGE) ./examples/* diff --git a/README.md b/README.md index 8f33fd4d..425ddb55 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ To build the development image, and push it to `localhost:5000/buildkit-syft-sca $ make dev IMAGE=localhost:5000/buildkit-syft-scanner:dev +To test the development image: + + $ make examples IMAGE=localhost:5000/buildkit-syft-scanner:dev + To scan an image during build with [buildctl](https://github.com/moby/buildkit) using the development image: diff --git a/cmd/check/main.go b/cmd/check/main.go new file mode 100644 index 00000000..0438a9e9 --- /dev/null +++ b/cmd/check/main.go @@ -0,0 +1,147 @@ +// Copyright 2023 buildkit-syft-scanner authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// check is a simple script that ensures a target JSON file matches a specified +// schema. +// +// The schema is another JSON file that should match the desired format +// structure of the target - check will ensure that the schema is a subset of +// the target. +// +// check also provides simple variables and comparisons to easily evaluate +// relationships in the target SPDX files. A property set to "=" will assign +// the variable value to the corresponding property from the target, while a +// property set to "==" will check the previously assigned variable to the +// corresponding property from the target. +// +// See the ./examples/*/checks/ directories for usage examples. + +package main + +import ( + "encoding/json" + "fmt" + "os" + "reflect" + "strings" +) + +func main() { + schemaFilename := os.Args[1] + targetFilename := os.Args[2] + + var schema, target map[string]interface{} + if dt, err := os.ReadFile(schemaFilename); err != nil { + panic(err) + } else { + if err := json.Unmarshal(dt, &schema); err != nil { + panic(err) + } + } + if dt, err := os.ReadFile(targetFilename); err != nil { + panic(err) + } else { + if err := json.Unmarshal(dt, &target); err != nil { + panic(err) + } + } + + vars := make(map[string]interface{}) + err := check(schema, target, vars, true, "") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = check(schema, target, vars, false, "") + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func check(schema interface{}, target interface{}, vars map[string]interface{}, assign bool, previous ...string) error { + schemaType := reflect.TypeOf(schema) + targetType := reflect.TypeOf(target) + + if schemaType.Kind() == reflect.String { + if strings.HasPrefix(schema.(string), "==") { + if !assign { + key := schema.(string)[2:] + if _, ok := vars[key]; !ok { + return fmt.Errorf("variable %s not found", key) + } + if target != vars[key] { + return fmt.Errorf("variable mismatch on %s, expected %v, got %v", strings.Join(previous, "."), vars[key], target) + } + } + return nil + } + if strings.HasPrefix(schema.(string), "=") { + key := schema.(string)[1:] + if assign { + vars[key] = target + } + return nil + } + } + + if schemaType.Kind() != targetType.Kind() { + return fmt.Errorf("type mismatch on %s, expected %s, got %s", strings.Join(previous, "."), schemaType.Kind(), targetType.Kind()) + } + switch schemaType.Kind() { + case reflect.Pointer: + return check(reflect.ValueOf(schema).Elem().Interface(), reflect.ValueOf(target).Elem().Interface(), vars, assign, previous...) + case reflect.Map: + return checkMap(schema.(map[string]interface{}), target.(map[string]interface{}), vars, assign, previous...) + case reflect.Array, reflect.Slice: + return checkSlice(schema.([]interface{}), target.([]interface{}), vars, assign, previous...) + default: + if !reflect.DeepEqual(schema, target) { + return fmt.Errorf("value mismatch on %s, expected %v, got %v", strings.Join(previous, "."), schema, target) + } + return nil + } +} + +func checkMap(schema map[string]interface{}, target map[string]interface{}, vars map[string]interface{}, assign bool, previous ...string) error { + for k, v := range schema { + v2, ok := target[k] + if !ok { + return fmt.Errorf("map mismatch on %s, expected %v", strings.Join(previous, ".")+"."+k, schema) + } + if err := check(v, v2, vars, assign, append(previous, k)...); err != nil { + return err + } + } + return nil +} + +func checkSlice(schema []interface{}, target []interface{}, vars map[string]interface{}, assign bool, previous ...string) error { + if len(schema) > len(target) { + return fmt.Errorf("length mismatch on %s, expected at least %d, got %d", strings.Join(previous, "."), len(schema), len(target)) + } + for i, v := range schema { + found := false + for _, v2 := range target { + if err := check(v, v2, vars, assign, append(previous, fmt.Sprintf("[%d]", i))...); err == nil { + found = true + break + } + } + if !found { + return fmt.Errorf("slice mismatch on %s, expected %v", strings.Join(previous, ".")+fmt.Sprintf("[%d]", i), schema) + } + } + return nil +} diff --git a/examples/alpine/checks/sbom-base.spdx.json b/examples/alpine/checks/sbom-base.spdx.json new file mode 100644 index 00000000..41dd232a --- /dev/null +++ b/examples/alpine/checks/sbom-base.spdx.json @@ -0,0 +1,26 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document", + "predicate": { + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "=package", + "name": "git" + } + ], + "files": [ + { + "SPDXID": "=filename", + "fileName": "/usr/bin/git" + } + ], + "relationships": [ + { + "spdxElementId": "==package", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "==filename" + } + ] + } +} \ No newline at end of file diff --git a/examples/alpine/checks/sbom.spdx.json b/examples/alpine/checks/sbom.spdx.json new file mode 100644 index 00000000..80f5b0c0 --- /dev/null +++ b/examples/alpine/checks/sbom.spdx.json @@ -0,0 +1,4 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document" +} \ No newline at end of file diff --git a/examples/centos/checks/sbom-base.spdx.json b/examples/centos/checks/sbom-base.spdx.json new file mode 100644 index 00000000..7a646dd5 --- /dev/null +++ b/examples/centos/checks/sbom-base.spdx.json @@ -0,0 +1,26 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document", + "predicate": { + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "=package", + "name": "findutils" + } + ], + "files": [ + { + "SPDXID": "=filename", + "fileName": "/usr/bin/xargs" + } + ], + "relationships": [ + { + "spdxElementId": "==package", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "==filename" + } + ] + } +} \ No newline at end of file diff --git a/examples/centos/checks/sbom.spdx.json b/examples/centos/checks/sbom.spdx.json new file mode 100644 index 00000000..80f5b0c0 --- /dev/null +++ b/examples/centos/checks/sbom.spdx.json @@ -0,0 +1,4 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document" +} \ No newline at end of file diff --git a/examples/scratch/checks/sbom.spdx.json b/examples/scratch/checks/sbom.spdx.json new file mode 100644 index 00000000..80f5b0c0 --- /dev/null +++ b/examples/scratch/checks/sbom.spdx.json @@ -0,0 +1,4 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document" +} \ No newline at end of file diff --git a/examples/ubuntu/checks/sbom-base.spdx.json b/examples/ubuntu/checks/sbom-base.spdx.json new file mode 100644 index 00000000..41dd232a --- /dev/null +++ b/examples/ubuntu/checks/sbom-base.spdx.json @@ -0,0 +1,26 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document", + "predicate": { + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "SPDXID": "=package", + "name": "git" + } + ], + "files": [ + { + "SPDXID": "=filename", + "fileName": "/usr/bin/git" + } + ], + "relationships": [ + { + "spdxElementId": "==package", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "==filename" + } + ] + } +} \ No newline at end of file diff --git a/examples/ubuntu/checks/sbom.spdx.json b/examples/ubuntu/checks/sbom.spdx.json new file mode 100644 index 00000000..80f5b0c0 --- /dev/null +++ b/examples/ubuntu/checks/sbom.spdx.json @@ -0,0 +1,4 @@ +{ + "_type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://spdx.dev/Document" +} \ No newline at end of file diff --git a/hack/check-example.sh b/hack/check-example.sh new file mode 100755 index 00000000..0a8b7a53 --- /dev/null +++ b/hack/check-example.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Copyright 2023 buildkit-syft-scanner authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -eu + +GENERATOR=$1 + +for example in "${@:2}"; do + example=$(basename "$example") + echo "[-] Building example ${example}..." + docker buildx build "./examples/${example}" --sbom=generator="${GENERATOR}" --output="./examples/${example}/build" + + echo "[-] Checking example ${example}..." + for file in "./examples/${example}"/checks/*.json; do + echo " [-] Checking schema ${file}..." + go run ./cmd/check "$PWD/$file" $PWD/examples/${example}/build/${file#"./examples/${example}/checks/"} + done + + echo "" +done