From 8b488f6f47ca5a4e588d5738131d5edec0f29924 Mon Sep 17 00:00:00 2001 From: Sebastian Schildt Date: Sun, 17 Mar 2024 13:41:38 +0900 Subject: [PATCH] CycloneDX SBOM for databroker Creates a CycloneDX Software Bill of Materials (SBOM) for the databroker. Refactor createbom so it can collect licenses from any CycloneDX input file, so it may be reused for other parts of the project as well. Signed-off-by: Sebastian Schildt --- .../workflows/kuksa_databroker-cli_build.yml | 26 ++- .github/workflows/kuksa_databroker_build.yml | 25 ++- kuksa_databroker/Dockerfile | 5 +- kuksa_databroker/Dockerfile-cli | 5 +- kuksa_databroker/build-all-targets-cli.sh | 104 ++++++----- kuksa_databroker/build-all-targets.sh | 103 ++++++----- .../createbom/bomutil/maplicensefile.py | 6 +- .../createbom/collectlicensefromcyclonedx.py | 170 ++++++++++++++++++ .../licensestore/BSD-3-Clause.txt.gz | Bin 37 -> 784 bytes .../createbom/licensestore/OpenSSL.txt.gz | Bin 0 -> 1958 bytes kuksa_databroker/licensecuration.yaml | 21 +++ 11 files changed, 336 insertions(+), 129 deletions(-) create mode 100644 kuksa_databroker/createbom/collectlicensefromcyclonedx.py create mode 100644 kuksa_databroker/createbom/licensestore/OpenSSL.txt.gz create mode 100644 kuksa_databroker/licensecuration.yaml diff --git a/.github/workflows/kuksa_databroker-cli_build.yml b/.github/workflows/kuksa_databroker-cli_build.yml index ab369aa41..6e953cdb3 100644 --- a/.github/workflows/kuksa_databroker-cli_build.yml +++ b/.github/workflows/kuksa_databroker-cli_build.yml @@ -62,7 +62,7 @@ jobs: - name: Building working-directory: ${{github.workspace}}/kuksa_databroker/ run: | - cargo install cargo-license cross + cargo install cargo-license cross cargo-cyclonedx ./build-all-targets-cli.sh - name: Docker meta @@ -81,7 +81,7 @@ jobs: type=semver,pattern={{major}} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx id: buildx @@ -148,23 +148,31 @@ jobs: - name: "Archiving RISCV64 artifacts" uses: actions/upload-artifact@v3 with: - name: databroker--cli-riscv64 + name: databroker-cli-riscv64 path: ${{github.workspace}}/dist/riscv64 bom: - name: Bill of Material Check + name: License Compliance Check runs-on: ubuntu-latest - needs: build-container steps: - uses: actions/checkout@v4 - - name: "Createbom: License check and Dash output generation" - working-directory: ${{github.workspace}}/kuksa_databroker/createbom + # Follows the pattern from + # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo + - name: "Using cargo to create Dash input" + working-directory: ${{github.workspace}}/kuksa_databroker/ + # target all is not really needed, and will also return i.e. wasm deps, however + # better safe than sorry, the alternative would be running this for each currently + # built target and combining the lists, but that would need adapting, when + # adding targets, or also when i.e. switching between MUSL/glibc. So this is safer run: | - cargo install cargo-license - python3 createbom.py --dash ${{github.workspace}}/dash-databroker-deps ../databroker + cargo tree -e normal --prefix none --no-dedupe --target all --all-features > ${{github.workspace}}/cargodeps + cat ${{github.workspace}}/cargodeps | sort -u \ + | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker \ + | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \ + > ${{github.workspace}}/dash-databroker-deps - name: Dash license check uses: eclipse-kuksa/kuksa-actions/check-dash@2 diff --git a/.github/workflows/kuksa_databroker_build.yml b/.github/workflows/kuksa_databroker_build.yml index 9ca24772a..06f53f6d3 100644 --- a/.github/workflows/kuksa_databroker_build.yml +++ b/.github/workflows/kuksa_databroker_build.yml @@ -88,7 +88,6 @@ jobs: build-container: runs-on: ubuntu-latest - needs: check_ghcr_push steps: @@ -99,7 +98,7 @@ jobs: - name: Building working-directory: ${{github.workspace}}/kuksa_databroker/ run: | - cargo install cargo-license cross + cargo install cargo-license cross cargo-cyclonedx ./build-all-targets.sh - name: Docker meta @@ -197,7 +196,7 @@ jobs: steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - uses: actions/checkout@v4 @@ -223,18 +222,26 @@ jobs: ${{github.workspace}}/kuksa_databroker/integration_test/run.sh bom: - name: Bill of Material Check + name: License Compliance Check runs-on: ubuntu-latest - needs: build-container steps: - uses: actions/checkout@v4 - - name: "Createbom: License check and Dash output generation" - working-directory: ${{github.workspace}}/kuksa_databroker/createbom + # Follows the pattern from + # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo + - name: "Using cargo to create Dash input" + working-directory: ${{github.workspace}}/kuksa_databroker/ + # target all is not really needed, and will also return i.e. wasm deps, however + # better safe than sorry, the alternative would be running this for each currently + # built target and combining the lists, but that would need adapting, when + # adding targets, or also when i.e. switching between MUSL/glibc. So this is safer run: | - cargo install cargo-license - python3 createbom.py --dash ${{github.workspace}}/dash-databroker-deps ../databroker + cargo tree -e normal --prefix none --no-dedupe --target all --all-features > ${{github.workspace}}/cargodeps + cat ${{github.workspace}}/cargodeps | sort -u \ + | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker \ + | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \ + > ${{github.workspace}}/dash-databroker-deps - name: Dash license check uses: eclipse-kuksa/kuksa-actions/check-dash@2 diff --git a/kuksa_databroker/Dockerfile b/kuksa_databroker/Dockerfile index badaa8297..47b2c3651 100644 --- a/kuksa_databroker/Dockerfile +++ b/kuksa_databroker/Dockerfile @@ -1,5 +1,5 @@ # /******************************************************************************** -# * Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation +# * Copyright (c) 2024 Contributors to the Eclipse Foundation # * # * See the NOTICE file(s) distributed with this work for additional # * information regarding copyright ownership. @@ -44,7 +44,8 @@ COPY ./target/riscv64gc-unknown-linux-gnu/release/databroker /app/databroker FROM target-$TARGETARCH as target ARG TARGETARCH -COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty +COPY ./dist/$TARGETARCH/sbom.json /app/ +COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty COPY ./data/vss-core/vss_release_3.1.1.json vss_release_3.1.1.json COPY ./data/vss-core/vss_release_4.0.json vss_release_4.0.json diff --git a/kuksa_databroker/Dockerfile-cli b/kuksa_databroker/Dockerfile-cli index 8b5b39193..3c2a3748c 100644 --- a/kuksa_databroker/Dockerfile-cli +++ b/kuksa_databroker/Dockerfile-cli @@ -1,5 +1,5 @@ # /******************************************************************************** -# * Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation +# * Copyright (c) 2024 Contributors to the Eclipse Foundation # * # * See the NOTICE file(s) distributed with this work for additional # * information regarding copyright ownership. @@ -50,7 +50,8 @@ RUN apk update && apk add ncurses-terminfo-base FROM target-$TARGETARCH as target ARG TARGETARCH -COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty +COPY ./dist/$TARGETARCH/sbom.json /app/ +COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty # Copy terminfo database COPY --from=terminfo-donor /etc/terminfo /etc/terminfo diff --git a/kuksa_databroker/build-all-targets-cli.sh b/kuksa_databroker/build-all-targets-cli.sh index 341ad631a..5aa4fe96d 100755 --- a/kuksa_databroker/build-all-targets-cli.sh +++ b/kuksa_databroker/build-all-targets-cli.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2024 Contributors to the Eclipse Foundation # # Building all currently supported targets for databroker-cli. # Uses cross for cross-compiling. Needs to be executed @@ -10,17 +10,55 @@ # # SPDX-License-Identifier: Apache-2.0 - # exit on error, to not waste any time set -e + CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse -# Create thirdparty bom -cd createbom/ -rm -rf ../databroker/thirdparty || true -python3 createbom.py ../databroker-cli -cd .. +# Builds for a given target and collects data to be distirbuted in docker. Needs +# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker +# architecture (i.e. amd64) as input +function build_target() { + target_rust=$1 + target_docker=$2 + + echo "Building databroker-cli for target $target_rust" + cross build --target $target_rust --bin databroker-cli --release + + echo "Create $target_rust SBOM" + cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ../Cargo.toml + + echo "Prepare $target_docker dist folder" + mkdir ../dist/$target_docker + cp ../target/$target_rust/release/databroker-cli ../dist/$target_docker + cp ./databroker-cli/databroker-cli_bin.cdx.json ../dist/$target_docker/sbom.json + + rm -rf ../dist/$target_docker/thirdparty-licenses || true + + cd createbom/ + rm -rf ../databroker/thirdparty-licenses || true + python3 collectlicensefromcyclonedx.py ../databroker-cli/databroker-cli_bin.cdx.json ../../dist/$target_docker/thirdparty-licenses --curation ../licensecuration.yaml + cd .. + + # We need to clean this folder in target, otherwise we get weird side + # effects building the aarch image, complaining libc crate can not find + # GLIBC, i.e + # Compiling libc v0.2.149 + #error: failed to run custom build command for `libc v0.2.149` + # + #Caused by: + # process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) + # --- stderr + # /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) + # + # It seems cross/cargo is reusing something from previous builds it shouldn't. + # the finished artifact resides in ../target/x86_64-unknown-linux-musl/release + # so deleting the temporary files in target/releae is no problem + echo "Cleaning up...." + rm -rf ../target/release + +} # Starting a fresh build echo "Cargo clean, to start fresh..." @@ -28,53 +66,13 @@ cargo clean rm -rf ../dist || true mkdir ../dist -# Buidling AMD46 -echo "Building AMD64" -cross build --target x86_64-unknown-linux-musl --bin databroker-cli --release -# We need to clean this folder in target, otherwise we get weird side -# effects building the aarch image, complaining libc crate can not find -# GLIBC, i.e -# Compiling libc v0.2.149 -#error: failed to run custom build command for `libc v0.2.149` -# -#Caused by: -# process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) -# --- stderr -# /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) -# -# It seems cross/cargo is reusing something from previous builds it shouldn't. -# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release -# so deleting the temporary files in trget/releae is no problem -echo "Cleaning up...." -rm -rf ../target/release - - -# Buidling ARM64 -echo "Building ARM64" -cross build --target aarch64-unknown-linux-musl --bin databroker-cli --release -echo "Cleaning up...." -rm -rf ../target/release +# Building AMD46 +build_target x86_64-unknown-linux-musl amd64 +# Building ARM64 +build_target aarch64-unknown-linux-musl arm64 # Build RISCV64, this is a glibc based build, as musl is not # yet supported -echo "Building RISCV64" -cross build --target riscv64gc-unknown-linux-gnu --bin databroker-cli --release -echo "Cleaning up...." -rm -rf ../target/release - -# Prepare dist folders -echo "Prepare amd64 dist folder" -mkdir ../dist/amd64 -cp ../target/x86_64-unknown-linux-musl/release/databroker-cli ../dist/amd64 -cp -r ./databroker-cli/thirdparty ../dist/amd64 - -echo "Prepare arm64 dist folder" -mkdir ../dist/arm64 -cp ../target/aarch64-unknown-linux-musl/release/databroker-cli ../dist/arm64 -cp -r ./databroker-cli/thirdparty ../dist/arm64 - -echo "Prepare riscv64 dist folder" -mkdir ../dist/riscv64 -cp ../target/riscv64gc-unknown-linux-gnu/release/databroker-cli ../dist/riscv64 -cp -r ./databroker-cli/thirdparty ../dist/riscv64 +# Building RISCV64 +build_target riscv64gc-unknown-linux-gnu riscv64 diff --git a/kuksa_databroker/build-all-targets.sh b/kuksa_databroker/build-all-targets.sh index b5d72b746..a2ed0d70d 100755 --- a/kuksa_databroker/build-all-targets.sh +++ b/kuksa_databroker/build-all-targets.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2024 Contributors to the Eclipse Foundation # # Building all currently supported targets. # Uses cross for cross-compiling. Needs to be executed @@ -13,13 +13,52 @@ # exit on error, to not waste any time set -e + CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse -# Create thirdparty bom -cd createbom/ -rm -rf ../databroker/thirdparty || true -python3 createbom.py ../databroker -cd .. +# Builds for a given target and collects data to be distirbuted in docker. Needs +# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker +# architecture (i.e. amd64) as input +function build_target() { + target_rust=$1 + target_docker=$2 + + echo "Building databroker for target $target_rust" + cross build --target $target_rust --bin databroker --release + + echo "Create $target_rust SBOM" + cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ../Cargo.toml + + echo "Prepare $target_docker dist folder" + mkdir ../dist/$target_docker + cp ../target/$target_rust/release/databroker ../dist/$target_docker + cp ./databroker/databroker_bin.cdx.json ../dist/$target_docker/sbom.json + + rm -rf ../dist/$target_docker/thirdparty-licenses || true + + cd createbom/ + rm -rf ../databroker/thirdparty-licenses || true + python3 collectlicensefromcyclonedx.py ../databroker/databroker_bin.cdx.json ../../dist/$target_docker/thirdparty-licenses --curation ../licensecuration.yaml + cd .. + + # We need to clean this folder in target, otherwise we get weird side + # effects building the aarch image, complaining libc crate can not find + # GLIBC, i.e + # Compiling libc v0.2.149 + #error: failed to run custom build command for `libc v0.2.149` + # + #Caused by: + # process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) + # --- stderr + # /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) + # + # It seems cross/cargo is reusing something from previous builds it shouldn't. + # the finished artifact resides in ../target/x86_64-unknown-linux-musl/release + # so deleting the temporary files in target/releae is no problem + echo "Cleaning up...." + rm -rf ../target/release + +} # Starting a fresh build echo "Cargo clean, to start fresh..." @@ -27,53 +66,13 @@ cargo clean rm -rf ../dist || true mkdir ../dist -# Buidling AMD46 -echo "Building AMD64" -cross build --target x86_64-unknown-linux-musl --bin databroker --release -# We need to clean this folder in target, otherwise we get weird side -# effects building the aarch image, complaining libc crate can not find -# GLIBC, i.e -# Compiling libc v0.2.149 -#error: failed to run custom build command for `libc v0.2.149` -# -#Caused by: -# process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1) -# --- stderr -# /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build) -# -# It seems cross/cargo is reusing something from previous builds it shouldn't. -# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release -# so deleting the temporary files in trget/releae is no problem -echo "Cleaning up...." -rm -rf ../target/release - - -# Buidling ARM64 -echo "Building ARM64" -cross build --target aarch64-unknown-linux-musl --bin databroker --release -echo "Cleaning up...." -rm -rf ../target/release +# Building AMD46 +build_target x86_64-unknown-linux-musl amd64 +# Building ARM64 +build_target aarch64-unknown-linux-musl arm64 # Build RISCV64, this is a glibc based build, as musl is not # yet supported -echo "Building RISCV64" -cross build --target riscv64gc-unknown-linux-gnu --bin databroker --release -echo "Cleaning up...." -rm -rf ../target/release - -# Prepare dist folders -echo "Prepare amd64 dist folder" -mkdir ../dist/amd64 -cp ../target/x86_64-unknown-linux-musl/release/databroker ../dist/amd64 -cp -r ./databroker/thirdparty ../dist/amd64 - -echo "Prepare arm64 dist folder" -mkdir ../dist/arm64 -cp ../target/aarch64-unknown-linux-musl/release/databroker ../dist/arm64 -cp -r ./databroker/thirdparty ../dist/arm64 - -echo "Prepare riscv64 dist folder" -mkdir ../dist/riscv64 -cp ../target/riscv64gc-unknown-linux-gnu/release/databroker ../dist/riscv64 -cp -r ./databroker/thirdparty ../dist/riscv64 +# Building RISCV64 +build_target riscv64gc-unknown-linux-gnu riscv64 diff --git a/kuksa_databroker/createbom/bomutil/maplicensefile.py b/kuksa_databroker/createbom/bomutil/maplicensefile.py index 044f587d1..ee0d984c1 100644 --- a/kuksa_databroker/createbom/bomutil/maplicensefile.py +++ b/kuksa_databroker/createbom/bomutil/maplicensefile.py @@ -35,6 +35,8 @@ "rustls-webpki": "webpki.LICENSE.txt.gz", # License text taken from https://spdx.org/licenses/0BSD.html "0BSD": "0BSD.txt.gz", - # License test taken from https://spdx.org/licenses/BSD-3-Clause.html - "BSD-3-Clause": "BSD-3-Clause.txt.gz" + # License text taken from https://spdx.org/licenses/BSD-3-Clause.html + "BSD-3-Clause": "BSD-3-Clause.txt.gz", + # License text taken from https://www.openssl.org/source/license-openssl-ssleay.txt + "OPENSSL": "OpenSSL.txt.gz" } diff --git a/kuksa_databroker/createbom/collectlicensefromcyclonedx.py b/kuksa_databroker/createbom/collectlicensefromcyclonedx.py new file mode 100644 index 000000000..15552058b --- /dev/null +++ b/kuksa_databroker/createbom/collectlicensefromcyclonedx.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +######################################################################## +# Copyright (c) 2024 Robert Bosch GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +######################################################################## + +""" +This script will parse a CycloneDX SBOM and collect all mentioned +license texts to be included in a distribution +""" + +import json +import argparse +import sys +import gzip +import os +import re +import logging + +import yaml + +from bomutil.maplicensefile import MAP as supported_licenses + +logger = logging.getLogger(__name__) + +COMP_CURATION = {} +EXPRESSION_CURATION = {} + + +def check_component_curation(component): + """Component curations define a license expression for a given package. + This does NOT overwrite what is in the SBOM, it is merely used to detect + which license to copy int the deliverable""" + logger.debug("Checking component curation for %s", component['name']) + if component['name'] in COMP_CURATION: + logger.info("Found component curation for %s: %s", component['name'], COMP_CURATION[component['name']]) + return COMP_CURATION[component['name']]['expression'] + return None + + +def check_expression_curation(license_exp): + """Can be used to apply general rules for license, i.e. if a license is A OR B, you may opt to use A only""" + logger.debug("Checking expression curation for %s", license_exp) + if license_exp in EXPRESSION_CURATION: + logger.info("Found expression curation for %s: %s", license_exp, EXPRESSION_CURATION[license_exp]) + return EXPRESSION_CURATION[license_exp] + return license_exp + + +def collect_license_id_from_expression(license_exp, component, license_ids): + ids = re.split(r"\s*AND\s*|\s*OR\s*|\(|\)", license_exp) + ids = list(filter(None, ids)) + logger.debug("%s license info is %s", component['name'], ids) + for lic_id in ids: + if lic_id not in license_ids: + license_ids.append(lic_id) + + +def get_expression_from_component(component): + try: + license_exp = component['licenses'][0]['expression'] + return license_exp + except KeyError: + logger.warning("Not able to determine license expression for %s", component['name']) + return None + + +def collect(sbom, targetpath): + logger.info("Collecting licenses from %s, exporting license texts to %s", sbom, targetpath) + + # Opening JSON file + f = open(sbom, encoding="utf-8", mode="r") + sbom = json.load(f) + + license_ids = [] + + for component in sbom["components"]: + license_exp = check_component_curation(component) + if license_exp is None: + license_exp = get_expression_from_component(component) + + license_exp = check_expression_curation(license_exp) + + # If we do not have a license expression, we cannot do anything + if license_exp is None: + logger.error("Could not find license expression for %s", component['name']) + logger.error("Consider adding a curation for this component") + sys.exit(-1) + + logger.info("License expression for %s is %s", component['name'], license_exp) + + collect_license_id_from_expression(license_exp, component, license_ids) + + logger.debug("All license ids %s:", license_ids) + + logger.info("Exporting relevant license texts to %s", targetpath) + # Exporting + os.mkdir(targetpath) + + for license_id in license_ids: + if license_id not in supported_licenses: + logger.error("License %s not supported. You need to extend this tools or check curations", license_id) + logger.error("This tool currently supports the following license identifiers: %s", + supported_licenses.keys()) + sys.exit(-3) + + logger.info("Copying %s", supported_licenses[license_id][:-3]) + with gzip.open("licensestore/" + supported_licenses[license_id], "rb") as inf: + content = inf.read() + with open(os.path.join(targetpath, supported_licenses[license_id][:-3]), "wb") as outf: + outf.write(content) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument("sbom", type=str, help="CycloneDX SBOM in JSON format") + parser.add_argument("dir", default="./", type=str, help="Output directory") + parser.add_argument("--curation", type=str, help="Curation file", default=None) + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the log level') + args = parser.parse_args(args) + + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%Y-%m-%dT%H:%M:%S%z', level=args.log_level) + if not os.path.exists(args.sbom): + logger.error("Input SBOM %s does not exist", args.sbom) + return -1 + + # We want to start with an empty folder, making sure it only contains license + # texts extracted form the input sbom + if os.path.exists(args.dir): + logger.error( + "Target folder %s already exists. Remove it before running this script.", args.dir + ) + return -2 + + if "curation" in args and args.curation is not None: + logger.info("Using curation file %s", args.curation) + with open(args.curation, encoding="utf-8", mode="r") as f: + cur = yaml.safe_load(f) + if "components" in cur: + logger.info("Adding %s component curations", len(cur['components'].keys())) + logger.debug(cur['components'].keys()) + global COMP_CURATION + COMP_CURATION = cur["components"] + if "expressions" in cur: + logger.info("Adding %s expression curations", len(cur['expressions'].keys())) + global EXPRESSION_CURATION + EXPRESSION_CURATION = cur["expressions"] + else: + logger.info("No curation file specified.") + + collect(args.sbom, args.dir) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/kuksa_databroker/createbom/licensestore/BSD-3-Clause.txt.gz b/kuksa_databroker/createbom/licensestore/BSD-3-Clause.txt.gz index eb4e7256d48e7d8e10be0f93ba302bab8ff474ce..6cf3b9e0d07b8dd8d714608f9945633725e9d7a5 100644 GIT binary patch literal 784 zcmV+r1MmDFiwFoT1NUVB142_oEi)}cY+-eCWiE7hbO5bV!EW0y487+oh@M)&$gu6U z6$KPqi3K>8C&^87bZw;-5L<<;G|1nNQW7Lx2kf#-LW|^kkB?04+rgQaR}Z)C9o!#u z;~wDN?g#B2rjv=#of*9|&pU7I0L{?BZq#4~812rrI=(!cp>YS;Sl5pU?2Uf~>*BxN zdFX9tHl}R?Kmv`^@TOgFyw@GPIs0xp4E$^3@kb-bc5C-$c!Aaqoe522Z0fcDF`0aw z!beFX*v(l>YdfsljUJr#4QdR4&9i+Esq>hF^`_Mc=FA9NL<#`cTtoMV5Tt9jjp?TJ8~_28THUcR8^7Bdd|#%pJq?Rc>_?j*KdNuN%> zPodN%HihKS^g2ih@kQ;c-F8?o*h^6~-S{zxKEhcyBJTm7b+`|dZ$S?ozUpuasH3-D z!_gG%LqPM6WgAQ!Z5Zvw@57$YXEVO(Hrx$1nQ%=mT+DF1opFrvRn8U<#!E6mGr;ze$E!ZMgknd@Yq=vs!U{xFHfdZ(@R!YoJe+pi%-Cz?N0P z5Dd`;DHWqK0am6(ow0I}00KbC6%=gA6qYHTMAFY~z~``}BF*uW%vizHI`Yk#Dg)OX z$p}V$ ztY=a&RVxY$&a-H=q~ejKRQ`m5%ji}uX@U%jL}G+Mt1yl6td=aA#!5wns4K;Id54p) z&=%B4ur-VB@-mK!4so##puvM^egdnU;=2fjMyE)yK%!%*x-LULG*Dg52qj$c&lf}C|6IqV2k9mA`OP2whj&4FZ(EXY0;eZc|fCE_8Tw0L@r!Z`(!^{_bBf@E6;KEH`~ali+~T z5^b}PMU|xDGY$uIMXoHS6qnegtoq;gndMTVWV?29Iiv!TP~z@0?=#ONtYjI-BO2wY zER8%MI^F#o9Grba|M~)-1Nw#rs(R4*-J+)B^n~8Lef#t2ci+BwOUXh~=royV^--pE zpU5i;vZoDcX{5fFSsy426v-?%wa#a&I#(r$GNYA|l$T`GN~hBPGRsS$A84-h()4JZ z*9%hG{;O6sqbyaH&+}9;MvsJ+R7t(e>sn@1X?34xc&Hbm#$OURMWNPtc}J-#vz#v( zM&Wx)S-*3tz3K01XGqOkwNjPg@XFM%s1{I{fyGSS^DE~Y2EnIPHDc1kiQG^DJcijy zwifQH1)*sn@}<^pb!FkzoGF05f|TC6fi*#KtpUKXl?F|!76FV!8) z+^OZVqCJ!-Rs4~o9*!4vUA;RyTd&uBpq0iHeWmZtP7a+Pe(2jmD#a4g5BKgL+6W!) zi9T$sMQ~{*SvrEKoIbM5lx8<+wr#0u+1hAelFm_%G{=i3B{tS}>IhqO z%NULwT|D`FkE%in!}*5{+4#oUdzvaOf9V+CrofMVw&+FOmWZgcH5CQoxN*qgCe5`y zwfUVyRY{>;RC4Xy>#W?hueEt&O_O4edz;hxrzcy5C|zx?(RqwQ^5{n`W0R6jbfgkrM7!O9%VKBbz(fKr?ahT93xDFB=OTr!tVwlG_DZHR- zKN?)&t9KrZg5=i9y$F&q1kwfMc{K5&Bp6IbUc@MqX*3CApV**b5D!LPaP1EBkCPyo zCO%z;;V@?G*pEI013!Mx_eWuD9XO5s9^@pR6%Q16FMiL@=hHZ_ehkKmA4Sti5`^Ou zzIzqkz)?`vn{rs$=i%6z4WEV4E#tC3Y)E=^bLHdrh+S;G=D{7-1#sXX>7ImC_%rF) zOJo0X6kPh_fzNM42D%Ai-|FE-LCj}@M)-{f$x~}S#}2wRkDVm$*$7i`LEi8KV1^9i z&Lc14z@?G(;NZ#yfXn_%Q7KzjPgZxTAF5j2X;Cfmw9}Sdt3{>z8ZU0us_7obQat=( zFY(!as+K3n;}<$d2QA5$Hkhl-py3_Y9~b**jk;$C2NZ%)fP55$Vu08!>pXq>W65WS zG!86j{PC2TW52vgG&ZI$tP|x+jGrwz>Gbs-zdh-+239`85N6{p zqX^$>#nY{6@yu?WznNNKHfc zS70)_Wh#}>wQ0Y!>BnOFo1{!pE3*w7uxZS!1wQ?=ULgfRj|}T4m>qE=T9T;&yQ)AX zo9ONB0LQ8<^3pO*2XgXw0ch?1hp&L2vHyMP&F>$y_H{S6&szH)Eqwp$wzrvKPOVpM zd-4^m`o+2HuD>uw+C_wW-&*O6j&^{L+VU&p9I?O#k~g)g)e0qK+O20(ybf+A*&^F7 z+77d;u$G13qjJQq2 zu0@}