CycloneDX SBOM for databroker

Creates a CycloneDX Software Bill of Materials (SBOM) for the databroker. Refactor createbom so it can collect licenses from a CycloneDX input file, so it may be reused for other parts of the project as well. Signed-off-by: Sebastian Schildt <[email protected]>
eclipse · Mar 17, 2024 · 4871f4f · 4871f4f
1 parent 7d9d335
commit 4871f4f
Show file tree

Hide file tree

Showing 10 changed files with 332 additions and 125 deletions.
diff --git a/.github/workflows/kuksa_databroker-cli_build.yml b/.github/workflows/kuksa_databroker-cli_build.yml
@@ -153,18 +153,26 @@ jobs:
 
 
   bom:
-    name: Bill of Material Check
+    name: License Compliance Check
     runs-on: ubuntu-latest
-    needs: build-container
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: "Createbom: License check and Dash output generation"
-        working-directory: ${{github.workspace}}/kuksa_databroker/createbom
+      # Follows the pattern from
+      # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo
+      - name: "Using cargo to create Dash input"
+        working-directory: ${{github.workspace}}/kuksa_databroker/
+        # target all is not really needed, and will also return i.e. wasm deps, however
+        # better safe than sorry, the alternative would be running this for each currently
+        # buit target  and combining the lists, but that would need adapting, when
+        # adding targets, or also when i.e. switcing between MUSL/glibc. So this is safer
         run: |
-         cargo install cargo-license
-         python3 createbom.py --dash ${{github.workspace}}/dash-databroker-deps ../databroker
+         cargo tree -e normal --prefix none --no-dedupe --target all --all-features  > ${{github.workspace}}/cargodeps
+         cat ${{github.workspace}}/cargodeps | sort -u  \
+         | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker  \
+         | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \
+         > ${{github.workspace}}/dash-databroker-deps
 
       - name: Dash license check
         uses: eclipse-kuksa/kuksa-actions/check-dash@2

diff --git a/.github/workflows/kuksa_databroker_build.yml b/.github/workflows/kuksa_databroker_build.yml
@@ -88,7 +88,6 @@ jobs:
 
   build-container:
     runs-on: ubuntu-latest
-
     needs: check_ghcr_push
 
     steps:
@@ -99,7 +98,7 @@ jobs:
     - name: Building
       working-directory: ${{github.workspace}}/kuksa_databroker/
       run: |
-        cargo install cargo-license cross
+        cargo install cargo-license cross cargo-cyclonedx
         ./build-all-targets.sh
 
     - name: Docker meta
@@ -223,18 +222,26 @@ jobs:
           ${{github.workspace}}/kuksa_databroker/integration_test/run.sh
 
   bom:
-    name: Bill of Material Check
+    name: License Compliance Check
     runs-on: ubuntu-latest
-    needs: build-container
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: "Createbom: License check and Dash output generation"
-        working-directory: ${{github.workspace}}/kuksa_databroker/createbom
+      # Follows the pattern from
+      # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo
+      - name: "Using cargo to create Dash input"
+        working-directory: ${{github.workspace}}/kuksa_databroker/
+        # target all is not really needed, and will also return i.e. wasm deps, however
+        # better safe than sorry, the alternative would be running this for each currently
+        # buit target  and combining the lists, but that would need adapting, when
+        # adding targets, or also when i.e. switcing between MUSL/glibc. So this is safer
         run: |
-         cargo install cargo-license
-         python3 createbom.py --dash ${{github.workspace}}/dash-databroker-deps ../databroker
+          cargo tree -e normal --prefix none --no-dedupe --target all --all-features  > ${{github.workspace}}/cargodeps
+          cat ${{github.workspace}}/cargodeps | sort -u  \
+          | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker  \
+          | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \
+          > ${{github.workspace}}/dash-databroker-deps
 
       - name: Dash license check
         uses: eclipse-kuksa/kuksa-actions/check-dash@2

diff --git a/kuksa_databroker/Dockerfile b/kuksa_databroker/Dockerfile
@@ -1,5 +1,5 @@
 # /********************************************************************************
-# * Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation
+# * Copyright (c) 2024 Contributors to the Eclipse Foundation
 # *
 # * See the NOTICE file(s) distributed with this work for additional
 # * information regarding copyright ownership.
@@ -44,7 +44,8 @@ COPY ./target/riscv64gc-unknown-linux-gnu/release/databroker /app/databroker
 FROM target-$TARGETARCH as target
 ARG TARGETARCH
 
-COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty
+COPY ./dist/$TARGETARCH/sbom.json /app/
+COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty
 
 COPY ./data/vss-core/vss_release_3.1.1.json vss_release_3.1.1.json
 COPY ./data/vss-core/vss_release_4.0.json vss_release_4.0.json

diff --git a/kuksa_databroker/Dockerfile-cli b/kuksa_databroker/Dockerfile-cli
@@ -1,5 +1,5 @@
 # /********************************************************************************
-# * Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation
+# * Copyright (c) 2024 Contributors to the Eclipse Foundation
 # *
 # * See the NOTICE file(s) distributed with this work for additional
 # * information regarding copyright ownership.
@@ -50,7 +50,8 @@ RUN apk update && apk add ncurses-terminfo-base
 FROM target-$TARGETARCH as target
 ARG TARGETARCH
 
-COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty
+COPY ./dist/$TARGETARCH/sbom.json /app/
+COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty
 
 # Copy terminfo database
 COPY --from=terminfo-donor /etc/terminfo /etc/terminfo

diff --git a/kuksa_databroker/build-all-targets-cli.sh b/kuksa_databroker/build-all-targets-cli.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2023 Contributors to the Eclipse Foundation
+# Copyright (c) 2024 Contributors to the Eclipse Foundation
 #
 # Building all currently supported targets for databroker-cli.
 # Uses cross for cross-compiling. Needs to be executed
@@ -10,71 +10,69 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-
 # exit on error, to not waste any time
 set -e
 
+
 CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
-# Create thirdparty bom
-cd createbom/
-rm -rf ../databroker/thirdparty || true
-python3 createbom.py ../databroker-cli
-cd ..
+# Builds for a given target and collects data to be distirbuted in docker. Needs
+# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker
+# architecture (i.e. amd64) as input
+function build_target() {
+    target_rust=$1
+    target_docker=$2
+
+    echo "Building databroker-cli for target $target_rust"
+    cross build --target $target_rust --bin databroker-cli --release
+
+    echo "Create $target_rust SBOM"
+    cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ../Cargo.toml
+
+    echo "Prepare $target_docker dist folder"
+    mkdir ../dist/$target_docker
+    cp ../target/$target_rust/release/databroker-cli ../dist/$target_docker
+    cp ./databroker-cli/databroker-cli_bin.cdx.json ../dist/$target_docker/sbom.json
+
+    rm -rf ../dist/$target_docker/thirdparty-licenses || true
+
+    cd createbom/
+    rm -rf ../databroker/thirdparty-licenses || true
+    python3 collectlicensefromcyclonedx.py ../databroker-cli/databroker_bin.cdx.json ../../dist/$target_docker/thirdparty-licenses --curation ../licensecuration.yaml
+    cd ..
+
+    # We need to clean this folder in target, otherwise we get weird side
+    # effects building the aarch image, complaining libc crate can not find
+    # GLIBC, i.e
+    #   Compiling libc v0.2.149
+    #error: failed to run custom build command for `libc v0.2.149`
+    #
+    #Caused by:
+    #  process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1)
+    #  --- stderr
+    #  /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build)
+    #
+    # It seems cross/cargo is reusing something from previous builds it shouldn't.
+    # the finished artifact resides in ../target/x86_64-unknown-linux-musl/release
+    # so deleting the temporary files in target/releae is no problem
+    echo "Cleaning up...."
+    rm -rf ../target/release
+
+}
 
 # Starting a fresh build
 echo "Cargo clean, to start fresh..."
 cargo clean
 rm -rf ../dist || true
 mkdir ../dist
 
-# Buidling AMD46
-echo "Building AMD64"
-cross build --target x86_64-unknown-linux-musl --bin databroker-cli --release
-# We need to clean this folder in target, otherwise we get weird side
-# effects building the aarch image, complaining libc crate can not find
-# GLIBC, i.e
-#   Compiling libc v0.2.149
-#error: failed to run custom build command for `libc v0.2.149`
-#
-#Caused by:
-#  process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1)
-#  --- stderr
-#  /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build)
-#
-# It seems cross/cargo is reusing something from previous builds it shouldn't.
-# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release
-# so deleting the temporary files in trget/releae is no problem
-echo "Cleaning up...."
-rm -rf ../target/release
-
-
-# Buidling ARM64
-echo "Building ARM64"
-cross build --target aarch64-unknown-linux-musl --bin databroker-cli --release
-echo "Cleaning up...."
-rm -rf ../target/release
+# Building AMD46
+build_target x86_64-unknown-linux-musl amd64
 
+# Building ARM64
+build_target  aarch64-unknown-linux-musl arm64
 
 # Build RISCV64, this is a glibc based build, as musl is not
 # yet supported
-echo "Building RISCV64"
-cross build --target riscv64gc-unknown-linux-gnu --bin databroker-cli --release
-echo "Cleaning up...."
-rm -rf ../target/release
-
-# Prepare dist folders
-echo "Prepare amd64 dist folder"
-mkdir ../dist/amd64
-cp ../target/x86_64-unknown-linux-musl/release/databroker-cli ../dist/amd64
-cp -r ./databroker-cli/thirdparty ../dist/amd64
-
-echo "Prepare arm64 dist folder"
-mkdir ../dist/arm64
-cp ../target/aarch64-unknown-linux-musl/release/databroker-cli ../dist/arm64
-cp -r ./databroker-cli/thirdparty ../dist/arm64
-
-echo "Prepare riscv64 dist folder"
-mkdir ../dist/riscv64
-cp ../target/riscv64gc-unknown-linux-gnu/release/databroker-cli ../dist/riscv64
-cp -r ./databroker-cli/thirdparty ../dist/riscv64
+# Building RISCV64
+build_target  riscv64gc-unknown-linux-gnu riscv64
diff --git a/kuksa_databroker/build-all-targets.sh b/kuksa_databroker/build-all-targets.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2023 Contributors to the Eclipse Foundation
+# Copyright (c) 2024 Contributors to the Eclipse Foundation
 #
 # Building all currently supported targets.
 # Uses cross for cross-compiling. Needs to be executed
@@ -13,67 +13,66 @@
 # exit on error, to not waste any time
 set -e
 
+
 CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
-# Create thirdparty bom
-cd createbom/
-rm -rf ../databroker/thirdparty || true
-python3 createbom.py ../databroker
-cd ..
+# Builds for a given target and collects data to be distirbuted in docker. Needs
+# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker
+# architecture (i.e. amd64) as input
+function build_target() {
+    target_rust=$1
+    target_docker=$2
+
+    echo "Building databroker for target $target_rust"
+    cross build --target $target_rust --bin databroker --release
+
+    echo "Create $target_rust SBOM"
+    cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ../Cargo.toml
+
+    echo "Prepare $target_docker dist folder"
+    mkdir ../dist/$target_docker
+    cp ../target/$target_rust/release/databroker ../dist/$target_docker
+    cp ./databroker/databroker_bin.cdx.json ../dist/$target_docker/sbom.json
+
+    rm -rf ../dist/$target_docker/thirdparty-licenses || true
+
+    cd createbom/
+    rm -rf ../databroker/thirdparty-licenses || true
+    python3 collectlicensefromcyclonedx.py ../databroker/databroker_bin.cdx.json ../../dist/$target_docker/thirdparty-licenses --curation ../licensecuration.yaml
+    cd ..
+
+    # We need to clean this folder in target, otherwise we get weird side
+    # effects building the aarch image, complaining libc crate can not find
+    # GLIBC, i.e
+    #   Compiling libc v0.2.149
+    #error: failed to run custom build command for `libc v0.2.149`
+    #
+    #Caused by:
+    #  process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1)
+    #  --- stderr
+    #  /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build)
+    #
+    # It seems cross/cargo is reusing something from previous builds it shouldn't.
+    # the finished artifact resides in ../target/x86_64-unknown-linux-musl/release
+    # so deleting the temporary files in target/releae is no problem
+    echo "Cleaning up...."
+    rm -rf ../target/release
+
+}
 
 # Starting a fresh build
 echo "Cargo clean, to start fresh..."
 cargo clean
 rm -rf ../dist || true
 mkdir ../dist
 
-# Buidling AMD46
-echo "Building AMD64"
-cross build --target x86_64-unknown-linux-musl --bin databroker --release
-# We need to clean this folder in target, otherwise we get weird side
-# effects building the aarch image, complaining libc crate can not find
-# GLIBC, i.e
-#   Compiling libc v0.2.149
-#error: failed to run custom build command for `libc v0.2.149`
-#
-#Caused by:
-#  process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1)
-#  --- stderr
-#  /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build)
-#
-# It seems cross/cargo is reusing something from previous builds it shouldn't.
-# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release
-# so deleting the temporary files in trget/releae is no problem
-echo "Cleaning up...."
-rm -rf ../target/release
-
-
-# Buidling ARM64
-echo "Building ARM64"
-cross build --target aarch64-unknown-linux-musl --bin databroker --release
-echo "Cleaning up...."
-rm -rf ../target/release
+# Building AMD46
+build_target x86_64-unknown-linux-musl amd64
 
+# Building ARM64
+build_target  aarch64-unknown-linux-musl arm64
 
 # Build RISCV64, this is a glibc based build, as musl is not
 # yet supported
-echo "Building RISCV64"
-cross build --target riscv64gc-unknown-linux-gnu --bin databroker --release
-echo "Cleaning up...."
-rm -rf ../target/release
-
-# Prepare dist folders
-echo "Prepare amd64 dist folder"
-mkdir ../dist/amd64
-cp ../target/x86_64-unknown-linux-musl/release/databroker ../dist/amd64
-cp -r ./databroker/thirdparty ../dist/amd64
-
-echo "Prepare arm64 dist folder"
-mkdir ../dist/arm64
-cp ../target/aarch64-unknown-linux-musl/release/databroker ../dist/arm64
-cp -r ./databroker/thirdparty ../dist/arm64
-
-echo "Prepare riscv64 dist folder"
-mkdir ../dist/riscv64
-cp ../target/riscv64gc-unknown-linux-gnu/release/databroker ../dist/riscv64
-cp -r ./databroker/thirdparty ../dist/riscv64
+# Building RISCV64
+build_target  riscv64gc-unknown-linux-gnu riscv64
diff --git a/kuksa_databroker/createbom/bomutil/maplicensefile.py b/kuksa_databroker/createbom/bomutil/maplicensefile.py
@@ -35,6 +35,8 @@
     "rustls-webpki": "webpki.LICENSE.txt.gz",
     # License text taken from https://spdx.org/licenses/0BSD.html
     "0BSD": "0BSD.txt.gz",
-    # License test taken from https://spdx.org/licenses/BSD-3-Clause.html
-    "BSD-3-Clause": "BSD-3-Clause.txt.gz"
+    # License text taken from https://spdx.org/licenses/BSD-3-Clause.html
+    "BSD-3-Clause": "BSD-3-Clause.txt.gz",
+    # License text taken from https://www.openssl.org/source/license-openssl-ssleay.txt
+    "OPENSSL": "OpenSSL.txt.gz"
 }