CycloneDX SBOM for databroker

Creates a CycloneDX Software Bill of Materials (SBOM) for the databroker. Refactor createbom so it can collect licenses from any CycloneDX input file, so it may be reused for other parts of the project as well. Signed-off-by: Sebastian Schildt <[email protected]>
eclipse · Mar 18, 2024 · 8b488f6 · 8b488f6
1 parent 7d9d335
commit 8b488f6
Show file tree

Hide file tree

Showing 11 changed files with 336 additions and 129 deletions.
diff --git a/.github/workflows/kuksa_databroker-cli_build.yml b/.github/workflows/kuksa_databroker-cli_build.yml
@@ -62,7 +62,7 @@ jobs:
     - name: Building
       working-directory: ${{github.workspace}}/kuksa_databroker/
       run: |
-          cargo install cargo-license cross
+          cargo install cargo-license cross cargo-cyclonedx
           ./build-all-targets-cli.sh
 
     - name: Docker meta
@@ -81,7 +81,7 @@ jobs:
           type=semver,pattern={{major}}
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
       id: buildx
@@ -148,23 +148,31 @@ jobs:
     - name: "Archiving RISCV64 artifacts"
       uses: actions/upload-artifact@v3
       with:
-        name: databroker--cli-riscv64
+        name: databroker-cli-riscv64
         path: ${{github.workspace}}/dist/riscv64
 
 
   bom:
-    name: Bill of Material Check
+    name: License Compliance Check
     runs-on: ubuntu-latest
-    needs: build-container
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: "Createbom: License check and Dash output generation"
-        working-directory: ${{github.workspace}}/kuksa_databroker/createbom
+      # Follows the pattern from
+      # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo
+      - name: "Using cargo to create Dash input"
+        working-directory: ${{github.workspace}}/kuksa_databroker/
+        # target all is not really needed, and will also return i.e. wasm deps, however
+        # better safe than sorry, the alternative would be running this for each currently
+        # built target  and combining the lists, but that would need adapting, when
+        # adding targets, or also when i.e. switching between MUSL/glibc. So this is safer
         run: |
-         cargo install cargo-license
-         python3 createbom.py --dash ${{github.workspace}}/dash-databroker-deps ../databroker
+         cargo tree -e normal --prefix none --no-dedupe --target all --all-features  > ${{github.workspace}}/cargodeps
+         cat ${{github.workspace}}/cargodeps | sort -u  \
+         | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker  \
+         | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \
+         > ${{github.workspace}}/dash-databroker-deps
 
       - name: Dash license check
         uses: eclipse-kuksa/kuksa-actions/check-dash@2

diff --git a/.github/workflows/kuksa_databroker_build.yml b/.github/workflows/kuksa_databroker_build.yml
@@ -88,7 +88,6 @@ jobs:
 
   build-container:
     runs-on: ubuntu-latest
-
     needs: check_ghcr_push
 
     steps:
@@ -99,7 +98,7 @@ jobs:
     - name: Building
       working-directory: ${{github.workspace}}/kuksa_databroker/
       run: |
-        cargo install cargo-license cross
+        cargo install cargo-license cross cargo-cyclonedx
         ./build-all-targets.sh
 
     - name: Docker meta
@@ -197,7 +196,7 @@ jobs:
     steps:
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - uses: actions/checkout@v4
 
@@ -223,18 +222,26 @@ jobs:
           ${{github.workspace}}/kuksa_databroker/integration_test/run.sh
 
   bom:
-    name: Bill of Material Check
+    name: License Compliance Check
     runs-on: ubuntu-latest
-    needs: build-container
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: "Createbom: License check and Dash output generation"
-        working-directory: ${{github.workspace}}/kuksa_databroker/createbom
+      # Follows the pattern from
+      # https://github.com/eclipse/dash-licenses?tab=readme-ov-file#example-rustcargo
+      - name: "Using cargo to create Dash input"
+        working-directory: ${{github.workspace}}/kuksa_databroker/
+        # target all is not really needed, and will also return i.e. wasm deps, however
+        # better safe than sorry, the alternative would be running this for each currently
+        # built target  and combining the lists, but that would need adapting, when
+        # adding targets, or also when i.e. switching between MUSL/glibc. So this is safer
         run: |
-         cargo install cargo-license
-         python3 createbom.py --dash ${{github.workspace}}/dash-databroker-deps ../databroker
+          cargo tree -e normal --prefix none --no-dedupe --target all --all-features  > ${{github.workspace}}/cargodeps
+          cat ${{github.workspace}}/cargodeps | sort -u  \
+          | grep -v '^[[:space:]]*$' | grep -v kuksa | grep -v databroker  \
+          | sed -E 's|([^ ]+) v([^ ]+).*|crate/cratesio/-/\1/\2|' \
+          > ${{github.workspace}}/dash-databroker-deps
 
       - name: Dash license check
         uses: eclipse-kuksa/kuksa-actions/check-dash@2

diff --git a/kuksa_databroker/Dockerfile b/kuksa_databroker/Dockerfile
@@ -1,5 +1,5 @@
 # /********************************************************************************
-# * Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation
+# * Copyright (c) 2024 Contributors to the Eclipse Foundation
 # *
 # * See the NOTICE file(s) distributed with this work for additional
 # * information regarding copyright ownership.
@@ -44,7 +44,8 @@ COPY ./target/riscv64gc-unknown-linux-gnu/release/databroker /app/databroker
 FROM target-$TARGETARCH as target
 ARG TARGETARCH
 
-COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty
+COPY ./dist/$TARGETARCH/sbom.json /app/
+COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty
 
 COPY ./data/vss-core/vss_release_3.1.1.json vss_release_3.1.1.json
 COPY ./data/vss-core/vss_release_4.0.json vss_release_4.0.json

diff --git a/kuksa_databroker/Dockerfile-cli b/kuksa_databroker/Dockerfile-cli
@@ -1,5 +1,5 @@
 # /********************************************************************************
-# * Copyright (c) 2022, 2023 Contributors to the Eclipse Foundation
+# * Copyright (c) 2024 Contributors to the Eclipse Foundation
 # *
 # * See the NOTICE file(s) distributed with this work for additional
 # * information regarding copyright ownership.
@@ -50,7 +50,8 @@ RUN apk update && apk add ncurses-terminfo-base
 FROM target-$TARGETARCH as target
 ARG TARGETARCH
 
-COPY ./dist/$TARGETARCH/thirdparty/ /app/thirdparty
+COPY ./dist/$TARGETARCH/sbom.json /app/
+COPY ./dist/$TARGETARCH/thirdparty-licenses/ /app/thirdparty
 
 # Copy terminfo database
 COPY --from=terminfo-donor /etc/terminfo /etc/terminfo

diff --git a/kuksa_databroker/build-all-targets-cli.sh b/kuksa_databroker/build-all-targets-cli.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2023 Contributors to the Eclipse Foundation
+# Copyright (c) 2024 Contributors to the Eclipse Foundation
 #
 # Building all currently supported targets for databroker-cli.
 # Uses cross for cross-compiling. Needs to be executed
@@ -10,71 +10,69 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-
 # exit on error, to not waste any time
 set -e
 
+
 CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
-# Create thirdparty bom
-cd createbom/
-rm -rf ../databroker/thirdparty || true
-python3 createbom.py ../databroker-cli
-cd ..
+# Builds for a given target and collects data to be distirbuted in docker. Needs
+# Rust target triplett (i.e. x86_64-unknown-linux-musl) and the corresponding docker
+# architecture (i.e. amd64) as input
+function build_target() {
+    target_rust=$1
+    target_docker=$2
+
+    echo "Building databroker-cli for target $target_rust"
+    cross build --target $target_rust --bin databroker-cli --release
+
+    echo "Create $target_rust SBOM"
+    cargo cyclonedx -v -f json --describe binaries --spec-version 1.4 --target $target_rust --manifest-path ../Cargo.toml
+
+    echo "Prepare $target_docker dist folder"
+    mkdir ../dist/$target_docker
+    cp ../target/$target_rust/release/databroker-cli ../dist/$target_docker
+    cp ./databroker-cli/databroker-cli_bin.cdx.json ../dist/$target_docker/sbom.json
+
+    rm -rf ../dist/$target_docker/thirdparty-licenses || true
+
+    cd createbom/
+    rm -rf ../databroker/thirdparty-licenses || true
+    python3 collectlicensefromcyclonedx.py ../databroker-cli/databroker-cli_bin.cdx.json ../../dist/$target_docker/thirdparty-licenses --curation ../licensecuration.yaml
+    cd ..
+
+    # We need to clean this folder in target, otherwise we get weird side
+    # effects building the aarch image, complaining libc crate can not find
+    # GLIBC, i.e
+    #   Compiling libc v0.2.149
+    #error: failed to run custom build command for `libc v0.2.149`
+    #
+    #Caused by:
+    #  process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1)
+    #  --- stderr
+    #  /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build)
+    #
+    # It seems cross/cargo is reusing something from previous builds it shouldn't.
+    # the finished artifact resides in ../target/x86_64-unknown-linux-musl/release
+    # so deleting the temporary files in target/releae is no problem
+    echo "Cleaning up...."
+    rm -rf ../target/release
+
+}
 
 # Starting a fresh build
 echo "Cargo clean, to start fresh..."
 cargo clean
 rm -rf ../dist || true
 mkdir ../dist
 
-# Buidling AMD46
-echo "Building AMD64"
-cross build --target x86_64-unknown-linux-musl --bin databroker-cli --release
-# We need to clean this folder in target, otherwise we get weird side
-# effects building the aarch image, complaining libc crate can not find
-# GLIBC, i.e
-#   Compiling libc v0.2.149
-#error: failed to run custom build command for `libc v0.2.149`
-#
-#Caused by:
-#  process didn't exit successfully: `/target/release/build/libc-2dd22ab6b5fb9fd2/#build-script-build` (exit status: 1)
-#  --- stderr
-#  /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.29' not found (required by /target/release/build/libc-2dd22ab6b5fb9fd2/build-script-build)
-#
-# It seems cross/cargo is reusing something from previous builds it shouldn't.
-# the finished artifact resides in ../target/x86_64-unknown-linux-musl/release
-# so deleting the temporary files in trget/releae is no problem
-echo "Cleaning up...."
-rm -rf ../target/release
-
-
-# Buidling ARM64
-echo "Building ARM64"
-cross build --target aarch64-unknown-linux-musl --bin databroker-cli --release
-echo "Cleaning up...."
-rm -rf ../target/release
+# Building AMD46
+build_target x86_64-unknown-linux-musl amd64
 
+# Building ARM64
+build_target  aarch64-unknown-linux-musl arm64
 
 # Build RISCV64, this is a glibc based build, as musl is not
 # yet supported
-echo "Building RISCV64"
-cross build --target riscv64gc-unknown-linux-gnu --bin databroker-cli --release
-echo "Cleaning up...."
-rm -rf ../target/release
-
-# Prepare dist folders
-echo "Prepare amd64 dist folder"
-mkdir ../dist/amd64
-cp ../target/x86_64-unknown-linux-musl/release/databroker-cli ../dist/amd64
-cp -r ./databroker-cli/thirdparty ../dist/amd64
-
-echo "Prepare arm64 dist folder"
-mkdir ../dist/arm64
-cp ../target/aarch64-unknown-linux-musl/release/databroker-cli ../dist/arm64
-cp -r ./databroker-cli/thirdparty ../dist/arm64
-
-echo "Prepare riscv64 dist folder"
-mkdir ../dist/riscv64
-cp ../target/riscv64gc-unknown-linux-gnu/release/databroker-cli ../dist/riscv64
-cp -r ./databroker-cli/thirdparty ../dist/riscv64
+# Building RISCV64
+build_target  riscv64gc-unknown-linux-gnu riscv64