From 2ecda2523a93a7bbcf8c502e5ce9c754d797d1a4 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 09:41:25 +0000 Subject: [PATCH 1/7] Support codenotary verify/signing --- Dockerfile | 14 ++++------ builder.sh | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7871a6b..f620079 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,18 @@ FROM $BUILD_FROM # Set shell SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV \ + VCN_OTP_EMPTY=true \ + LANG=C.UTF-8 # Setup locals RUN apt-get update && apt-get install -y --no-install-recommends \ jq \ git \ + curl \ python3-setuptools \ + && bash <(curl https://getvcn.codenotary.com -L) \ && rm -rf /var/lib/apt/lists/* \ -ENV LANG C.UTF-8 # Install docker # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ @@ -28,14 +32,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ containerd.io \ && rm -rf /var/lib/apt/lists/* -# Setup arm binary support -ARG BUILD_ARCH -RUN if [ "$BUILD_ARCH" != "amd64" ]; then exit 0; else \ - apt-get update && apt-get install -y --no-install-recommends \ - qemu-user-static \ - binfmt-support \ - && rm -rf /var/lib/apt/lists/*; fi - COPY builder.sh /usr/bin/ WORKDIR /data diff --git a/builder.sh b/builder.sh index 0bb4ded..9805cc9 100755 --- a/builder.sh +++ b/builder.sh @@ -17,6 +17,8 @@ DOCKER_PUSH=true DOCKER_USER= DOCKER_PASSWORD= DOCKER_LOCAL=false +VCN_NOTARY=false +VCN_FROM= SELF_CACHE=false CUSTOM_CACHE_TAG= RELEASE_TAG=false @@ -139,6 +141,15 @@ Options: Build the landingpage for machines. --homeassistant-machine Build the machine based image for a release. + + Security: + --with-codenotary + Enable signing images with CodeNotary. Need set follow env: + VCN_USER + VCN_PASSWORD + VCN_NOTARIZATION_PASSWORD + --validate-from + Validate the FROM image which is used to build the image. EOF bashio::exit.nok @@ -263,6 +274,9 @@ function run_build() { docker_cli+=("--build-arg" "BUILD_ARCH=$build_arch") fi + # Validate the base image + codenotary_validate "$build_from" + # Build image bashio::log.info "Run build for $repository/$image:$version" docker build --pull -t "$repository/$image:$version" \ @@ -312,6 +326,9 @@ function run_build() { done done fi + + # Singing image + codenotary_sign "${repository}/${image}:${version}" } @@ -688,6 +705,59 @@ function init_crosscompile() { > /dev/null 2>&1 || bashio::log.warning "Can't enable crosscompiling feature" } +#### Security CodeNotary #### + +function codenotary_probe() { + if ! bashio::var.has_value "${VCN_USER}" || ! bashio::var.has_value "${VCN_PASSWORD}" || ! bashio::var.has_value "${VCN_NOTARIZATION_PASSWORD}"; then + bashio::exit.nok "Missing ENV values for CodeNotary" + fi +} + + +function codenotary_setup() { + if bashio::var.false "${DOCKER_PUSH}" || bashio::var.false "${VCN_NOTARY}"; then + return 0 + fi + + vcn login /dev/null 2>&1 || bashio::exit.nok "Login to CodeNotary fails!" +} + +function codenotary_sign() { + local image=$1 + + if bashio::var.false "${DOCKER_PUSH}" || bashio::var.false "${VCN_NOTARY}"; then + return 0 + fi + + vcn notarize --public "docker://${image}" +} + +function codenotary_validate() { + local image=$1 + local state= + local vcn_cli=() + + if ! bashio::var.has_value "${VCN_FROM}"; then + return 0 + fi + + bashio::log.info "Download base image ${image} for CodeNotary validation" + docker pull "${image}" > /dev/null 2>&1 || bashio::exit.nok "Can't pull image ${image}" + + if [[ "${VCN_FROM}" =~ 0x.* ]]; then + vcn_cli+=("--signerID" "${VCN_FROM}") + else + vcn_cli+=("--org" "${VCN_FROM}") + fi + + state="$(vcn authenticate "${vcn_cli[@]}" --output json "docker://{image}" | jq '.verification.status // 2')" + if [[ "${state}" != "0" ]]; then + bashio::exit.nok "Validation of base image fails!" + fi + bashio::log.info "Base imge ${image} is trusted" +} + + #### Error handling #### function error_handling() { @@ -842,7 +912,14 @@ while [[ $# -gt 0 ]]; do SELF_CACHE=true shift ;; - + --with-codenotary) + VCN_NOTARY=true + ;; + --validate-from) + codenotary_probe + VCN_FROM=$2 + shift + ;; *) bashio::exit.nok "$0 : Argument '$1' unknown" ;; @@ -869,10 +946,11 @@ mkdir -p /data init_crosscompile start_docker -# Login into dockerhub +# Login into dockerhub & setup CodeNotary if [ -n "$DOCKER_USER" ] && [ -n "$DOCKER_PASSWORD" ]; then docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD" fi +codenotary_setup # Load external repository if [ -n "$GIT_REPOSITORY" ]; then From 6714ea8ed792426f34fb079031dcc2811e8a6cff Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 09:56:36 +0000 Subject: [PATCH 2/7] Migrate to alpine --- Dockerfile | 64 +++++++++++++++++++++++++++++++++--------------------- build.json | 11 +++++++--- builder.sh | 30 ------------------------- 3 files changed, 47 insertions(+), 58 deletions(-) diff --git a/Dockerfile b/Dockerfile index f620079..1ec0790 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,50 @@ ARG BUILD_FROM FROM $BUILD_FROM -# Set shell -SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Setup locals ENV \ VCN_OTP_EMPTY=true \ LANG=C.UTF-8 -# Setup locals -RUN apt-get update && apt-get install -y --no-install-recommends \ - jq \ - git \ - curl \ - python3-setuptools \ - && bash <(curl https://getvcn.codenotary.com -L) \ - && rm -rf /var/lib/apt/lists/* \ +ARG BUILD_ARCH +ARG VCN_VERSION -# Install docker -# https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ -RUN apt-get update && apt-get install -y --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - curl \ - software-properties-common \ - gpg-agent \ - && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ - && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ - && apt-get update && apt-get install -y --no-install-recommends \ - docker-ce \ - docker-ce-cli \ - containerd.io \ - && rm -rf /var/lib/apt/lists/* +RUN \ + set -x \ + && apk add --no-cache \ + git \ + docker \ + && apk add --no-cache --virtual .build-dependencies \ + build-base \ + go \ + \ + && git clone -b v${VCN_VERSION} --depth 1 \ + https://github.com/codenotary/vcn \ + && cd vcn \ + \ + # Fix: https://github.com/codenotary/vcn/issues/131 + && go get github.com/codenotary/immudb@4cf9e2ae06ac2e6ec98a60364c3de3eab5524757 \ + \ + && if [ "${BUILD_ARCH}" = "armhf" ]; then \ + GOARM=6 GOARCH=arm go build -o vcn -ldflags="-s -w" ./cmd/vcn; \ + elif [ "${BUILD_ARCH}" = "armv7" ]; then \ + GOARM=7 GOARCH=arm go build -o vcn -ldflags="-s -w" ./cmd/vcn; \ + elif [ "${BUILD_ARCH}" = "aarch64" ]; then \ + GOARCH=arm64 go build -o vcn -ldflags="-s -w" ./cmd/vcn; \ + elif [ "${BUILD_ARCH}" = "i386" ]; then \ + GOARCH=386 go build -o vcn -ldflags="-s -w" ./cmd/vcn; \ + elif [ "${BUILD_ARCH}" = "amd64" ]; then \ + GOARCH=amd64 go build -o vcn -ldflags="-s -w" ./cmd/vcn; \ + else \ + exit 1; \ + fi \ + \ + && rm -rf /root/go /root/.cache \ + && mv vcn /usr/bin/vcn \ + \ + && apk del .build-dependencies \ + && rm -rf /usr/src/vcn COPY builder.sh /usr/bin/ diff --git a/build.json b/build.json index d2398a4..8c341a7 100644 --- a/build.json +++ b/build.json @@ -1,9 +1,14 @@ { "image": "homeassistant/{arch}-builder", "build_from": { - "aarch64": "homeassistant/aarch64-base-ubuntu:18.04", - "armv7": "homeassistant/armv7-base-ubuntu:18.04", - "amd64": "homeassistant/amd64-base-ubuntu:18.04" + "aarch64": "homeassistant/aarch64-base:3.13", + "armv7": "homeassistant/armv7-base:3.13", + "armhf": "homeassistant/armhf-base:3.13", + "amd64": "homeassistant/amd64-base:3.13", + "i386": "homeassistant/ir86-base:3.13" + }, + "args": { + "VCN_VERSION": "0.9.4" }, "labels": { "io.hass.type": "builder" diff --git a/builder.sh b/builder.sh index 9805cc9..0243843 100755 --- a/builder.sh +++ b/builder.sh @@ -641,36 +641,6 @@ function build_homeassistant_landingpage() { } -function build_wheels() { - local build_arch=$1 - - local version="" - local image="{arch}-wheels" - local build_from="homeassistant/${build_arch}-base-python:${PYTHON}" - local docker_cli=() - local docker_tags=() - - # Read version - if [ "$VERSION" == "dev" ]; then - version="dev" - else - version="$(python3 "$TARGET/setup.py" -V)" - fi - - # If latest python version/build - if [ "$RELEASE_TAG" == "true" ]; then - docker_tags=("$version") - fi - - # Metadata - docker_cli+=("--label" "io.hass.type=wheels") - - # Start build - run_build "$TARGET" "$DOCKER_HUB" "$image" "$version-${PYTHON}" \ - "$build_from" "$build_arch" docker_cli[@] docker_tags[@] -} - - function extract_machine_build() { local list=$1 local array=() From 4e3f0bccf1d3939485adfc8a8fb29e38dc4a8942 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 09:58:58 +0000 Subject: [PATCH 3/7] Adjust hadolint --- .hadolint.yaml | 3 ++- builder.sh | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.hadolint.yaml b/.hadolint.yaml index d700141..4cc136b 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -1,3 +1,4 @@ ignored: + - DL3003 - DL3006 - - DL3008 + - DL3018 diff --git a/builder.sh b/builder.sh index 0243843..540772a 100755 --- a/builder.sh +++ b/builder.sh @@ -125,8 +125,6 @@ Options: Default on. Run all things for an addon build. --generic Build based on the build.json - --builder-wheels - Build the wheels builder for Home Assistant. --base Build our base images. --base-python From b8c66a2bdbf2325539770c19b6e1cf39e78edc8d Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 10:00:23 +0000 Subject: [PATCH 4/7] better cleanup --- builder.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/builder.sh b/builder.sh index 540772a..be80831 100755 --- a/builder.sh +++ b/builder.sh @@ -28,7 +28,6 @@ TARGET= VERSION= IMAGE= RELEASE= -PYTHON= ALPINE= BUILD_LIST=() BUILD_TYPE="addon" @@ -874,12 +873,6 @@ while [[ $# -gt 0 ]]; do extract_machine_build "$(echo "$2" | cut -d '=' -f 2)" shift ;; - --builder-wheels) - BUILD_TYPE="builder-wheels" - PYTHON=$2 - SELF_CACHE=true - shift - ;; --with-codenotary) VCN_NOTARY=true ;; From 9e2d0152d47e5bb48a998572b034323974199bbc Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 10:01:33 +0000 Subject: [PATCH 5/7] fix name --- builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder.sh b/builder.sh index be80831..28abc8f 100755 --- a/builder.sh +++ b/builder.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bashio ###################### -# Hass.io Build-env +# Home Assistant Build-env ###################### set -e set +u From 1554590a60ddf3903e8c1ef7d77e934b2fdccabe Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 10:04:12 +0000 Subject: [PATCH 6/7] fix base image name --- build.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.json b/build.json index 8c341a7..1a81912 100644 --- a/build.json +++ b/build.json @@ -5,7 +5,7 @@ "armv7": "homeassistant/armv7-base:3.13", "armhf": "homeassistant/armhf-base:3.13", "amd64": "homeassistant/amd64-base:3.13", - "i386": "homeassistant/ir86-base:3.13" + "i386": "homeassistant/i386-base:3.13" }, "args": { "VCN_VERSION": "0.9.4" From 8971699c6eb313178f866bb1b57a92ebe10a7df0 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 22 Mar 2021 16:32:23 +0100 Subject: [PATCH 7/7] Simplify base build (#74) * Simplify base build * cleanup --- builder.sh | 203 +++++++++++++++++------------------------------------ 1 file changed, 63 insertions(+), 140 deletions(-) diff --git a/builder.sh b/builder.sh index 28abc8f..04622f1 100755 --- a/builder.sh +++ b/builder.sh @@ -26,9 +26,10 @@ GIT_REPOSITORY= GIT_BRANCH="master" TARGET= VERSION= +VERSION_BASE= +VERSION_FROM= IMAGE= RELEASE= -ALPINE= BUILD_LIST=() BUILD_TYPE="addon" BUILD_TASKS=() @@ -81,6 +82,8 @@ Options: Additional version information like for base images. --release-tag Use this as main tag. + --version-from + Use this to set build_from tag if not specified. Architecture --armhf @@ -126,14 +129,6 @@ Options: Build based on the build.json --base Build our base images. - --base-python - Build our base python images. - --base-raspbian - Build our base raspbian images. - --base-ubuntu - Build our base ubuntu images. - --base-debian - Build our base debian images. --homeassisant-landingpage Build the landingpage for machines. --homeassistant-machine @@ -329,122 +324,81 @@ function run_build() { } -#### HassIO functions #### +#### Build functions #### function build_base_image() { - local build_arch=$1 + local build_arch=${1} local build_from="" - local image="{arch}-base" - local docker_cli=() - local docker_tags=() - - # Set type - docker_cli+=("--label" "io.hass.type=base") - docker_cli+=("--label" "io.hass.base.version=$RELEASE") - docker_cli+=("--label" "io.hass.base.name=alpine") - docker_cli+=("--label" "io.hass.base.image=$DOCKER_HUB/$image") - - # Start build - run_build "$TARGET/$build_arch" "$DOCKER_HUB" "$image" "$VERSION" \ - "$build_from" "$build_arch" docker_cli[@] docker_tags[@] -} - -function build_base_python_image() { - local build_arch=$1 - - local image="{arch}-base-python" - local build_from="homeassistant/${build_arch}-base:${ALPINE}" - local version="${VERSION}-alpine${ALPINE}" + local image="" + local repository="" + local raw_image="" + local version_tag=false + local args="" local docker_cli=() local docker_tags=() - # If latest python version/build - if [ "$RELEASE_TAG" == "true" ]; then - docker_tags=("$VERSION") + # Read build.json + if bashio::fs.file_exists "${TARGET}/build.json"; then + build_from="$(jq --raw-output ".build_from.${build_arch} // empty" "${TARGET}/build.json")" + args="$(jq --raw-output '.args // empty | keys[]' "${TARGET}/build.json")" + labels="$(jq --raw-output '.labels // empty | keys[]' "${TARGET}/build.json")" + raw_image="$(jq --raw-output '.image // empty' "${TARGET}/build.json")" + version_tag="$(jq --raw-output '.version_tag // false' "${TARGET}/build.json")" fi - # Set type - docker_cli+=("--label" "io.hass.type=base") - docker_cli+=("--label" "io.hass.base.version=$RELEASE") - docker_cli+=("--label" "io.hass.base.name=python") - docker_cli+=("--label" "io.hass.base.image=$DOCKER_HUB/$image") - - # Start build - run_build "$TARGET/$VERSION" "$DOCKER_HUB" "$image" "$version" \ - "$build_from" "$build_arch" docker_cli[@] docker_tags[@] -} - - -function build_base_ubuntu_image() { - local build_arch=$1 - - local build_from="" - local image="{arch}-base-ubuntu" - local docker_cli=() - local docker_tags=() - - # Select builder image - if [ "$build_arch" == "armhf" ]; then - bashio::log.error "$build_arch not supported for ubuntu" + # Set defaults build things + if ! bashio::var.has_value "${build_from}"; then + bashio::log.error "${build_arch} not supported for this build" return 1 fi - # Set type - docker_cli+=("--label" "io.hass.type=base") - docker_cli+=("--label" "io.hass.base.version=$RELEASE") - docker_cli+=("--label" "io.hass.base.name=ubuntu") - docker_cli+=("--label" "io.hass.base.image=$DOCKER_HUB/$image") - - # Start build - run_build "$TARGET/$build_arch" "$DOCKER_HUB" "$image" "$VERSION" \ - "$build_from" "$build_arch" docker_cli[@] docker_tags[@] -} - - -function build_base_debian_image() { - local build_arch=$1 - - local build_from="" - local image="{arch}-base-debian" - local docker_cli=() - local docker_tags=() - - # Set type - docker_cli+=("--label" "io.hass.type=base") - docker_cli+=("--label" "io.hass.base.version=$RELEASE") - docker_cli+=("--label" "io.hass.base.name=debian") - docker_cli+=("--label" "io.hass.base.image=$DOCKER_HUB/$image") - - # Start build - run_build "$TARGET/$build_arch" "$DOCKER_HUB" "$image" "$VERSION" \ - "$build_from" "$build_arch" docker_cli[@] docker_tags[@] -} + # Modify build_from + if [[ "${build_from}" =~ :$ ]]; then + if bashio::var.has_value "${VERSION_FROM}"; then + build_from="${build_from}:${VERSION_FROM}" + else + build_from="${build_from}:${VERSION_BASE}" + fi + fi + # Read data from image + if ! bashio::var.has_value "${raw_image}"; then + bashio::log.error "Can't find the image tag on build.json" + return 1 + fi + repository="$(echo "${raw_image}" | cut -f 1 -d '/')" + image="$(echo "${raw_image}" | cut -f 2 -d '/')" -function build_base_raspbian_image() { - local build_arch=$1 + # Additional build args + if bashio::var.has_value "${args}"; then + for arg in ${args}; do + value="$(jq --raw-output ".args.${arg}" "${TARGET}/build.json")" + docker_cli+=("--build-arg" "${arg}=${value}") + done + fi - local build_from="$VERSION" - local image="{arch}-base-raspbian" - local docker_cli=() - local docker_tags=() + # Additional build labels + if bashio::var.has_value "${labels}"; then + for label in ${labels}; do + value="$(jq --raw-output ".labels.\"${label}\"" "${TARGET}/build.json")" + docker_cli+=("--label" "${label}=${value}") + done + fi - # Select builder image - if [ "$build_arch" != "armhf" ]; then - bashio::log.error "$build_arch not supported for raspbian" - return 1 + # Tag with version/build + if bashio::var.true "${RELEASE_TAG}"; then + docker_tags=("${VERSION}") fi # Set type docker_cli+=("--label" "io.hass.type=base") - docker_cli+=("--label" "io.hass.base.version=$RELEASE") - docker_cli+=("--label" "io.hass.base.name=raspbian") - docker_cli+=("--label" "io.hass.base.image=$DOCKER_HUB/$image") + docker_cli+=("--label" "io.hass.base.version=${RELEASE}") + docker_cli+=("--label" "io.hass.base.image=${build_from}") # Start build - run_build "$TARGET" "$DOCKER_HUB" "$image" "$VERSION" \ - "$build_from" "$build_arch" docker_cli[@] docker_tags[@] + run_build "${TARGET}" "${repository}" "${image}" "${VERSION_BASE}" \ + "${build_from}" "${build_arch}" docker_cli[@] docker_tags[@] } @@ -790,6 +744,10 @@ while [[ $# -gt 0 ]]; do DOCKER_HUB=$2 shift ;; + --version-from) + VERSION_FROM=$2 + shift + ;; --docker-hub-check) DOCKER_HUB_CHECK=true ;; @@ -825,32 +783,7 @@ while [[ $# -gt 0 ]]; do --base) BUILD_TYPE="base" SELF_CACHE=true - VERSION=$2 - shift - ;; - --base-python) - BUILD_TYPE="base-python" - SELF_CACHE=true - VERSION="$(echo "$2" | cut -d '=' -f 1)" - ALPINE="$(echo "$2" | cut -d '=' -f 2)" - shift - ;; - --base-ubuntu) - BUILD_TYPE="base-ubuntu" - SELF_CACHE=true - VERSION=$2 - shift - ;; - --base-debian) - BUILD_TYPE="base-debian" - SELF_CACHE=true - VERSION=$2 - shift - ;; - --base-raspbian) - BUILD_TYPE="base-raspbian" - SELF_CACHE=true - VERSION=$2 + VERSION_BASE=$2 shift ;; --generic) @@ -894,7 +827,7 @@ if [ "${#BUILD_LIST[@]}" -eq 0 ] && ! [[ "$BUILD_TYPE" =~ ^homeassistant-(machin fi # Check other args -if [ "$BUILD_TYPE" != "addon" ] && [ "$BUILD_TYPE" != "generic" ] && [ -z "$DOCKER_HUB" ]; then +if [[ "$BUILD_TYPE" =~ (addon|generic|base) ]] && ! bashio::var.has_value "$DOCKER_HUB"; then bashio::exit.nok "Please set a docker hub!" fi @@ -930,16 +863,6 @@ if [ "${#BUILD_LIST[@]}" -ne 0 ]; then (build_generic "$arch") & elif [ "$BUILD_TYPE" == "base" ]; then (build_base_image "$arch") & - elif [ "$BUILD_TYPE" == "base-python" ]; then - (build_base_python_image "$arch") & - elif [ "$BUILD_TYPE" == "base-ubuntu" ]; then - (build_base_ubuntu_image "$arch") & - elif [ "$BUILD_TYPE" == "base-debian" ]; then - (build_base_debian_image "$arch") & - elif [ "$BUILD_TYPE" == "base-raspbian" ]; then - (build_base_raspbian_image "$arch") & - elif [ "$BUILD_TYPE" == "builder-wheels" ]; then - (build_wheels "$arch") & elif [[ "$BUILD_TYPE" =~ ^homeassistant-(machine|landingpage)$ ]]; then continue # Handled in the loop below else