From bc708b45377ea55c985280ffec956edf93460049 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 26 Jul 2023 12:46:56 +0200 Subject: [PATCH] Support OEM systemd-sysext images and Flatcar extensions The OEM software and future official Flatcar extensions will be shipped as systemd-sysext images, coupled to the Flatcar version and A/B updated. Instead of adding support for this in the update-engine C++ code the plan was to use a new helper binary in the post-inst action for downloading of the payload. Since this is not ready we use curl and a small script to decode the payload. The A/B update mechanism includes a migration path for old instances that first need a fallback download, because the old update-engine client doesn't pass the XML dump, then they need to go through another update cycle to have systemd-sysext images for both partitions, and then the migration can take place in the initrd where the old OEM contents get cleaned up. --- decode_payload | 20 ++- flatcar-postinst | 160 ++++++++++++++++++ src/update_engine/omaha_request_action.cc | 3 + .../omaha_request_action_unittest.cc | 1 + src/update_engine/prefs.cc | 1 + src/update_engine/prefs_interface.h | 1 + 6 files changed, 181 insertions(+), 5 deletions(-) diff --git a/decode_payload b/decode_payload index 423754d..e77140b 100755 --- a/decode_payload +++ b/decode_payload @@ -1,15 +1,19 @@ #!/bin/bash set -euo pipefail -CHECKINPUT=0 +DEBUG="${DEBUG-0}" +CHECKINPUT="${CHECKINPUT-0}" SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" +PROTOPATH="${PROTOPATH-"${SCRIPTFOLDER}"/src/update_engine}" + if [ $# -lt 3 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "Usage: $0 PUBKEY PAYLOAD OUTPUT [KERNELOUTPUT]" echo "Decodes a payload, writing the decoded payload to stdout and payload information to stderr" echo "Only one optional kernel payload is supported (the output file will be zero-sized if no payload was found)." echo "Only a signle signature is supported as only one pubkey is accepted and in the multi-signature case only the second signature is looked at." echo "Pass CHECKINPUT=1 to process the inline checksums in addition to the final checksum." + echo "Pass PROTOPATH=folder/ to specify the location of a directory with protobuf files." exit 1 fi @@ -23,9 +27,11 @@ KERNEL="${4-}" MLEN=$(dd status=none bs=1 skip=12 count=8 if="${FILE}" | od --endian=big -An -vtu8 -w1024 | tr -d ' ') # The manifest starts at offset 20 (with tail we do a +1 compared to dd) and we feed it into protoc for decoding (we assume that the text output format is stable) -DESC=$(protoc --decode=chromeos_update_engine.DeltaArchiveManifest --proto_path "${SCRIPTFOLDER}"/src/update_engine "${SCRIPTFOLDER}"/src/update_engine/update_metadata.proto < <({ tail -c +21 "${FILE}" || true ; } |head -c "${MLEN}")) +DESC=$(protoc --decode=chromeos_update_engine.DeltaArchiveManifest --proto_path "${PROTOPATH}" "${PROTOPATH}"/update_metadata.proto < <({ tail -c +21 "${FILE}" || true ; } |head -c "${MLEN}")) -echo "${DESC}" >&2 +if [ "${DEBUG}" = 1 ]; then + echo "${DESC}" >&2 +fi # Truncate true > "${PARTOUT}" @@ -117,8 +123,12 @@ fi # The signature protobuf message is at the signature offset # Decoding the "data" field caues some troubles and needs a workaround below for the dev key -SIGDESC=$(protoc --decode=chromeos_update_engine.Signatures --proto_path "${SCRIPTFOLDER}"/src/update_engine "${SCRIPTFOLDER}"/src/update_engine/update_metadata.proto < <({ tail -c +$((21 + MLEN + SIGOFFSET)) "${FILE}" || true ; } |head -c "${SIGSIZE}")) -echo "${SIGDESC}" >&2 +SIGDESC=$(protoc --decode=chromeos_update_engine.Signatures --proto_path "${PROTOPATH}" "${PROTOPATH}"/update_metadata.proto < <({ tail -c +$((21 + MLEN + SIGOFFSET)) "${FILE}" || true ; } |head -c "${SIGSIZE}")) + +if [ "${DEBUG}" = 1 ]; then + echo "${SIGDESC}" >&2 +fi + VERSION=2 # Init for the single-signature case, in the many-signature case the first signature ("version 1" but better would be "number 1") is, # at least for Flatcar production, a dummy and parsing it overwrites this variable with 1 and it will be ignored, # the second signature is "version 2" and the one we want to check. Even if only one signature is there it becomes "version 2", diff --git a/flatcar-postinst b/flatcar-postinst index 328d174..66570c2 100644 --- a/flatcar-postinst +++ b/flatcar-postinst @@ -35,6 +35,7 @@ fi # shellcheck source=/dev/null source "${INSTALL_MNT}/lib/os-release" NEXT_VERSION_ID=${VERSION_ID} +NEXT_VERSION="${VERSION}" # shellcheck source=/dev/null source /usr/lib/os-release @@ -43,6 +44,165 @@ tee_journal() { tee >(systemd-cat -t coreos-postinst) } +OEMID=$({ grep -m 1 -o "^ID=.*" "${OEM_MNT}"/oem-release || true ; } | cut -d = -f 2) + +# Must not be used as "if sysext_download; then" or "sysext_download ||" because that makes set -e a no-op, and also must not use "( sysext_download )" because we want to set the global SUCCESS variable. +sysext_download() { + local name="$1" # Payload name + local target="$2" # Path to write the payload to, writing does not need to be atomic because the caller later does an atomic move + local from="${3-}" # Either path to XML dump or the constant "release-server" + local base="" + local entries="" + local hash="" + local size="" + local url="" + local ret + SUCCESS=false + set +e + ( + set -e + # TODO: Replace the below with invoking an ue-rs helper binary for downloading the payload "name", either from the XML data or the release server ("from"), and write unpacked, verified file to "target" + if [ "${from}" = "release-server" ]; then + url="https://update.release.flatcar-linux.net/${FLATCAR_BOARD}/${NEXT_VERSION}/${name}" + elif [ "${from}" = "bincache-server" ]; then + url="https://bincache.flatcar-linux.net/images/${FLATCAR_BOARD/-usr}/${NEXT_VERSION}/${name}" + else + base=$(grep -m 1 -o 'codebase="[^"]*"' "${from}" | cut -d '"' -f 2) + entries=$(grep -m 1 -o "]*" "${from}") + url="${base}/${name}" + size=$(echo "${entries}" | grep -o 'size="[0-9]*' | cut -d '"' -f 2) + hash=$(echo "${entries}" | grep -o -P 'hash="[^"]*' | cut -d '"' -f 2) # openssl dgst -binary -sha1 < "$PAYLOAD" | base64 + fi + rm -f "${target}.tmp" + curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 -o "${target}.tmp" "${url}" + if [ "${size}" != "" ] && [ "${hash}" != "" ]; then + if [ "$(stat --printf='%s' "${target}.tmp")" != "${size}" ]; then + echo "Size mismatch for ${name}" >&2 + return 1 # jump to ret= + fi + if [ "$(openssl dgst -binary -sha1 < "${target}.tmp" | base64)" != "${hash}" ]; then + echo "Hash mismatch for ${name}" >&2 + return 1 # jump to ret= + fi + fi + # Using "${INSTALL_MNT}" here is ok because it was verified first by update-engine + PROTOPATH="${INSTALL_MNT}"/share/update_engine/ "${INSTALL_MNT}"/share/update_engine/decode_payload /usr/share/update_engine/update-payload-key.pub.pem "${target}.tmp" "${target}" + ) + ret=$? + set -e + rm -f "${target}.tmp" + if [ "${ret}" -eq 0 ]; then + SUCCESS=true + fi +} + +# To know whether an OEM update payload is expected we can't rely on checking if the Omaha response contains one +# because users may run their own instance and forget to supply it, or this in an instance that doesn't hand us +# the XML dump over. In both cases we do a fallback download and rely on a hardcoded list of OEM sysexts which we +# anyway need to maintain for the migration actions. Besides checking that an entry in the list exists we can also +# check for the active-oem-OEM flag file to support custom OEMs (but they must be part of the Omaha response). +if [ "${OEMID}" != "" ] && { [ -e "${INSTALL_MNT}/share/flatcar/oems/${OEMID}" ] || [ -e "${OEM_MNT}/sysext/active-oem-${OEMID}" ]; }; then + mkdir -p "${OEM_MNT}"/sysext/ /etc/flatcar/oem-sysext/ + # Delete sysext images that belonged to the now overwritten /usr partition but keep the sysext image for the current version + KEEP="${OEM_MNT}/sysext/oem-${OEMID}-${VERSION}.raw" + if [ ! -e "${KEEP}" ]; then + KEEP="/etc/flatcar/oem-sysext/oem-${OEMID}-${VERSION}.raw" + fi + if [ ! -e "${KEEP}" ]; then + KEEP="${OEM_MNT}/sysext/oem-${OEMID}-initial.raw" # It may not exist as well but that's ok (also, it can only exist on the OEM partition) + fi + shopt -s nullglob + for OLD_IMAGE in "${OEM_MNT}"/sysext/oem*raw /etc/flatcar/oem-sysext/oem*raw; do + if [ "${OLD_IMAGE}" != "${KEEP}" ] && [ -f "${OLD_IMAGE}" ]; then + rm -f "${OLD_IMAGE}" + fi + done + # Note that in the case of VERSION=NEXT_VERSION we will replace the running sysext and maybe it's better + # to do so than not because it allows to recover from a corrupted file (where the corruption happened on disk) + SUCCESS=false + # Preferred is to download from the location given by the Omaha response + # which only works with a new update-engine client that creates "full-response", + # and we also have to check that this file was created fresh for this update operation + # (relies on the reset of /var/lib/update_engine/prefs/previous-version that old clients also do) + if [ -e /var/lib/update_engine/prefs/full-response ] && [ $(stat -L --printf='%Y' /var/lib/update_engine/prefs/full-response) -gt $(stat -L --printf='%Y' /var/lib/update_engine/prefs/previous-version) ]; then + rm -f "/var/lib/update_engine/oem-${OEMID}.raw" + sysext_download "oem-${OEMID}.gz" "/var/lib/update_engine/oem-${OEMID}.raw" /var/lib/update_engine/prefs/full-response + fi + # If that was not provided due to updating from an old version or if the download failed, try the release server + if [ "${SUCCESS}" = false ]; then + rm -f "/var/lib/update_engine/oem-${OEMID}.raw" + sysext_download "oem-${OEMID}.gz" "/var/lib/update_engine/oem-${OEMID}.raw" release-server + fi + # Finally, try downloading the dey-key signed payload from bincache (won't be accepted on release builds, requires dev key being used) + if [ "${SUCCESS}" = false ]; then + rm -f "/var/lib/update_engine/oem-${OEMID}.raw" + sysext_download "flatcar_test_update-oem-${OEMID}.gz" "/var/lib/update_engine/oem-${OEMID}.raw" bincache-server + fi + if [ "${SUCCESS}" = false ]; then + rm -f "/var/lib/update_engine/oem-${OEMID}.raw" + echo "Failed to download required OEM update payload" >&2 + exit 1 + fi + NEW_SYSEXT="${OEM_MNT}/sysext/oem-${OEMID}-${NEXT_VERSION}.raw" + # We don't need to check if it's the initial MVP OEM because it's an update payload provided for a particular version + echo "Trying to place /var/lib/update_engine/oem-${OEMID}-${NEXT_VERSION}.raw on OEM partition" >&2 + if ! mv "/var/lib/update_engine/oem-${OEMID}.raw" "${NEW_SYSEXT}"; then + echo "That failed, moving it to right location on root partition" >&2 + NEW_SYSEXT="/etc/flatcar/oem-sysext/oem-${OEMID}-${NEXT_VERSION}.raw" + mv "/var/lib/update_engine/oem-${OEMID}.raw" "${NEW_SYSEXT}" + fi + if [ -e "${KEEP}" ] && [ -e "${NEW_SYSEXT}" ] && [ ! -e "${OEM_MNT}/sysext/active-oem-${OEMID}" ]; then + if [ -e "${INSTALL_MNT}/share/flatcar/oems/${OEMID}" ]; then + touch "${OEM_MNT}/sysext/migrate-oem-${OEMID}" + fi + touch "${OEM_MNT}/sysext/active-oem-${OEMID}" + fi +fi + +# Download official Flatcar extensions +# The enabled-sysext.conf file is read from /etc and /usr and contains one name per line, +# and when the name is prefixed with a "-" it means that the extension should be disabled if enabled by default in the file from /usr. +# It may contain comments starting with "#" at the beginning of a line or after a name. +# The file is also used in bootengine to know which extensions to enable. +# Note that we don't need "{ grep || true ; }" to suppress the match return code because in for _ in $(grep...) return codes are ignored +for NAME in $(grep -h -o '^[^#]*' /etc/flatcar/enabled-sysext.conf /usr/share/flatcar/enabled-sysext.conf | grep -v -x -f <(grep '^-' /etc/flatcar/enabled-sysext.conf | cut -d - -f 2-) | grep -v -P '^(-).*'); do + KEEP="/etc/flatcar/sysext/flatcar-${NAME}-${VERSION}.raw" + shopt -s nullglob + # Delete sysext images that belonged to the now overwritten /usr partition but keep the sysext image for the current version + for OLD_IMAGE in /etc/flatcar/sysext/flatcar*raw; do + if [ "${OLD_IMAGE}" != "${KEEP}" ] && [ -f "${OLD_IMAGE}" ]; then + rm -f "${OLD_IMAGE}" + fi + done + # Note that in the case of VERSION=NEXT_VERSION we will replace the running sysext and maybe it's better + # to do so than not because it allows to recover from a corrupted file (where the corruption happened on disk) + SUCCESS=false + # Preferred is to download from the location given by the Omaha response + # which only works with a new update-engine client that creates "full-response", + # and we also have to check that this file was created fresh for this update operation + # (relies on the reset of /var/lib/update_engine/prefs/previous-version that old clients also do) + if [ -e /var/lib/update_engine/prefs/full-response ] && [ $(stat -L --printf='%Y' /var/lib/update_engine/prefs/full-response) -gt $(stat -L --printf='%Y' /var/lib/update_engine/prefs/previous-version) ]; then + rm -f "/var/lib/update_engine/flatcar-${NAME}.raw" + sysext_download "flatcar-${NAME}.gz" "/var/lib/update_engine/flatcar-${NAME}.raw" /var/lib/update_engine/prefs/full-response + fi + # If that was not provided due to updating from an old version or if the download failed, try the release server + if [ "${SUCCESS}" = false ]; then + rm -f "/var/lib/update_engine/flatcar-${NAME}.raw" + sysext_download "flatcar-${NAME}.gz" "/var/lib/update_engine/flatcar-${NAME}.raw" release-server + fi + # Finally, try downloading the dey-key signed payload from bincache (won't be accepted on release builds, requires dev key being used) + if [ "${SUCCESS}" = false ]; then + rm -f "/var/lib/update_engine/flatcar-${NAME}.raw" + sysext_download "flatcar_test_update-flatcar-${NAME}.gz" "/var/lib/update_engine/flatcar-${NAME}.raw" bincache-server + fi + if [ "${SUCCESS}" = false ]; then + rm -f "/var/lib/update_engine/flatcar-${NAME}.raw" + echo "Failed to download required OEM update payload" >&2 + exit 1 + fi + mv "/var/lib/update_engine/flatcar-${NAME}.raw" "/etc/flatcar/sysext/flatcar-${NAME}-${NEXT_VERSION}.raw" +done + # Keep old nodes on cgroup v1 if [[ "${BUILD_ID}" != "dev-"* ]]; then if [ "${VERSION_ID%%.*}" -lt 2956 ]; then diff --git a/src/update_engine/omaha_request_action.cc b/src/update_engine/omaha_request_action.cc index 25ea3af..269a003 100644 --- a/src/update_engine/omaha_request_action.cc +++ b/src/update_engine/omaha_request_action.cc @@ -614,6 +614,9 @@ void OmahaRequestAction::TransferComplete(HttpFetcher *fetcher, string current_response(response_buffer_.begin(), response_buffer_.end()); LOG(INFO) << "Omaha request response: " << current_response; + LOG_IF(WARNING, !system_state_->prefs()->SetString(kPrefsFullResponse, current_response)) + << "Unable to write full response."; + // Events are best effort transactions -- assume they always succeed. if (IsEvent()) { CHECK(!HasOutputPipe()) << "No output pipe allowed for event requests."; diff --git a/src/update_engine/omaha_request_action_unittest.cc b/src/update_engine/omaha_request_action_unittest.cc index 8d7d178..10c95bb 100644 --- a/src/update_engine/omaha_request_action_unittest.cc +++ b/src/update_engine/omaha_request_action_unittest.cc @@ -517,6 +517,7 @@ TEST(OmahaRequestActionTest, FormatUpdateCheckOutputTest) { NiceMock prefs; EXPECT_CALL(prefs, GetString(kPrefsPreviousVersion, _)) .WillOnce(DoAll(SetArgumentPointee<1>(string("")), Return(true))); + EXPECT_CALL(prefs, SetString(kPrefsFullResponse, _)).Times(1); EXPECT_CALL(prefs, SetString(kPrefsPreviousVersion, _)).Times(1); ASSERT_FALSE(TestUpdateCheck(&prefs, GetDefaultTestParams(), diff --git a/src/update_engine/prefs.cc b/src/update_engine/prefs.cc index 5470ee1..114a3a2 100644 --- a/src/update_engine/prefs.cc +++ b/src/update_engine/prefs.cc @@ -38,6 +38,7 @@ const char kPrefsCurrentUrlIndex[] = "current-url-index"; const char kPrefsCurrentUrlFailureCount[] = "current-url-failure-count"; const char kPrefsBackoffExpiryTime[] = "backoff-expiry-time"; const char kPrefsAlephVersion[] = "aleph-version"; +const char kPrefsFullResponse[] = "full-response"; bool Prefs::Init(const files::FilePath& prefs_dir) { prefs_dir_ = prefs_dir; diff --git a/src/update_engine/prefs_interface.h b/src/update_engine/prefs_interface.h index b0b15fd..4899203 100644 --- a/src/update_engine/prefs_interface.h +++ b/src/update_engine/prefs_interface.h @@ -28,6 +28,7 @@ extern const char kPrefsCurrentUrlIndex[]; extern const char kPrefsCurrentUrlFailureCount[]; extern const char kPrefsBackoffExpiryTime[]; extern const char kPrefsAlephVersion[]; +extern const char kPrefsFullResponse[]; // The prefs interface allows access to a persistent preferences // store. The two reasons for providing this as an interface are