From e8247129faa79f91b7054a594597449f854ab993 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 20 Sep 2023 17:41:24 +0200 Subject: [PATCH 1/6] show-changes: Allow running the script from different place The show-changes script required to be run from the parent directory of the scripts/coreos-overlay/portage-stable repositories. The requirement may be inconvenient in some situations (like the repo having a different name, or being forced to change the directory before invoking the script). Allow pointing the show-changes script to the locations of scripts, coreos-overlay and portage-stable repositories. If the locations are not overridden, then the script has the old expectations. --- show-changes | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/show-changes b/show-changes index 04c3992..298e617 100755 --- a/show-changes +++ b/show-changes @@ -7,11 +7,15 @@ SCRIPTFOLDER="$(dirname "$(readlink -f "$0")")" if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "Usage: $0 OLD [NEW]" echo "Shows the changes between the git references by assembling the changelog/ folder entries" - echo "Should be run in the folder that contains the coreos-overlay, portage-stable, and scripts repository folders." + echo "By default the script assumes that it is being run in the folder that contains the coreos-overlay, portage-stable, and scripts repository folders. Set SCRIPTS_REPO, COREOS_OVERLAY_REPO and PORTAGE_REPO to contain the paths to the respective repos if the default assumption is wrong." echo "The NEW reference can be omitted and will then default to HEAD." exit 1 fi +: "${SCRIPTS_REPO:='scripts'}" +: "${COREOS_OVERLAY_REPO:='coreos-overlay'}" +: "${PORTAGE_STABLE_REPO:='portage-stable'}" + OLD="$1" NEW="${2-HEAD}" OLD_FMT="" @@ -48,39 +52,41 @@ for section in security bugfixes changes updates; do esac echo for repo in coreos-overlay portage-stable scripts; do + var_name=${repo//-/_} + var_name="${var_name^^}_REPO" OLDREF="${OLD}" NEWREF="${NEW}" - OLDREPOPATH="${repo}" - NEWREPOPATH="${repo}" + OLDREPOPATH="${!var_name}" + NEWREPOPATH="${!var_name}" OLDPREPEND="" NEWPREPEND="" if [ "${repo}" != "scripts" ]; then if [ "${FETCH}" = 1 ]; then - git -C "scripts" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; } + git -C "${SCRIPTS_REPO}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; } fi # Check if we are in the monorepo case where we can use the scripts ref or not - if [ "$(git -C scripts show "${OLD}":.gitmodules 2>/dev/null)" != "" ]; then + if [ "$(git -C "${SCRIPTS_REPO}" show "${OLD}":.gitmodules 2>/dev/null)" != "" ]; then # Old version is not a monorepo but has submodules. # Find the pinned submodule refs because there may be no release tags inside the submodules # Pipe to awk instead of using --object-only for git 2.35 support - OLDREF=$(git -C "scripts" ls-tree "${OLD}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') + OLDREF=$(git -C "${SCRIPTS_REPO}" ls-tree "${OLD}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') # We can't assume anymore that the submodule repo is available under scripts/ - if [ ! -d "${repo}" ]; then - git clone "git@github.com:flatcar/${repo}.git" + if [ ! -d "${OLDREPOPATH}" ]; then + git clone "git@github.com:flatcar/${repo}.git" "${OLDREPOPATH}" fi else OLDPREPEND="sdk_container/src/third_party/${repo}/" - OLDREPOPATH="scripts" + OLDREPOPATH="${SCRIPTS_REPO}" fi - if [ "$(git -C scripts show "${NEW}":.gitmodules 2>/dev/null)" != "" ]; then + if [ "$(git -C "${SCRIPTS_REPO}" show "${NEW}":.gitmodules 2>/dev/null)" != "" ]; then # New version is not a monorepo but has submodules. - NEWREF=$(git -C "scripts" ls-tree "${NEW}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') - if [ ! -d "${repo}" ]; then - git clone "git@github.com:flatcar/${repo}.git" + NEWREF=$(git -C "${SCRIPTS_REPO}" ls-tree "${NEW}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') + if [ ! -d "${NEWREPOPATH}" ]; then + git clone "git@github.com:flatcar/${repo}.git" "${NEWREPOPATH}" fi else NEWPREPEND="sdk_container/src/third_party/${repo}/" - NEWREPOPATH="scripts" + NEWREPOPATH="${SCRIPTS_REPO}" fi fi if [ "${FETCH}" = 1 ]; then @@ -99,12 +105,12 @@ for section in security bugfixes changes updates; do fi # The assumption is that the old ref is really older, so we can assume that old would have submodules while new doesn't have them anymore - if [ "${OLDREPOPATH}" != "${NEWREPOPATH}" ] && [ "${NEWREPOPATH}" = "scripts" ]; then + if [ "${OLDREPOPATH}" != "${NEWREPOPATH}" ] && [ "${NEWREPOPATH}" = "${SCRIPTS_REPO}" ]; then # One patch before the ./checkout helper disappeared we still had submodules - LAST_SUBMOD_SCRIPTS_REF="$(git -C scripts rev-list -n 1 "${NEWREF}" -- checkout)~1" - LAST_SUBMOD_REF=$(git -C scripts ls-tree "${LAST_SUBMOD_SCRIPTS_REF}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') + LAST_SUBMOD_SCRIPTS_REF="$(git -C "${SCRIPTS_REPO}" rev-list -n 1 "${NEWREF}" -- checkout)~1" + LAST_SUBMOD_REF=$(git -C "${SCRIPTS_REPO}" ls-tree "${LAST_SUBMOD_SCRIPTS_REF}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }') # The patch that removed the submodule overrides README has the merge history - FIRST_MONO_REF=$(git -C scripts rev-list -n 1 "${NEWREF}" -- sdk_container/git-override/README.md) + FIRST_MONO_REF=$(git -C "${SCRIPTS_REPO}" rev-list -n 1 "${NEWREF}" -- sdk_container/git-override/README.md) git -C "${OLDREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${LAST_SUBMOD_REF}" -- "${OLDPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${FIRST_MONO_REF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; } else From 90f2a9e49b709464c5dc8d5be54c76efc6303fa3 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 21 Sep 2023 10:51:30 +0200 Subject: [PATCH 2/6] show-changes: Fetch changes from repos once Without the change, the script could be fetching changes twelve times (once for scripts, once for coreos-overlay and once for portage-stable, all repeated four times for four change categories). Just fetch the changes once for each repo if the repo directory exists. This makes the script much faster. --- show-changes | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/show-changes b/show-changes index 298e617..2c44eed 100755 --- a/show-changes +++ b/show-changes @@ -31,6 +31,16 @@ fi echo "_Changes since **${OLD_FMT}**_" +if [[ ${FETCH} = 1 ]]; then + for repo in coreos-overlay portage-stable scripts; do + var_name=${repo//-/_} + var_name="${var_name^^}_REPO" + if [[ -d ${!var_name} ]]; then + git -C "${!var_name}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed for ${repo}" ; exit 1 ; } + fi + done +fi + for section in security bugfixes changes updates; do echo case "${section}" in @@ -61,9 +71,6 @@ for section in security bugfixes changes updates; do OLDPREPEND="" NEWPREPEND="" if [ "${repo}" != "scripts" ]; then - if [ "${FETCH}" = 1 ]; then - git -C "${SCRIPTS_REPO}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; } - fi # Check if we are in the monorepo case where we can use the scripts ref or not if [ "$(git -C "${SCRIPTS_REPO}" show "${OLD}":.gitmodules 2>/dev/null)" != "" ]; then # Old version is not a monorepo but has submodules. @@ -89,10 +96,6 @@ for section in security bugfixes changes updates; do NEWREPOPATH="${SCRIPTS_REPO}" fi fi - if [ "${FETCH}" = 1 ]; then - git -C "${OLDREPOPATH}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; } - git -C "${NEWREPOPATH}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; } - fi if [ "${section}" = "security" ] && [ "${repo}" = "coreos-overlay" ]; then FROM_KERNEL=$(git -C "${OLDREPOPATH}" show "${OLDREF}":"${OLDPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) TO_KERNEL=$(git -C "${NEWREPOPATH}" show "${NEWREF}":"${NEWPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3) From 68c3bbc462a1049f7c6eb6b74868e9dd2320407b Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 21 Sep 2023 11:02:06 +0200 Subject: [PATCH 3/6] show-changes: Fix some shellcheck warnings --- show-changes | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/show-changes b/show-changes index 2c44eed..daa1c43 100755 --- a/show-changes +++ b/show-changes @@ -24,9 +24,9 @@ OLD_FMT="" # what we need in the formatted output is (Alpha|Beta|Stable|LTS XXXX.Y.Z). # The given code transform the given name into the desired output. if [[ $OLD == *"lts"* ]]; then - OLD_FMT=$(echo $OLD | tr "-" " " | tr '[:lower:]' '[:upper:]') + OLD_FMT=$(echo "${OLD}" | tr "-" " " | tr '[:lower:]' '[:upper:]') else - OLD_FMT=$(echo $OLD | tr "-" " " | sed 's/./\U&/') + OLD_FMT=$(echo "${OLD}" | tr "-" " " | sed 's/./\U&/') fi echo "_Changes since **${OLD_FMT}**_" From 6a9f642b84742776b81e41351a9e30b000ed346c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 21 Sep 2023 17:14:49 +0200 Subject: [PATCH 4/6] package-diff: Generalize contents file processing Instead of adding another special case for contents file (as it would be the case by adding support for OEM sysext image content files) just expect all files ending with _contents.txt (or contents_wtd.txt) to be of specific format. --- package-diff | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package-diff b/package-diff index b07ed71..5eababd 100755 --- a/package-diff +++ b/package-diff @@ -78,9 +78,9 @@ function download { } if [[ "${WITHNEWSTUFF}" = '1' ]]; then - if [[ "${FILE}" = flatcar_production_image_contents.txt ]] && \ + if [[ "${FILE}" = *_contents.txt ]] && \ [[ "${CALCSIZE}" = 1 ]]; then - if download flatcar_production_image_disk_usage.txt; then + if download "${FILE%_*}_disk_usage.txt"; then NEWCALCSIZE=1 else echo "Disk usage files missing, falling back to the contents files" @@ -88,7 +88,7 @@ if [[ "${WITHNEWSTUFF}" = '1' ]]; then fi if [[ "${NEWCALCSIZE}" = '0' ]]; then - if [[ "${FILE}" = flatcar_production_image_contents.txt ]] || [[ "${FILE}" = flatcar_production_image_initrd_contents.txt ]] || [[ "${FILE}" = flatcar_developer_container_contents.txt ]]; then + if [[ "${FILE}" = *_contents.txt ]]; then if download "${FILE%.*}_wtd.txt"; then WTD=1 else @@ -192,7 +192,7 @@ function no_hardlinks { if [[ "${NEWCALCSIZE}" = '1' ]]; then : # nothing to do, A and B are already prepared -elif [ "$FILE" = flatcar_production_image_contents.txt ] || [ "$FILE" = flatcar_developer_container_contents.txt ] || [ "$FILE" = flatcar_production_image_initrd_contents.txt ]; then +elif [[ "$FILE" = *_contents.txt ]]; then if [[ "${WTD}" = '0' ]]; then # Cut date and time noise away sed -i 's/....-..-.. ..:.. //g' "$A" "$B" From 11f07d39d528c4304d1426121bdd35f33dc9418e Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 20 Sep 2023 17:45:50 +0200 Subject: [PATCH 5/6] size-change-report: Add support for OEM sysext image content files --- size-change-report.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/size-change-report.sh b/size-change-report.sh index 9156dbd..83188f8 100755 --- a/size-change-report.sh +++ b/size-change-report.sh @@ -18,7 +18,7 @@ ### ### channel: alpha, beta, stable, lts ### board: amd64-usr, arm64-usr -### kind: old, wtd, initrd-old, initrd-wtd +### kind: old, wtd, initrd-old, initrd-wtd, oem-${OEM}-old, oem-${OEM}-wtd ### arch: amd64, arm64 ### ### options: @@ -94,6 +94,7 @@ done function file_from_kind { local spec="${1}"; shift local kind="${1}"; shift + local oemid case "${kind}" in old) echo 'flatcar_production_image_contents.txt' @@ -107,6 +108,18 @@ function file_from_kind { initrd-wtd) echo 'flatcar_production_image_initrd_contents_wtd.txt' ;; + oem-*-old) + oemid=${kind} + oemid=${oemid#oem-} + oemid=${oemid%-old} + echo "oem-${oemid}_contents.txt" + ;; + oem-*-wtd) + oemid=${kind} + oemid=${oemid#oem-} + oemid=${oemid%-wtd} + echo "oem-${oemid}_contents_wtd.txt" + ;; *) fail "Invalid kind '${kind}' in spec '${spec}', should either be 'old' or 'wtd'" ;; @@ -252,6 +265,9 @@ function simplified_kind { initrd-old|initrd-wtd) kind="${kind#initrd-}" ;; + oem-*-old|oem-*-wtd) + kind=${kind##*-} + ;; *) fail "Unexpected kind '${kind}' passed through initial checks." ;; From b5a7a37500d40940ae2648c71b2cf3dcdbdce999 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 22 Sep 2023 11:22:24 +0200 Subject: [PATCH 6/6] size-change-report: Handle the case of unchanged file listings We generate some intermediate files using `git diff` with an option to generate a large number of context lines in order to print all the unchanged lines. But if both files are the same, git diff prints no output at all. In such case, we need to copy one of the files and prepend a space to it to have a "diff" file with unchanged lines only. This fixes some ominous warnings about expecting a bad report. --- size-change-report.sh | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/size-change-report.sh b/size-change-report.sh index 83188f8..f28a0ca 100755 --- a/size-change-report.sh +++ b/size-change-report.sh @@ -351,20 +351,30 @@ if any_missing "${wd}/output" "${wd}/detailed_output" "${wd}/for_cache_key_cache fi # Generate detailed output, without the diff noise and cache keys. File format; # - xgit diff \ - --unified="${lineno}" \ - --no-index \ - -- \ - "${wd}/A.7ba.final-form-no-cache-key" "${wd}/B.7ba.final-form-no-cache-key" | \ - tail --lines +6 >"${wd}/detailed_output" + if git diff \ + --unified="${lineno}" \ + --no-index \ + -- \ + "${wd}/A.7ba.final-form-no-cache-key" "${wd}/B.7ba.final-form-no-cache-key" | \ + tail --lines +6; then + # If both files are the same, diff will show no output. In + # such case, print one of the files prepending an empty space + # (meaning no change in diffesque). + sed -e 's/^/ /' "${wd}/A.7ba.final-form-no-cache-key" + fi >"${wd}/detailed_output" # Generate detailed output, without the diff noise, size and hardlink info. File format; # - xgit diff \ - --unified="${lineno}" \ - --no-index \ - -- \ - "${wd}/A.7bb.final-form-only-cache-key" "${wd}/B.7bb.final-form-only-cache-key" | \ - tail --lines +6 >"${wd}/for_cache_key_cache" + if git diff \ + --unified="${lineno}" \ + --no-index \ + -- \ + "${wd}/A.7bb.final-form-only-cache-key" "${wd}/B.7bb.final-form-only-cache-key" | \ + tail --lines +6; then + # If both files are the same, diff will show no output. In + # such case, print one of the files prepending an empty space + # (meaning no change in diffesque). + sed -e 's/^/ /' "${wd}/A.7bb.final-form-only-cache-key" + fi >"${wd}/for_cache_key_cache" fi # @@ -410,7 +420,7 @@ function get_old_cache_key { gock_cache_key_var_ref="${minus_key_cache[${path}]:-}" if [[ -z "${gock_cache_key_var_ref}" ]]; then - gock_cache_key_var_ref="${space_key_cache[${path}]}" + gock_cache_key_var_ref="${space_key_cache[${path}]:-}" if [[ -z "${gock_cache_key_var_ref}" ]]; then echo "NO OLD CACHE KEY FOUND FOR '${path}', EXPECT A BAD REPORT!" fi @@ -424,7 +434,7 @@ function get_new_cache_key { gnck_cache_key_var_ref="${plus_key_cache[${path}]:-}" if [[ -z "${gnck_cache_key_var_ref}" ]]; then - gnck_cache_key_var_ref="${space_key_cache[${path}]}" + gnck_cache_key_var_ref="${space_key_cache[${path}]:-}" if [[ -z "${gnck_cache_key_var_ref}" ]]; then echo "NO NEW CACHE KEY FOUND FOR '${path}', EXPECT A BAD REPORT!" fi