Merge pull request #849 from justaugustus/shas

Improve SHA generation and enable SHA256SUMS/SHA512SUMS upload to GitHub

anago

@@ -1079,9 +1079,15 @@ update_github_release () {
   local release_verb="Posting"
   local prerelease="true"
   local draft="true"
-  local tarball="$TREE_ROOT/_output-$RELEASE_VERSION_PRIME/gcs-stage"
-        tarball+="/$RELEASE_VERSION_PRIME/kubernetes.tar.gz"
-  local sha_hash=$(common::sha $tarball 512)
+  local staging_dir="${TREE_ROOT}/_output-${RELEASE_VERSION_PRIME}/gcs-stage/${RELEASE_VERSION_PRIME}"
+  local tarball="${staging_dir}/kubernetes.tar.gz"
+  local sha256_hash
+  local sha512_hash
+  local sha256sums_file="${staging_dir}/SHA256SUMS"
+  local sha512sums_file="${staging_dir}/SHA512SUMS"
+
+  sha256_hash=$(common::sha "$tarball" 256)
+  sha512_hash=$(common::sha "$tarball" 512)
 
   ((FLAGS_official)) && prerelease="false"
   if ((FLAGS_nomock)); then
@@ -1132,7 +1138,7 @@ update_github_release () {
     "tag_name": "'$RELEASE_VERSION_PRIME'",
     "target_commitish": "'$RELEASE_BRANCH'",
     "name": "'$RELEASE_VERSION_PRIME'",
-    "body": "See [kubernetes-announce@](https://groups.google.com/forum/#!forum/kubernetes-announce) and ['$CHANGELOG_FILE']('$changelog_url'#'${RELEASE_VERSION_PRIME//\./}') for details.\n\nSHA512 for `kubernetes.tar.gz`: `'$sha_hash'`\n\nAdditional binary downloads are linked in the ['$CHANGELOG_FILE']('$changelog_url'#downloads-for-'${RELEASE_VERSION_PRIME//\./}').",
+    "body": "See [kubernetes-announce@](https://groups.google.com/forum/#!forum/kubernetes-announce) and ['$CHANGELOG_FILE']('$changelog_url'#'${RELEASE_VERSION_PRIME//\./}') for details.\n\nSHA256 for `kubernetes.tar.gz`: `'${sha256_hash}'`\n\nSHA512 for `kubernetes.tar.gz`: `'${sha512_hash}'`\n\nAdditional binary downloads are linked in the ['$CHANGELOG_FILE']('$changelog_url'#downloads-for-'${RELEASE_VERSION_PRIME//\./}').",
     "draft": '$draft',
     "prerelease": '$prerelease'
     }' |jq -r '.id')
@@ -1144,16 +1150,36 @@ update_github_release () {
     return 1
   fi
 
+  local assets_upload_url="${K8S_GITHUB_API/api\./uploads\.}/releases/${release_id}/assets"
+
   # publish binary
   logecho -n "Uploading binary to github: "
   if $GHCURL -H "Content-Type:application/x-compressed" \
-   --data-binary @$tarball \
-   "${K8S_GITHUB_API/api\./uploads\.}/releases/$release_id/assets?name=${tarball##*/}"; then
-    logecho $OK
+   --data-binary "@$tarball" \
+   "${assets_upload_url}?name=${tarball##*/}"; then
+    logecho "$OK"
   else
-    logecho $FAILED
+    logecho "$FAILED"
   fi
 
+  local file
+
+  # Upload SHA sums to GitHub
+  logecho -n "Uploading SHA sums to github: "
+  for file in "$sha256sums_file" "$sha512sums_file"; do
+    if [[ ! -f $file ]]; then
+      logecho -n "Unable to find $file. Skipping..."
+    else
+      if $GHCURL -H "Content-Type:application/octet-stream" \
+      --data-binary @$file \
+      "${assets_upload_url}?name=${file##*/}"; then
+        logecho "$OK"
+      else
+        logecho "$FAILED"
+      fi
+    fi
+  done
+
   if $draft; then
     logecho
     logecho "$ATTENTION: A draft release of $RELEASE_VERSION_PRIME was" \

lib/common.sh

@@ -638,31 +638,29 @@ common::argc_validate () {
   fi
 }
 
-
 ###############################################################################
-# Get the md5 hash of a file
+# Get the SHA hash of a file
 # @param file - The file
-# @print the md5 hash
-common::md5 () {
-  local file=$1
-
-  if which md5 >/dev/null 2>&1; then
-    md5 -q "$1"
-  else
-    md5sum "$file" | awk '{print $1}'
-  fi
-}
-
-###############################################################################
-# Get the sha1 hash of a file
-# @param file - The file
-# @param algo - Algorithm 1 (default), 224, 256, 384, 512, 512224, 512256
+# @param algo - Algorithm 1, 224, 256 (default), 384, 512, 512224, 512256
+# output_type - Specifies output for the SHA:
+#               hash (default): outputs just the SHA
+#               file: outputs the SHA, two spaces, and the basename of the file
 # @print the sha hash
 common::sha () {
   local file=$1
-  local algo=${2:-1}
+  local algo=${2:-256}
+  local output_type=${3:-hash}
+  local shasum_output
+
+  which shasum >/dev/null 2>&1 || return 1
 
-  which shasum >/dev/null 2>&1 && LANG=C shasum -a$algo $file | awk '{print $1}'
+  shasum_output=$(shasum -a"$algo" "$file")
+
+  if [[ "$output_type" != "full" ]]; then
+    echo "$shasum_output" | awk '{print $1}'
+  else
+    echo "$shasum_output" | sed 's/  .*\//  /'
+  fi
 }
 
 ###############################################################################

lib/common_test.sh

@@ -17,9 +17,11 @@
 # common.sh unit tests
 #
 # shellcheck source=./lib/testing.sh
+# shellcheck disable=SC1091
 source "$(dirname "$(readlink -ne "${BASH_SOURCE[0]}")")/testing.sh"
 
 # shellcheck source=./lib/common.sh
+# shellcheck disable=SC1091
 source "$(dirname "$(readlink -ne "${BASH_SOURCE[0]}")")/common.sh"
 readonly TESTDATA="$( cd "$(dirname "$0")" && pwd )/testdata"
 
@@ -28,28 +30,67 @@ readonly TESTDATA="$( cd "$(dirname "$0")" && pwd )/testdata"
 #       - add `set -o nounset`
 #
 #       We can do that when all the things we source do not rely on unset
-#       varaibales and ignoring errors. This will require quite some
+#       variables and ignoring errors. This will require quite some
 #       refactoring, so this is the best we can do for now.
 set -o errexit
 set -o pipefail
 
 TEST_run_stateful() {
-  tmpDir="$( mktemp -d )"
-  trap 'rm -rf -- "$tmpDir"' EXIT
-
-  # override some vars and func to not clutter output
-  common::timestamp() { :; }
-  # shellcheck disable=SC2034
-  PROGSTATE="${tmpDir}/whats-a-progstate-even.txt" \
-    LOGFILE="${tmpDir}/some-log-file.log" \
-    HR='' \
-    TPUT[BOLD]='' \
-    TPUT[OFF]=''
+  test_scaffold
 
   assert_equal_content \
     <( common::run_stateful --strip-args 'printf %s\n%s arg1 arg2' ) \
     <( echo -e "\n\nprintf\n\n\narg1\narg2" ) \
     "passing command and arguments"
 }
 
+TEST_generate_sha() {
+  test_scaffold
+
+  local expected_dir
+  local testfilename1
+  local testfilename2
+  local testfile1
+  local testfile2
+
+  expected_dir="$TESTDATA/common/shas"
+
+  testfilename1="shafile1.txt"
+  testfilename2="shafile2.txt"
+
+  # shellcheck disable=SC2154
+  testfile1="${tmp_dir}/${testfilename1}"
+  testfile2="${tmp_dir}/${testfilename2}"
+
+  # shellcheck disable=SC2154
+  echo -n "This is the first file to test the generation of SHA hashes." > "$testfile1"
+
+  # shellcheck disable=SC2154
+  echo -n "This is the second file to test the generation of SHA hashes." > "$testfile2"
+
+  find "$tmp_dir" -type f | while read -r path; do
+    for bits in "256" "512"; do
+      sum="$(common::sha "${path}" "${bits}" "full")" || return 1
+      echo "$sum" > "${path}.sha${bits}"
+      echo "$sum" >> "${tmp_dir}/SHA${bits}SUMS"
+    done
+  done
+
+  for sha_file in "$testfile1" "$testfile2"; do
+    for bits in "256" "512"; do
+      assert_equal_content \
+        "${tmp_dir}/$(basename "${sha_file}").sha${bits}" \
+        "${expected_dir}/$(basename "${sha_file}").sha${bits}" \
+        "Validated sha${bits} hash for ${sha_file}"
+    done
+  done
+
+  for bits in "256" "512"; do
+    assert_equal_content \
+      "$tmp_dir/SHA${bits}SUMS" \
+      "$expected_dir/SHA${bits}SUMS" \
+      "Validated SHA${bits}SUMS"
+  done
+}
+
 test_main "$@"

lib/releaselib.sh

@@ -487,11 +487,11 @@ release::gcs::check_release_bucket() {
 }
 
 ###############################################################################
-# Create a unique bucket name for releasing Kube and make sure it exists.
+# Creates a tarball for upload to a GCS staging directory.
 # @param gcs_stage - the staging directory
 # @param source and destination arguments
-# @return 1 if tar fails
-release::gcs::stage_and_hash() {
+# @return 1 if tar directory or tarball creation fails
+release::gcs::prepare_tarball() {
   local gcs_stage=$1
   shift
   local src
@@ -602,7 +602,7 @@ release::gcs::locally_stage_release_artifacts() {
 
   # Stage everything in release directory
   logecho "- Staging locally to ${gcs_stage##$build_output/}..."
-  release::gcs::stage_and_hash $gcs_stage $release_tars/* . || return 1
+  release::gcs::prepare_tarball $gcs_stage $release_tars/* . || return 1
 
   if [[ "$release_kind" == "kubernetes" ]]; then
     local gce_path=$release_stage/full/kubernetes/cluster/gce
@@ -621,20 +621,20 @@ release::gcs::locally_stage_release_artifacts() {
     [[ -f $gce_path/configure-vm.sh ]] \
      && configure_vm="$gce_path/configure-vm.sh"
 
-    release::gcs::stage_and_hash $gcs_stage $configure_vm extra/gce \
+    release::gcs::prepare_tarball $gcs_stage $configure_vm extra/gce \
       || return 1
-    release::gcs::stage_and_hash $gcs_stage $gci_path/node.yaml extra/gce \
+    release::gcs::prepare_tarball $gcs_stage $gci_path/node.yaml extra/gce \
       || return 1
-    release::gcs::stage_and_hash $gcs_stage $gci_path/master.yaml extra/gce \
+    release::gcs::prepare_tarball $gcs_stage $gci_path/master.yaml extra/gce \
       || return 1
-    release::gcs::stage_and_hash $gcs_stage $gci_path/configure.sh extra/gce \
+    release::gcs::prepare_tarball $gcs_stage $gci_path/configure.sh extra/gce \
       || return 1
 
     # shutdown.sh was introduced starting from v1.11 to make Preemptible COS nodes
     # on GCP not reboot immediately when terminated. Avoid including it in the release
     # bundle if it is not found (for backwards compatibility).
     if [[ -f $gci_path/shutdown.sh ]]; then
-      release::gcs::stage_and_hash $gcs_stage $gci_path/shutdown.sh extra/gce \
+      release::gcs::prepare_tarball $gcs_stage $gci_path/shutdown.sh extra/gce \
        || return 1
     fi
 
@@ -643,15 +643,15 @@ release::gcs::locally_stage_release_artifacts() {
     windows_local_path=$gce_path/windows
     windows_gcs_path=extra/gce/windows
     if [[ -d $windows_local_path ]]; then
-      release::gcs::stage_and_hash $gcs_stage $windows_local_path/configure.ps1 $windows_gcs_path \
+      release::gcs::prepare_tarball $gcs_stage $windows_local_path/configure.ps1 $windows_gcs_path \
         || return 1
-      release::gcs::stage_and_hash $gcs_stage $windows_local_path/common.psm1 $windows_gcs_path \
+      release::gcs::prepare_tarball $gcs_stage $windows_local_path/common.psm1 $windows_gcs_path \
         || return 1
-      release::gcs::stage_and_hash $gcs_stage $windows_local_path/k8s-node-setup.psm1 $windows_gcs_path \
+      release::gcs::prepare_tarball $gcs_stage $windows_local_path/k8s-node-setup.psm1 $windows_gcs_path \
         || return 1
-      release::gcs::stage_and_hash $gcs_stage $windows_local_path/testonly/install-ssh.psm1 $windows_gcs_path \
+      release::gcs::prepare_tarball $gcs_stage $windows_local_path/testonly/install-ssh.psm1 $windows_gcs_path \
         || return 1
-      release::gcs::stage_and_hash $gcs_stage $windows_local_path/testonly/user-profile.psm1 $windows_gcs_path \
+      release::gcs::prepare_tarball $gcs_stage $windows_local_path/testonly/user-profile.psm1 $windows_gcs_path \
         || return 1
     fi
 
@@ -667,25 +667,32 @@ release::gcs::locally_stage_release_artifacts() {
     if [[ -d "$release_stage/server/$platform" ]]; then
       src="$release_stage/server/$platform/$release_kind/server/bin/*"
     fi
-    release::gcs::stage_and_hash $gcs_stage "$src" "$dst" || return 1
+    release::gcs::prepare_tarball $gcs_stage "$src" "$dst" || return 1
 
     # Upload node binaries if they exist and this isn't a 'server' platform.
     if [[ ! -d "$release_stage/server/$platform" ]]; then
       if [[ -d "$release_stage/node/$platform" ]]; then
         src="$release_stage/node/$platform/$release_kind/node/bin/*"
-        release::gcs::stage_and_hash $gcs_stage "$src" "$dst" || return 1
+        release::gcs::prepare_tarball $gcs_stage "$src" "$dst" || return 1
       fi
     fi
   done
 
   logecho "- Hashing files in ${gcs_stage##$build_output/}..."
-  find $gcs_stage -type f | while read path; do
-    common::md5 $path > "$path.md5" || return 1
-    common::sha $path 1 > "$path.sha1" || return 1
-    common::sha $path 512 > "$path.sha512" || return 1
+  find "$gcs_stage" -type f | while read -r path; do
+    for bits in "256" "512"; do
+      sum="$(common::sha "$path" "$bits" "full")" || return 1
+      echo "$sum" > "${path}.sha${bits}"
+    done
   done
-}
 
+  logecho "- Writing artifact hashes to SHA256SUMS/SHA512SUMS files..."
+  for bits in "256" "512"; do
+    for sha_file in "${gcs_stage}"/*.sha"${bits}"; do
+      cat "$sha_file" >> "${gcs_stage}/SHA${bits}SUMS"
+    done
+  done
+}
 
 ###############################################################################
 # Publish a new version, (latest or stable,) but only if the

lib/testdata/common/shas/SHA256SUMS

@@ -0,0 +1,2 @@
+25073faf6553b2ff6c6cfc66c3b52f6bbf7d0f083fd2d171174f00ae70cd7eb7  shafile1.txt
+fd3d841e433b61936dc4ed38be28a577cadee21c4181ca2364b43416bb971f04  shafile2.txt

lib/testdata/common/shas/SHA512SUMS

@@ -0,0 +1,2 @@
+e414bed12a1fb345620d96f4850652ea4e0fc9e9630451ed8bbd7cf8ae78bb0d78a53d36c847ea4e7bf035d2c553717f1456447d0c088b4ba56d9992c8840a52  shafile1.txt
+3bd0e80f05efc814afc76c33fbc3e569de780de40e7f74ca7b67ffcbb58bee595e0e1b5047196a488a101ffb6f77cbee5afa029373d59285e5f5ffb1ff4b2477  shafile2.txt

lib/testdata/common/shas/shafile1.txt

@@ -0,0 +1 @@
+This is the first file to test the generation of SHA hashes.

lib/testdata/common/shas/shafile1.txt.sha256

@@ -0,0 +1 @@
+25073faf6553b2ff6c6cfc66c3b52f6bbf7d0f083fd2d171174f00ae70cd7eb7  shafile1.txt

lib/testdata/common/shas/shafile1.txt.sha512

@@ -0,0 +1 @@
+e414bed12a1fb345620d96f4850652ea4e0fc9e9630451ed8bbd7cf8ae78bb0d78a53d36c847ea4e7bf035d2c553717f1456447d0c088b4ba56d9992c8840a52  shafile1.txt

lib/testdata/common/shas/shafile2.txt

@@ -0,0 +1 @@
+This is the second file to test the generation of SHA hashes.

lib/testdata/common/shas/shafile2.txt.sha256

@@ -0,0 +1 @@
+fd3d841e433b61936dc4ed38be28a577cadee21c4181ca2364b43416bb971f04  shafile2.txt

lib/testdata/common/shas/shafile2.txt.sha512

@@ -0,0 +1 @@
+3bd0e80f05efc814afc76c33fbc3e569de780de40e7f74ca7b67ffcbb58bee595e0e1b5047196a488a101ffb6f77cbee5afa029373d59285e5f5ffb1ff4b2477  shafile2.txt

lib/testing.sh

@@ -35,6 +35,21 @@ test_main() {
   done
 }
 
+test_scaffold() {
+  func_name="${FUNCNAME[1]}"
+  tmp_dir="$(mktemp -d "${func_name}.XXXXXX")"
+  trap 'rm -rf -- "$tmp_dir"' EXIT
+
+  # override some vars and func to not clutter output
+  common::timestamp() { :; }
+  # shellcheck disable=SC2034
+  PROGSTATE="${tmp_dir}/${func_name}-state.txt" \
+    LOGFILE="${tmp_dir}/${func_name}.log" \
+    HR='' \
+    TPUT[BOLD]='' \
+    TPUT[OFF]=''
+}
+
 assert_equal_content() {
   local actual_file="$1"
   local expected_file="$2"