Generate checksums during package (#679)

* Generate checksums during package Signed-off-by: Yael Harel <[email protected]> * Calculate the sha256 while writing to the file Signed-off-by: Yael Harel <[email protected]> * Add the sha256 artifacts to the release Signed-off-by: Yael Harel <[email protected]> * rename linux to linux-x86-64 in the build and release yml files Signed-off-by: Yael Harel <[email protected]> * Add a new line at the end of each sha file Signed-off-by: Yael Harel <[email protected]>
buildpacks · Aug 12, 2021 · 00119d3 · 00119d3
1 parent 8f801a5
commit 00119d3
Show file tree

Hide file tree

Showing 4 changed files with 105 additions and 31 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -118,14 +118,26 @@ jobs:
         with:
           name: lifecycle-linux-x86-64
           path: out/lifecycle-v*+linux.x86-64.tgz
+      - uses: actions/upload-artifact@v2
+        with:
+          name: lifecycle-linux-x86-64-sha256
+          path: out/lifecycle-v*+linux.x86-64.tgz.sha256
       - uses: actions/upload-artifact@v2
         with:
           name: lifecycle-linux-arm64
           path: out/lifecycle-v*+linux.arm64.tgz
+      - uses: actions/upload-artifact@v2
+        with:
+          name: lifecycle-linux-arm64-sha256
+          path: out/lifecycle-v*+linux.arm64.tgz.sha256
       - uses: actions/upload-artifact@v2
         with:
           name: lifecycle-windows-x86-64
           path: out/lifecycle-v*+windows.x86-64.tgz
+      - uses: actions/upload-artifact@v2
+        with:
+          name: lifecycle-windows-x86-64-sha256
+          path: out/lifecycle-v*+windows.x86-64.tgz.sha256
       - uses: azure/docker-login@v1
         if: github.event_name == 'push'
         with:
@@ -142,11 +154,11 @@ jobs:
         run: |
           LIFECYCLE_IMAGE_TAG=$(git describe --always)
           DOCKER_CLI_EXPERIMENTAL=enabled
-          go run ./tools/image/main.go -lifecyclePath ./out/lifecycle-v*+linux.x86-64.tgz -tag buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-linux
+          go run ./tools/image/main.go -lifecyclePath ./out/lifecycle-v*+linux.x86-64.tgz -tag buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-linux-x86-64
           go run ./tools/image/main.go -lifecyclePath ./out/lifecycle-v*+linux.arm64.tgz -tag buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-linux-arm64 -arch arm64
           go run ./tools/image/main.go -lifecyclePath ./out/lifecycle-v*+windows.x86-64.tgz -tag buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-windows -os windows
           docker manifest create buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG} \
-              buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-linux \
+              buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-linux-x86-64 \
               buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-linux-arm64 \
               buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}-windows
           docker manifest push buildpacksio/lifecycle:${LIFECYCLE_IMAGE_TAG}
@@ -244,3 +256,4 @@ jobs:
           $env:LIFECYCLE_PATH="..\lifecycle-v${{ env.LIFECYCLE_VERSION }}+windows.x86-64.tgz"
           $env:LIFECYCLE_IMAGE="buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}"
           make acceptance
+
diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml
@@ -22,7 +22,7 @@ jobs:
             exit 1
           fi
           echo "LIFECYCLE_VERSION=$version" >> $GITHUB_ENV
-      - name: Determine download urls for linux and windows
+      - name: Determine download urls for linux-x86-64 and windows
         id: artifact-urls
         uses: actions/[email protected]
         with:
@@ -72,31 +72,49 @@ jobs:
                   .map(artifact => [artifact.name, artifact.archive_download_url]);
                 let urlMap = new Map();
                 tuples.forEach(function(tuple) {
-                  if (tuple[0].includes("linux")) {
-                    urlMap.set("linux", tuple[1])
+                  if (tuple[0].includes("linux-x86-64")) {
+                    if (tuple[0].includes("sha256")) {
+                      urlMap.set("linux_x86_64_sha256", tuple[1])
+                    } else {
+                      urlMap.set("linux_x86_64", tuple[1])
+                    }
                   }
                   if (tuple[0].includes("windows")) {
-                    urlMap.set("windows", tuple[1])
+                    if (tuple[0].includes("sha256")) {
+                      urlMap.set("windows_sha256", tuple[1])
+                    } else {
+                      urlMap.set("windows", tuple[1])
+                    }
                   }
                 })
                 if (urlMap.size === 0) {
                   throw "no artifacts found"
                 }
-                if (urlMap.size != 2) {
-                  throw "there should be exactly two artifacts"
+                if (urlMap.size != 4) {
+                  throw "there should be exactly four artifacts"
                 }
                 return Object.fromEntries(urlMap.entries())
               })
-      - name: Download linux artifact
+      - name: Download linux-x86-64 artifact
         run: |
-          url=$(echo '${{ steps.artifact-urls.outputs.result }}' | jq -r .linux )
+          url=$(echo '${{ steps.artifact-urls.outputs.result }}' | jq -r .linux_x86_64 )
           curl -sL -w 'RESP_CODE:%{response_code}\n' \
             --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
-            -o artifact-linux.zip $url
-          mkdir artifact-linux
-          unzip -d artifact-linux artifact-linux.zip
-          lifecycle_path=$(ls artifact-linux/lifecycle-*linux.x86-64.tgz)
-          echo "ARTIFACT_LINUX_PATH=$PWD/$lifecycle_path" >> $GITHUB_ENV
+            -o artifact-linux-x86-64.zip $url
+          mkdir artifact-linux-x86-64
+          unzip -d artifact-linux-x86-64 artifact-linux-x86-64.zip
+          lifecycle_path=$(ls artifact-linux-x86-64/lifecycle-*linux.x86-64.tgz)
+          echo "ARTIFACT_LINUX_X86_64_PATH=$PWD/$lifecycle_path" >> $GITHUB_ENV
+      - name: Download linux-x86-64-sha256 artifact
+        run: |
+          url=$(echo '${{ steps.artifact-urls.outputs.result }}' | jq -r .linux_x86_64_sha256 )
+          curl -sL -w 'RESP_CODE:%{response_code}\n' \
+            --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+            -o artifact-linux-x86-64-sha256.zip $url
+          mkdir artifact-linux-x86-64-sha256
+          unzip -d artifact-linux-x86-64-sha256 artifact-linux-x86-64-sha256.zip
+          lifecycle_path=$(ls artifact-linux-x86-64-sha256/lifecycle-*linux.x86-64.tgz.sha256)
+          echo "ARTIFACT_LINUX_X86_64_SHA_PATH=$PWD/$lifecycle_path" >> $GITHUB_ENV
       - name: Download windows artifact
         run: |
           url=$(echo '${{ steps.artifact-urls.outputs.result }}' | jq -r .windows )
@@ -107,6 +125,16 @@ jobs:
           unzip -d artifact-windows artifact-windows.zip
           lifecycle_path=$(ls artifact-windows/lifecycle-*windows.x86-64.tgz)
           echo "ARTIFACT_WINDOWS_PATH=$PWD/$lifecycle_path" >> $GITHUB_ENV
+      - name: Download windows-sha256 artifact
+        run: |
+          url=$(echo '${{ steps.artifact-urls.outputs.result }}' | jq -r .windows_sha256 )
+          curl -sL -w 'RESP_CODE:%{response_code}\n' \
+            --header 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+            -o artifact-windows-sha256.zip $url
+          mkdir artifact-windows-sha256
+          unzip -d artifact-windows-sha256 artifact-windows-sha256.zip
+          lifecycle_path=$(ls artifact-windows-sha256/lifecycle-*windows.x86-64.tgz.sha256)
+          echo "ARTIFACT_WINDOWS_SHA_PATH=$PWD/$lifecycle_path" >> $GITHUB_ENV
       - name: Set pre-release kind
         if: contains(env.LIFECYCLE_VERSION, 'rc') # e.g., 0.99.0-rc.1
         run: |
@@ -138,7 +166,8 @@ jobs:
         if: contains(env.LIFECYCLE_VERSION, 'rc') # e.g., 0.99.0-rc.1
         run: |
           gh release create v${{ env.LIFECYCLE_VERSION }} \
-            ${{ env.ARTIFACT_LINUX_PATH }} ${{ env.ARTIFACT_WINDOWS_PATH }} \
+            ${{ env.ARTIFACT_LINUX_X86_64_PATH }} ${{ env.ARTIFACT_LINUX_X86_64_SHA_PATH }} \
+            ${{ env.ARTIFACT_WINDOWS_PATH }} ${{ env.ARTIFACT_WINDOWS_SHA_PATH }} \
             --draft \
             --notes-file body.txt \
             --prerelease \
@@ -150,7 +179,8 @@ jobs:
         if: "!contains(env.LIFECYCLE_VERSION, 'rc')"
         run: |
           gh release create v${{ env.LIFECYCLE_VERSION }} \
-            ${{ env.ARTIFACT_LINUX_PATH }} ${{ env.ARTIFACT_WINDOWS_PATH }} \
+            ${{ env.ARTIFACT_LINUX_X86_64_PATH }} ${{ env.ARTIFACT_LINUX_X86_64_SHA_PATH }} \
+            ${{ env.ARTIFACT_WINDOWS_PATH }} ${{ env.ARTIFACT_WINDOWS_SHA_PATH }} \
             --draft \
             --notes-file body.txt \
             --target $GITHUB_REF \

diff --git a/.github/workflows/post-release.yml b/.github/workflows/post-release.yml
@@ -6,7 +6,7 @@ on:
       - published # trigger for releases and pre-releases
 
 jobs:
-  retag-lifecycle-images-linux:
+  retag-lifecycle-images-linux-x86-64:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -20,15 +20,15 @@ jobs:
           echo "LIFECYCLE_IMAGE_TAG=$(git describe --always)" >> $GITHUB_ENV
       - name: Retag lifecycle images - semver
         run: |
-          docker pull buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux
-          docker image tag buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-linux
-          docker push buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-linux
+          docker pull buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux-x86-64
+          docker image tag buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux-x86-64 buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-linux-x86-64
+          docker push buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-linux-x86-64
       - name: Retag lifecycle images - latest
         if: "!contains(env.LIFECYCLE_VERSION, 'rc')"
         run: |
-          docker pull buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux
-          docker image tag buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux buildpacksio/lifecycle:latest-linux
-          docker push buildpacksio/lifecycle:latest-linux
+          docker pull buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux-x86-64
+          docker image tag buildpacksio/lifecycle:${{ env.LIFECYCLE_IMAGE_TAG }}-linux-x86-64 buildpacksio/lifecycle:latest-linux-x86-64
+          docker push buildpacksio/lifecycle:latest-linux-x86-64
   retag-lifecycle-images-windows:
     runs-on: windows-latest
     steps:
@@ -54,7 +54,7 @@ jobs:
           docker push buildpacksio/lifecycle:latest-windows
   create-manifest-lists:
     runs-on: ubuntu-latest
-    needs: [retag-lifecycle-images-linux, retag-lifecycle-images-windows]
+    needs: [retag-lifecycle-images-linux-x86-64, retag-lifecycle-images-windows]
     steps:
       - uses: azure/docker-login@v1
         with:
@@ -65,10 +65,10 @@ jobs:
           echo "LIFECYCLE_VERSION=$(echo ${{ github.event.release.tag_name }} | cut -d "v" -f2)" >> $GITHUB_ENV
       - name: Create lifecycle image manifest lists - semver
         run: |
-          DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }} buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-linux buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-windows
+          DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }} buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-linux-x86-64 buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}-windows
           DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push buildpacksio/lifecycle:${{ env.LIFECYCLE_VERSION }}
       - name: Create lifecycle image manifest lists - latest
         if: "!contains(env.LIFECYCLE_VERSION, 'rc')"
         run: |
-          DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create buildpacksio/lifecycle:latest buildpacksio/lifecycle:latest-linux buildpacksio/lifecycle:latest-windows
+          DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create buildpacksio/lifecycle:latest buildpacksio/lifecycle:latest-linux-x86-64 buildpacksio/lifecycle:latest-windows
           DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push buildpacksio/lifecycle:latest
diff --git a/tools/packager/main.go b/tools/packager/main.go
@@ -4,8 +4,11 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"crypto/sha256"
+	"encoding/hex"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -47,15 +50,15 @@ func doPackage() error {
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Failed to open -archivePath %s", archivePath))
 	}
-	defer f.Close()
 
-	zw := gzip.NewWriter(f)
-	defer zw.Close()
+	hasher := sha256.New()
+	mw := io.MultiWriter(f, hasher) // calculate the sha256 while writing to f
+
+	zw := gzip.NewWriter(mw)
 
 	tw := archive.NewNormalizingTarWriter(tar.NewWriter(zw))
 	tw.WithUID(0)
 	tw.WithGID(0)
-	defer tw.Close()
 
 	templateContents, err := ioutil.ReadFile(descriptorPath)
 	if err != nil {
@@ -112,6 +115,34 @@ func doPackage() error {
 		return errors.Wrap(err, "Failed to write dir to archive")
 	}
 
+	err = tw.Close()
+	if err != nil {
+		return errors.Wrap(err, "Failed to close tar writer")
+	}
+
+	err = zw.Close()
+	if err != nil {
+		return errors.Wrap(err, "Failed to close gzip writer")
+	}
+
+	err = f.Close()
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Failed to close -archivePath %s", archivePath))
+	}
+
+	hashFileName := archivePath + ".sha256"
+	hashFile, err := os.OpenFile(hashFileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0777)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Failed to open %s", hashFileName))
+	}
+	defer hashFile.Close()
+
+	sha := hex.EncodeToString(hasher.Sum(nil))
+	_, err = hashFile.Write([]byte(sha + "\n"))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Failed to write sha256:%s to %s", sha, hashFileName))
+	}
+
 	return nil
 }