Skip to content

Commit

Permalink
Source binaries for linux artifacts from docker images (spiffe#4491)
Browse files Browse the repository at this point in the history
* Source binaries for linux artifacts from docker images

This gives us static binaries linked against musl for our release
artifacts, unifying our libc dependency for both docker and non-docker
and simplifying our build tooling.

Since artifact building is now fairly complicated and really only part
of the CI/CD pipeline, got rid of the Makefile target for it.

Fixes: spiffe#4346

Signed-off-by: Andrew Harding <[email protected]>
  • Loading branch information
azdagron authored Sep 13, 2023
1 parent 7a5a528 commit 3ba6756
Show file tree
Hide file tree
Showing 7 changed files with 180 additions and 142 deletions.
22 changes: 12 additions & 10 deletions .github/workflows/pr_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ jobs:
artifacts:
name: artifacts (linux)
runs-on: ubuntu-20.04
needs: [cache-deps]
needs: [cache-deps, images]
timeout-minutes: 30

permissions:
Expand All @@ -132,16 +132,16 @@ jobs:
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Load cached deps
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Load cached build tools
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
- name: Install regctl
uses: regclient/actions/regctl-installer@b6614f5f56245066b533343a85f4109bdc38c8cc # main
- name: Download archived images
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
path: .build
key: ${{ runner.os }}-tools-${{ hashFiles('.go-version','Makefile') }}
name: images
path: .
- name: Expand archived images
run: |
tar xvf images.tar.gz
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive artifacts
Expand Down Expand Up @@ -543,6 +543,8 @@ jobs:
update: true
install: >-
git base-devel mingw-w64-x86_64-toolchain zip unzip
- name: Build binaries
run: make build
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive binaries
Expand Down
23 changes: 13 additions & 10 deletions .github/workflows/release_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ jobs:
artifacts:
name: artifacts (linux)
runs-on: ubuntu-20.04
needs: [cache-deps]
needs: [cache-deps, images]
timeout-minutes: 30

permissions:
contents: read
Expand All @@ -125,16 +126,16 @@ jobs:
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Load cached deps
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Load cached build tools
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
- name: Install regctl
uses: regclient/actions/regctl-installer@b6614f5f56245066b533343a85f4109bdc38c8cc # main
- name: Download archived images
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
path: .build
key: ${{ runner.os }}-tools-${{ hashFiles('.go-version','Makefile') }}
name: images
path: .
- name: Expand archived images
run: |
tar xvf images.tar.gz
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive artifacts
Expand Down Expand Up @@ -531,6 +532,8 @@ jobs:
update: true
install: >-
git base-devel mingw-w64-x86_64-toolchain zip unzip
- name: Build binaries
run: make build
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive binaries
Expand Down
42 changes: 21 additions & 21 deletions .github/workflows/scripts/build_artifacts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,32 @@
set -e

usage() {
echo "usage: ${BASH_SOURCE[0]} <Linux|Windows|macOS>"
echo "usage: ${BASH_SOURCE[0]} <Linux|Windows>"
exit 1
}

[[ $# -eq 1 ]] || usage

os="$1"
declare -a supported_archs
if [[ "${os}" == "Linux" ]] || [[ "${os}" == "macOS" ]]; then
supported_archs=(amd64 arm64)
elif [[ "${os}" == "Windows" ]]; then
supported_archs=(amd64)
else
echo "unrecognized OS: ${os}"
usage
fi
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

export TAG=
if [[ "$GITHUB_REF" =~ ^refs/tags/v[0-9.]+$ ]]; then
# Strip off the leading "v" from the release tag. Release artifacts are
# named just with the version number (e.g. v0.9.3 tag produces
# spire-0.9.3-linux-x64.tar.gz).
TAG="${GITHUB_REF##refs/tags/v}"
# Strip off the leading "v" from the release tag. Release artifacts are
# named just with the version number (e.g. v0.9.3 tag produces
# spire-0.9.3-linux-x64.tar.gz).
TAG="${GITHUB_REF##refs/tags/v}"
fi

# Make references the $TAG environment variable set above
for arch in "${supported_archs[@]}"; do
GOARCH=$arch make artifact
done
[[ $# -eq 1 ]] || usage

os="$1"
case "${os}" in
Linux)
"${SCRIPTDIR}"/build_linux_artifacts.sh
;;
Windows)
"${SCRIPTDIR}"/build_windows_artifacts.sh
;;
*)
echo "Only artifacts for Linux and Windows are supported" 1>&2
usage
;;
esac
83 changes: 83 additions & 0 deletions .github/workflows/scripts/build_linux_artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/bin/bash

set -e

REPODIR=$(git rev-parse --show-toplevel)

TAG=${TAG:-$(git log -n1 --pretty=%h)}
OUTDIR=${OUTDIR:-"${REPODIR}/artifacts"}

TAROPTS=("--owner=root" "--group=root")

TMPDIR=$(mktemp -d)
cleanup() {
rm -rf "${TMPDIR}"
}
trap cleanup EXIT

copy_binary_from_multiarch_tar() {
local arch=$1
local binary=$2
local destdir=$3

local srcpath="/opt/spire/bin/${binary}"
local destpath="${destdir}/${binary}"
local ocidir="ocidir://${TMPDIR}/${arch}/oci/${binary}"
local imagetar="${REPODIR}/${binary}-image.tar"
local platform="linux/${arch}"

echo "Importing multiarch image ${imagetar}..."
regctl image import "${ocidir}" "${imagetar}"

echo "Copying ${srcpath} for platform ${platform}..."
regctl image get-file "${ocidir}" "${srcpath}" "${destpath}" -p "${platform}"

# file does not retain permission bits, so fix up the executable bit.
chmod +x "${destpath}"
}

build_artifact() {
local arch="$1"

local artifact="${OUTDIR}/spire-${TAG}-linux-${arch}-musl.tar.gz"
local checksum="${OUTDIR}/spire-${TAG}-linux-${arch}-musl_sha256sum.txt"

local extras_artifact="${OUTDIR}/spire-extras-${TAG}-linux-${arch}-musl.tar.gz"
local extras_checksum="${OUTDIR}/spire-extras-${TAG}-linux-${arch}-musl_sha256sum.txt"

local tardir="${TMPDIR}/${arch}/tar"
local staging="${tardir}"/spire/spire-${TAG}
local extras_staging="${tardir}"/spire-extras/spire-extras-${TAG}

mkdir -p "${staging}"/bin
mkdir -p "${extras_staging}"/bin
mkdir -p "${OUTDIR}"

echo "Creating \"${artifact}\" and \"${extras_artifact}\""

# Copy in the contents under release/
cp -r "${REPODIR}"/release/posix/spire/* "${staging}"
cp -r "${REPODIR}"/release/posix/spire-extras/* "${extras_staging}"

# Copy in the LICENSE
cp "${REPODIR}"/LICENSE "${staging}"
cp "${REPODIR}"/LICENSE "${extras_staging}"

# Copy in the SPIRE binaries from the docker images:
# 1. import the image from the multiarch tarball into the OCI directory
copy_binary_from_multiarch_tar "$arch" "spire-server" "${staging}/bin"
copy_binary_from_multiarch_tar "$arch" "spire-agent" "${staging}/bin"
copy_binary_from_multiarch_tar "$arch" "oidc-discovery-provider" "${extras_staging}/bin"

# Create the tarballs and checksums
(cd "${tardir}/spire"; tar -cvzf "${artifact}" "${TAROPTS[@]}" -- *)
(cd "${tardir}/spire-extras"; tar -cvzf "${extras_artifact}" "${TAROPTS[@]}" -- *)

(cd "$(dirname "${artifact}")"; shasum -a 256 "$(basename "${artifact}")" > "${checksum}" )
(cd "$(dirname "${extras_artifact}")"; shasum -a 256 "$(basename "${extras_artifact}")" > "${extras_checksum}" )
}

command -v regctl >/dev/null 2>&1 || { echo -e "The regctl cli is required to run this script." >&2 ; exit 1; }

build_artifact amd64
build_artifact arm64
51 changes: 51 additions & 0 deletions .github/workflows/scripts/build_windows_artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/bash

set -e

REPODIR=$(git rev-parse --show-toplevel)
BINDIR="${REPODIR}/bin"

TAG=${TAG:-$(git log -n1 --pretty=%h)}
OUTDIR=${OUTDIR:-"${REPODIR}/artifacts"}

ARCH=amd64

ARTIFACT="${OUTDIR}/spire-${TAG}-windows-${ARCH}.zip"
CHECKSUM="${OUTDIR}/spire-${TAG}-windows-${ARCH}_sha256sum.txt"

EXTRAS_ARTIFACT="${OUTDIR}/spire-extras-${TAG}-windows-${ARCH}.zip"
EXTRAS_CHECKSUM="${OUTDIR}/spire-extras-${TAG}-windows-${ARCH}_sha256sum.txt"

TMPDIR=$(mktemp -d)
cleanup() {
rm -rf "${TMPDIR}"
}
trap cleanup EXIT

STAGING="${TMPDIR}"/spire/spire-${TAG}
EXTRAS_STAGING="${TMPDIR}"/spire-extras/spire-extras-${TAG}
mkdir -p "${STAGING}" "${EXTRAS_STAGING}"

echo "Creating \"${ARTIFACT}\" and \"${EXTRAS_ARTIFACT}\""

# Copy in the contents under release/
cp -r "${REPODIR}"/release/windows/spire/* "${STAGING}"
cp -r "${REPODIR}"/release/windows/spire-extras/* "${EXTRAS_STAGING}"

# Copy in the LICENSE
cp "${REPODIR}"/LICENSE "${STAGING}"
cp "${REPODIR}"/LICENSE "${EXTRAS_STAGING}"

# Copy in the SPIRE binaries
mkdir -p "${STAGING}"/bin "${EXTRAS_STAGING}"/bin
cp "${BINDIR}"/spire-server.exe "${STAGING}"/bin
cp "${BINDIR}"/spire-agent.exe "${STAGING}"/bin
cp "${BINDIR}"/oidc-discovery-provider.exe "${EXTRAS_STAGING}"/bin

mkdir -p "${OUTDIR}"

(cd "${TMPDIR}/spire"; zip -rv "${ARTIFACT}" -- *)
(cd "${TMPDIR}/spire-extras"; zip -rv "${EXTRAS_ARTIFACT}" -- *)

(cd "$(dirname "${ARTIFACT}")"; CertUtil -hashfile "$(basename "${ARTIFACT}")" SHA256 > "${CHECKSUM}")
(cd "$(dirname "${EXTRAS_ARTIFACT}")"; CertUtil -hashfile "$(basename "${EXTRAS_ARTIFACT}")" SHA256 > "${EXTRAS_CHECKSUM}")
10 changes: 0 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ help:
@echo
@echo "$(bold)Build:$(reset)"
@echo " $(cyan)build$(reset) - build all SPIRE binaries (default)"
@echo " $(cyan)artifact$(reset) - build SPIRE tarball artifact"
@echo
@echo "$(bold)Test:$(reset)"
@echo " $(cyan)test$(reset) - run unit tests"
Expand Down Expand Up @@ -318,15 +317,6 @@ endif
integration-windows:
$(E)./test/integration/test-windows.sh $(SUITES)

#############################################################################
# Build Artifact
#############################################################################

.PHONY: artifact

artifact: build
$(E)OUTDIR="$(OUTDIR)" TAG="$(TAG)" ./script/build-artifact.sh

#############################################################################
# Docker Images
#############################################################################
Expand Down
91 changes: 0 additions & 91 deletions script/build-artifact.sh

This file was deleted.

0 comments on commit 3ba6756

Please sign in to comment.