Skip to content

Commit

Permalink
Merge pull request #9326 from habitat-sh/aarch64-linux-cicd
Browse files Browse the repository at this point in the history
Pipeline setup for aarch64-linux
  • Loading branch information
sajjaphani authored Nov 19, 2024
2 parents 8605fc3 + 5866c8c commit c511d64
Show file tree
Hide file tree
Showing 28 changed files with 739 additions and 49 deletions.
88 changes: 82 additions & 6 deletions .expeditor/release_habitat.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,6 @@ steps:
# Release!
#######################################################################

- label: "[:linux: build hab (ARM)]"
command:
- .expeditor/scripts/release_habitat/package_and_upload_arm_binary.sh
agents:
queue: docker-linux-arm64

- label: "[:linux: build hab]"
command:
- .expeditor/scripts/release_habitat/build_component.sh hab
Expand All @@ -38,6 +32,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh hab

- label: "[:linux: :two: build hab]"
command:
- .expeditor/scripts/release_habitat/build_component.sh hab
Expand Down Expand Up @@ -111,6 +114,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab-plan-build]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh plan-build

- label: "[:linux: :two: build hab-plan-build]"
command:
- .expeditor/scripts/release_habitat/build_component.sh plan-build
Expand Down Expand Up @@ -146,6 +158,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab-backline]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh backline

- label: "[:linux: :two: build hab-backline]"
command:
- .expeditor/scripts/release_habitat/build_component.sh backline
Expand All @@ -170,6 +191,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab-studio]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh studio

- label: "[:linux: :two: build hab-studio]"
command:
- .expeditor/scripts/release_habitat/build_component.sh studio
Expand Down Expand Up @@ -208,6 +238,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 launcher]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh launcher

- label: "[:linux: :two: build launcher]"
command:
- .expeditor/scripts/release_habitat/build_component.sh launcher
Expand Down Expand Up @@ -241,6 +280,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab-sup]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh sup

- label: "[:linux: :two: build hab-sup]"
command:
- .expeditor/scripts/release_habitat/build_component.sh sup
Expand Down Expand Up @@ -274,6 +322,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab-pkg-export-container]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-container

- label: "[:windows: build hab-pkg-export-container]"
command:
- powershell .expeditor/scripts/release_habitat/build_component.ps1 pkg-export-container
Expand All @@ -295,6 +352,15 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: build aarch64 hab-pkg-export-tar]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar

- label: "[:linux: :two: build hab-pkg-export-tar]"
command:
- .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar
Expand Down Expand Up @@ -352,6 +418,16 @@ steps:
environment:
- BUILD_PKG_TARGET=x86_64-linux

- label: "[:linux: upload hab aarch64 binary]"
agents:
queue: default-privileged-aarch64
env:
HAB_FALLBACK_CHANNEL: "LTS-2024"
BUILD_PKG_TARGET: "aarch64-linux"
command:
- sudo -E .expeditor/scripts/release_habitat/package_and_upload_binary.sh
if: build.creator.name == 'Chef Expeditor' || build.env("UPLOAD_AND_PROMOTE") == 'true'

- label: "[:linux: :two: upload hab binary]"
command:
- .expeditor/scripts/release_habitat/package_and_upload_binary.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ channel=$(get_release_channel)
echo "--- Channel: $channel - bldr url: $HAB_BLDR_URL"

declare -g hab_binary
install_release_channel_hab_binary "x86_64-linux"
install_release_channel_hab_binary "$BUILD_PKG_TARGET"

import_gpg_keys

Expand Down
3 changes: 2 additions & 1 deletion .expeditor/scripts/shared.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ curlbash_hab() {
sudo rm -rf /hab/pkgs/core/hab/0.82.0 || \
rm -rf /hab/pkgs/core/hab/0.82.0
fi

curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$pkg_target"
case "${pkg_target}" in
x86_64-linux | x86_64-linux-kernel2)
x86_64-linux | aarch64-linux | x86_64-linux-kernel2)
hab_binary="/bin/hab"
;;
x86_64-darwin)
Expand Down
30 changes: 30 additions & 0 deletions .expeditor/scripts/verify/build_package-aarch64.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash

set -eou pipefail

package_path=${1?package_path argument required}

# Install hab from a temporarily uploaded aarch64 package
curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$BUILD_PKG_TARGET" -c "$HAB_FALLBACK_CHANNEL" -v 1.6.1178

# Since we are only verifying we don't have build failures, make everything
# temp!
export HAB_ORIGIN
HAB_ORIGIN=core
# let's make a selfcontained tempdir for this job
export JOB_TEMP_ROOT
JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX)
export HAB_CACHE_KEY_PATH
HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys"

export HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL
HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL="$HAB_FALLBACK_CHANNEL"
export HAB_PREFER_LOCAL_CHEF_DEPS="true"

echo "--- :key: Generating temporary origin key"
hab origin key generate "$HAB_ORIGIN"
echo "--- :hab: Running hab pkg build for $package_path"
# Install the temporarily built hab-studio.
# Once hab is released in the LTS channel, this step may no longer be required.
hab pkg install core/hab-studio -c "$HAB_FALLBACK_CHANNEL"
hab pkg build "$package_path"
78 changes: 78 additions & 0 deletions .expeditor/scripts/verify/run_cargo_test-aarch64.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/bin/bash

set -eou pipefail

# Install hab from a temporarily uploaded aarch64 package
curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t aarch64-linux -c lts24-aarch64-linux -v 1.6.1178

# shellcheck source=.expeditor/scripts/shared.sh
source .expeditor/scripts/verify/shared.sh

if [[ ${1:-"--"} = "--" ]]; then
scope="habitat workspace"
else
component="$1"
shift
scope="$component"
fi

toolchain=$(get_toolchain)

sudo -E hab pkg install core/zeromq
sudo -E hab pkg install core/protobuf
sudo -E hab pkg install core/rust/"$toolchain"
sudo -E hab pkg install core/xz
sudo -E hab pkg install core/coreutils
sudo -E hab pkg install core/openssl
sudo -E hab pkg install core/perl
sudo -E hab pkg install core/make

export OPENSSL_DIR
OPENSSL_DIR="$(hab pkg path core/openssl)/bin"
export OPENSSL_LIB_DIR
OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib"

export LIBZMQ_PREFIX
LIBZMQ_PREFIX=$(hab pkg path core/zeromq)
# now include zeromq and gcc so they exist in the runtime library path when cargo test is run
export LD_LIBRARY_PATH
LD_LIBRARY_PATH="$(hab pkg path core/gcc-base)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/xz)/lib:$(hab pkg path core/openssl)/lib"

export LIBRARY_PATH
LIBRARY_PATH="$(hab pkg path core/xz)/lib"

export PROTOC_NO_VENDOR=1
export PROTOC
PROTOC=$(hab pkg path core/protobuf)/bin/protoc

_oldPth=$PATH
_pth="$(hab pkg path core/coreutils)/bin:$(hab pkg path core/openssl)/bin:$(hab pkg path core/perl)/bin:$(hab pkg path core/make)/bin"
eval "$(hab pkg env core/rust/"$toolchain"):$PATH"
export PATH="$PATH:$_pth:$_oldPth"

# Set testing filesystem root
export FS_ROOT
FS_ROOT=$(mktemp -d /tmp/testing-fs-root-XXXXXX)

export RUST_BACKTRACE=1

# Build the all the hab binaries so that we can run integration tests
if [[ "$scope" == "sup" ]]; then
cargo build
fi

echo "--- Running cargo test with scope '$scope' and args '$*'"

if [[ -n ${component:-} ]]; then
cd "components/$component"
fi

# We do not have any packages in the stable channel for aarch64 and probably never will.
# Set the HAB_INTERPRETER_IDENT to point to LTS-2024 to proceed with the tests.
export HAB_INTERPRETER_IDENT="core/busybox-static/1.36.1/20240805133911"

# Always add `--quiet` to avoid the noise of compilation in test output.
# The invocation to this script can add `--format pretty` to the test runner
# args (that is, after --, like --nocapture and --test-threads) if the names
# of the tests being run is desired in the output.
cargo test --quiet "$@"
Loading

0 comments on commit c511d64

Please sign in to comment.