From 73fcf2175405992b3e0ebbf91227742766238043 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 23 Jul 2024 16:33:02 +0530 Subject: [PATCH 01/55] build hab on aarch64-linux Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index f36c65138a..655ec556ac 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -656,6 +656,20 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 hab" + env: + HAB_LICENSE: "accept-no-persist" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/hab + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: launcher" env: HAB_LICENSE: "accept-no-persist" From ba5175694769c77cd8dca1ea9ec98267be8b0b08 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 23 Jul 2024 18:53:43 +0530 Subject: [PATCH 02/55] try with arm64-linux Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 655ec556ac..9dbd04950b 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -659,7 +659,7 @@ steps: - label: "[build] :linux: aarch64 hab" env: HAB_LICENSE: "accept-no-persist" - BUILD_PKG_TARGET: "aarch64-linux" + BUILD_PKG_TARGET: "arm64-linux" command: - .expeditor/scripts/verify/build_package.sh components/hab expeditor: From 36dd3eddacbc3303f6faceecff8c556c39d22644 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 23 Jul 2024 18:55:31 +0530 Subject: [PATCH 03/55] try with arm-linux Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 9dbd04950b..aa7684894f 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -659,7 +659,7 @@ steps: - label: "[build] :linux: aarch64 hab" env: HAB_LICENSE: "accept-no-persist" - BUILD_PKG_TARGET: "arm64-linux" + BUILD_PKG_TARGET: "arm-linux" command: - .expeditor/scripts/verify/build_package.sh components/hab expeditor: From d60ff0e11fc2c6c2ff4485b01ae01c454f3dcc8c Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 23 Jul 2024 20:17:37 +0530 Subject: [PATCH 04/55] with docker-linux-arm64 agent Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index aa7684894f..7605fff187 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -657,6 +657,8 @@ steps: limit: 1 - label: "[build] :linux: aarch64 hab" + agents: + queue: docker-linux-arm64 env: HAB_LICENSE: "accept-no-persist" BUILD_PKG_TARGET: "arm-linux" From e66c5536bb9a792af015cb89066ee6ec3e20e930 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 23 Jul 2024 21:27:28 +0530 Subject: [PATCH 05/55] another try Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 7605fff187..859837f310 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -664,13 +664,6 @@ steps: BUILD_PKG_TARGET: "arm-linux" command: - .expeditor/scripts/verify/build_package.sh components/hab - expeditor: - executor: - docker: - privileged: true - retry: - automatic: - limit: 1 - label: "[build] :linux: launcher" env: From 82843de3d05b4fb57ef81d0a42c12c8d9f2a4d16 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 23 Jul 2024 21:35:00 +0530 Subject: [PATCH 06/55] fix target Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 859837f310..9843cbd147 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -661,7 +661,7 @@ steps: queue: docker-linux-arm64 env: HAB_LICENSE: "accept-no-persist" - BUILD_PKG_TARGET: "arm-linux" + BUILD_PKG_TARGET: "aarch64-linux" command: - .expeditor/scripts/verify/build_package.sh components/hab From 562c30b51682bf786d972fa3a3286e52a8f40319 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 6 Sep 2024 11:08:42 +0530 Subject: [PATCH 07/55] temporary channel for arm64 Signed-off-by: Phani Sajja --- .expeditor/scripts/shared.sh | 11 +++++++++-- .expeditor/scripts/verify/build_package.sh | 5 +++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.expeditor/scripts/shared.sh b/.expeditor/scripts/shared.sh index 73af9f96cd..02fd1127c8 100755 --- a/.expeditor/scripts/shared.sh +++ b/.expeditor/scripts/shared.sh @@ -20,9 +20,16 @@ curlbash_hab() { sudo rm -rf /hab/pkgs/core/hab/0.82.0 || \ rm -rf /hab/pkgs/core/hab/0.82.0 fi - curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$pkg_target" + + # Use the temporarily uploaded Habitat package from the 'aarch64-linux' channel for aarch64. + # Once hab is released in the LTS channel, we can use either HAB_BLDR_CHANNEL or HAB_REFRESH_CHANNEL to install hab. + install_args="-t $pkg_target" + if [ "$pkg_target" = "aarch64-linux" ]; then + install_args="$install_args -c aarch64-linux -v 1.6.706" + fi + curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- $install_args case "${pkg_target}" in - x86_64-linux | x86_64-linux-kernel2) + x86_64-linux | aarch64-linux | x86_64-linux-kernel2) hab_binary="/bin/hab" ;; x86_64-darwin) diff --git a/.expeditor/scripts/verify/build_package.sh b/.expeditor/scripts/verify/build_package.sh index 61e81d6cd2..c37e3a4d2d 100755 --- a/.expeditor/scripts/verify/build_package.sh +++ b/.expeditor/scripts/verify/build_package.sh @@ -21,5 +21,10 @@ HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" echo "--- :key: Generating fake origin key" ${hab_binary} origin key generate +# Install the hab-studio from the 'aarch64-linux' channel. +# Once hab is released in the LTS channel, we can use either HAB_BLDR_CHANNEL or HAB_REFRESH_CHANNEL to install the studio. +if [ "$BUILD_PKG_TARGET" = "aarch64-linux" ]; then + ${hab_binary} pkg install core/hab-studio/1.6.706 --channel aarch64-linux +fi echo "--- :hab: Running hab pkg build for $package_path" ${hab_binary} pkg build "$package_path" From f01ea3d6f19e7926eb721809febc6bbc4890ca93 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 6 Sep 2024 17:09:53 +0530 Subject: [PATCH 08/55] adjust rustc_target from pkg_target Signed-off-by: Phani Sajja --- components/hab/habitat/plan.sh | 10 +++- components/launcher/habitat/plan.sh | 2 +- .../pkg-export-container/habitat/plan.sh | 2 +- components/pkg-export-tar/habitat/plan.sh | 2 +- components/sup/habitat/plan.sh | 2 +- components/sup/static/plan.sh | 2 +- .../habitat/aarch64-linux/config/config.toml | 3 ++ .../aarch64-linux/config/render_context.json | 7 +++ .../habitat/aarch64-linux/default.toml | 3 ++ .../habitat/aarch64-linux/health_exit | 1 + .../habitat/aarch64-linux/hooks/health-check | 6 +++ .../habitat/aarch64-linux/hooks/init | 15 ++++++ .../habitat/aarch64-linux/hooks/post-run | 4 ++ .../habitat/aarch64-linux/hooks/post-stop | 14 ++++++ .../habitat/aarch64-linux/hooks/run | 20 ++++++++ .../test-probe/habitat/aarch64-linux/plan.sh | 47 +++++++++++++++++++ 16 files changed, 134 insertions(+), 6 deletions(-) create mode 100755 test-services/test-probe/habitat/aarch64-linux/config/config.toml create mode 100755 test-services/test-probe/habitat/aarch64-linux/config/render_context.json create mode 100755 test-services/test-probe/habitat/aarch64-linux/default.toml create mode 100644 test-services/test-probe/habitat/aarch64-linux/health_exit create mode 100644 test-services/test-probe/habitat/aarch64-linux/hooks/health-check create mode 100644 test-services/test-probe/habitat/aarch64-linux/hooks/init create mode 100644 test-services/test-probe/habitat/aarch64-linux/hooks/post-run create mode 100644 test-services/test-probe/habitat/aarch64-linux/hooks/post-stop create mode 100644 test-services/test-probe/habitat/aarch64-linux/hooks/run create mode 100644 test-services/test-probe/habitat/aarch64-linux/plan.sh diff --git a/components/hab/habitat/plan.sh b/components/hab/habitat/plan.sh index daa15efe6e..1c36321f71 100644 --- a/components/hab/habitat/plan.sh +++ b/components/hab/habitat/plan.sh @@ -53,7 +53,15 @@ do_before() { do_prepare() { _common_prepare - export rustc_target="x86_64-unknown-linux-musl" + # With the musl target, the ring crate is looking for aarch64-linux-musl-gcc, + # but the core/musl package provides musl-gcc. This workaround is necessary until the appropriate changes are made to core/musl for aarch64. + if [[ "${pkg_target%%-*}" == "aarch64" ]]; then + if [[ ! -r "$(pkg_path_for musl)/bin/musl-gcc" ]]; then + ln -sv "$(pkg_path_for musl)/bin/musl-gcc" "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" + fi + fi + + export rustc_target="${pkg_target%%-*}-unknown-linux-musl" build_line "Setting rustc_target=$rustc_target" # Used to find libgcc_s.so.1 when compiling `build.rs` in dependencies. Since diff --git a/components/launcher/habitat/plan.sh b/components/launcher/habitat/plan.sh index d2f3bd3334..214818f836 100644 --- a/components/launcher/habitat/plan.sh +++ b/components/launcher/habitat/plan.sh @@ -30,7 +30,7 @@ do_prepare() { # Can be either `--release` or `--debug` to determine cargo build strategy build_line "Building artifacts with \`${cargo_build_mode#--}' mode" - export rustc_target="x86_64-unknown-linux-gnu" + export rustc_target="${pkg_target%%-*}-unknown-linux-gnu" build_line "Setting rustc_target=$rustc_target" # Used by Cargo to use a pristine, isolated directory for all compilation diff --git a/components/pkg-export-container/habitat/plan.sh b/components/pkg-export-container/habitat/plan.sh index 805bcfe311..f1efbce790 100644 --- a/components/pkg-export-container/habitat/plan.sh +++ b/components/pkg-export-container/habitat/plan.sh @@ -58,7 +58,7 @@ do_before() { do_prepare() { _common_prepare - export rustc_target="x86_64-unknown-linux-musl" + export rustc_target="${pkg_target%%-*}-unknown-linux-musl" build_line "Setting rustc_target=$rustc_target" # Used to find libgcc_s.so.1 when compiling `build.rs` in dependencies. Since diff --git a/components/pkg-export-tar/habitat/plan.sh b/components/pkg-export-tar/habitat/plan.sh index 41ab3b9da1..6518e15c93 100644 --- a/components/pkg-export-tar/habitat/plan.sh +++ b/components/pkg-export-tar/habitat/plan.sh @@ -52,7 +52,7 @@ do_before() { do_prepare() { _common_prepare - export rustc_target="x86_64-unknown-linux-musl" + export rustc_target="${pkg_target%%-*}-unknown-linux-musl" build_line "Setting rustc_target=$rustc_target" # Used to find libgcc_s.so.1 when compiling `build.rs` in dependencies. Since diff --git a/components/sup/habitat/plan.sh b/components/sup/habitat/plan.sh index 238f4c900f..c0fbd5b49b 100644 --- a/components/sup/habitat/plan.sh +++ b/components/sup/habitat/plan.sh @@ -57,7 +57,7 @@ do_before() { do_prepare() { _common_prepare - export rustc_target="x86_64-unknown-linux-gnu" + export rustc_target="${pkg_target%%-*}-unknown-linux-gnu" build_line "Setting rustc_target=$rustc_target" export LIBZMQ_PREFIX=$(pkg_path_for zeromq) diff --git a/components/sup/static/plan.sh b/components/sup/static/plan.sh index 5248d5642a..2330aa2a39 100644 --- a/components/sup/static/plan.sh +++ b/components/sup/static/plan.sh @@ -17,7 +17,7 @@ do_begin() { do_prepare() { _common_prepare - export rustc_target="x86_64-unknown-linux-musl" + export rustc_target="${pkg_target%%-*}-unknown-linux-musl" build_line "Setting rustc_target=$rustc_target" # Used to find libgcc_s.so.1 when compiling `build.rs` in dependencies. Since diff --git a/test-services/test-probe/habitat/aarch64-linux/config/config.toml b/test-services/test-probe/habitat/aarch64-linux/config/config.toml new file mode 100755 index 0000000000..4dd3616399 --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/config/config.toml @@ -0,0 +1,3 @@ +host = "{{cfg.host}}" +port = {{cfg.port}} +render_context_file = "{{cfg.render_context_file}}" diff --git a/test-services/test-probe/habitat/aarch64-linux/config/render_context.json b/test-services/test-probe/habitat/aarch64-linux/config/render_context.json new file mode 100755 index 0000000000..92cf97f92f --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/config/render_context.json @@ -0,0 +1,7 @@ +{ + "sys": {{toJson sys}}, + "pkg": {{toJson pkg}}, + "cfg": {{toJson cfg}}, + "svc": {{toJson svc}}, + "bind": {{toJson bind}} +} diff --git a/test-services/test-probe/habitat/aarch64-linux/default.toml b/test-services/test-probe/habitat/aarch64-linux/default.toml new file mode 100755 index 0000000000..39442ac6a2 --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/default.toml @@ -0,0 +1,3 @@ +host = "0.0.0.0" +port = 8000 +render_context_file = "/hab/svc/test-probe/config/render_context.json" diff --git a/test-services/test-probe/habitat/aarch64-linux/health_exit b/test-services/test-probe/habitat/aarch64-linux/health_exit new file mode 100644 index 0000000000..c227083464 --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/health_exit @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/test-services/test-probe/habitat/aarch64-linux/hooks/health-check b/test-services/test-probe/habitat/aarch64-linux/hooks/health-check new file mode 100644 index 0000000000..b1a02df9e9 --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/hooks/health-check @@ -0,0 +1,6 @@ +#!{{pkgPathFor "core/bash"}}/bin/bash + +echo "Running health_check hook: {{pkg.ident}} (PID: $$, PPID: $PPID, PGID: $(ps h -o pgid -p $$))" +sleep 2 +echo "health_check finished!" +exit $(cat {{pkg.path}}/health_exit) diff --git a/test-services/test-probe/habitat/aarch64-linux/hooks/init b/test-services/test-probe/habitat/aarch64-linux/hooks/init new file mode 100644 index 0000000000..3122e1580f --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/hooks/init @@ -0,0 +1,15 @@ +#!{{pkgPathFor "core/bash"}}/bin/bash + +echo "Initializing package {{pkg.ident}} (PID: $$, PPID: $PPID, PGID: $(ps h -o pgid -p $$))" +sleep 1 +echo "... reticulating splines ..." +echo "... integrating curves ..." +echo "... relaxing splines ..." +echo "... calculating inverse probability matrices ..." +sleep 1 +echo "Deliberately taking a long time in the init hook" +for i in {1..10}; do + sleep 1 + echo "Sleeping ($i)/10..." +done +echo "init hook DONE" diff --git a/test-services/test-probe/habitat/aarch64-linux/hooks/post-run b/test-services/test-probe/habitat/aarch64-linux/hooks/post-run new file mode 100644 index 0000000000..4f3f72ca5c --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/hooks/post-run @@ -0,0 +1,4 @@ +#!{{pkgPathFor "core/bash"}}/bin/bash + +echo "Running post-run script: {{pkg.ident}} (PID: $$, PPID: $PPID, PGID: $(ps h -o pgid -p $$))" +echo "Done" diff --git a/test-services/test-probe/habitat/aarch64-linux/hooks/post-stop b/test-services/test-probe/habitat/aarch64-linux/hooks/post-stop new file mode 100644 index 0000000000..84411ed399 --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/hooks/post-stop @@ -0,0 +1,14 @@ +#!{{pkgPathFor "core/bash"}}/bin/bash + +byebye(){ + echo "Got a signal!" +} +trap byebye INT TERM + + +echo "Deliberately long post-stop hook executing: {{pkg.ident}} (PID: $$, PPID: $PPID, PGID: $(ps h -o pgid -p $$))" +for i in {1..15}; do + sleep 1 + echo "Sleeping ($i)/15..." +done +echo "post-stop hook DONE" diff --git a/test-services/test-probe/habitat/aarch64-linux/hooks/run b/test-services/test-probe/habitat/aarch64-linux/hooks/run new file mode 100644 index 0000000000..1005340ca9 --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/hooks/run @@ -0,0 +1,20 @@ +#!{{pkgPathFor "core/bash"}}/bin/bash + + +exec 2>&1 + +echo "Running {{pkg.ident}}" # (PID: $$, PPID: $PPID, PGID: $(ps h -o pgid -p $$))" +{{ #if bind.thing_with_a_port }} +echo "*************************************************************" +echo "Running with a bound service group for 'thing_with_a_port'" +{{ #each bind.thing_with_a_port.members as |m| ~}} +echo "- {{m.sys.hostname}}" +{{/each ~}} +echo "*************************************************************" +{{ else }} +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +echo "Running WITHOUT a bound service group for 'thing_with_a_port'" +echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" +{{ /if }} + +exec test-probe -c "{{pkg.svc_config_path}}/config.toml" diff --git a/test-services/test-probe/habitat/aarch64-linux/plan.sh b/test-services/test-probe/habitat/aarch64-linux/plan.sh new file mode 100644 index 0000000000..cfb9f84a0a --- /dev/null +++ b/test-services/test-probe/habitat/aarch64-linux/plan.sh @@ -0,0 +1,47 @@ +# shellcheck disable=2034,2154 +pkg_name=test-probe +pkg_origin=habitat-testing +pkg_version="0.1.0" +pkg_maintainer="The Habitat Maintainers " +pkg_license=("Apache-2.0") +pkg_bin_dirs=(bin) +pkg_deps=(core/glibc + core/gcc-libs + core/procps-ng + core/bash) +pkg_build_deps=(core/coreutils + core/rust + core/gcc + core/git + core/make) +pkg_binds_optional=( + [thing_with_a_port]="port" +) + +bin="test-probe" + +do_prepare() { + export rustc_target="aarch64-unknown-linux-gnu" + build_line "Setting rustc_target=$rustc_target" + + # Used by Cargo to use a pristine, isolated directory for all compilation + export CARGO_TARGET_DIR="$HAB_CACHE_SRC_PATH/$pkg_dirname" + build_line "Setting CARGO_TARGET_DIR=$CARGO_TARGET_DIR" +} + +do_build() { + ( + cd "$PLAN_CONTEXT"/.. || exit + cargo build --target="$rustc_target" --verbose + ) +} + +do_install() { + install -v -D "$CARGO_TARGET_DIR/$rustc_target/debug/$bin" \ + "$pkg_prefix/bin/$bin" + install -v -D "$PLAN_CONTEXT/health_exit" "$pkg_prefix" +} + +do_strip() { + return 0 +} From 1e36f37afa7f401f4801061a8c52ea9cc6d9349a Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 12:10:21 +0530 Subject: [PATCH 09/55] build pkgs on aarch64 in ci Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 174 ++++++++++++++++++++++ components/hab/habitat/plan.sh | 2 +- components/pkg-export-tar/habitat/plan.sh | 8 + 3 files changed, 183 insertions(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 9843cbd147..a4a20cded1 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -625,6 +625,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 test-probe" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh test-services/test-probe + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + # SHOULD BE UNCOMMENTED WHEN NEWER VERSION OF HAB IS RELEASED # - label: "[build] :linux: backline" @@ -642,6 +659,24 @@ steps: # automatic: # limit: 1 + # This build currently fails due to a circular dependency in 'LTS-2024' and needs investigation. + - label: "[build] :linux: aarch64 backline" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/backline + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: hab" env: HAB_LICENSE: "accept-no-persist" @@ -661,6 +696,7 @@ steps: queue: docker-linux-arm64 env: HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - .expeditor/scripts/verify/build_package.sh components/hab @@ -679,6 +715,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 launcher" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/launcher + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + # SHOULD BE UNCOMMENTED WHEN NEWER VERSION OF HAB IS RELEASED # - label: "[build] :linux: pkg-cfize" @@ -696,6 +749,24 @@ steps: # automatic: # limit: 1 + # This build currently fails due to a dependency on 'pkg-export-container' + - label: "[build] :linux: aarch64 pkg-cfize" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/pkg-cfize + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: pkg-export-container" env: HAB_LICENSE: "accept-no-persist" @@ -710,6 +781,24 @@ steps: automatic: limit: 1 + # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. + - label: "[build] :linux: aarch64 pkg-export-containe" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/pkg-export-containe + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: pkg-export-tar" env: HAB_LICENSE: "accept-no-persist" @@ -724,6 +813,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 pkg-export-tar" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/pkg-export-tar + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: pkg-mesosize" env: HAB_LICENSE: "accept-no-persist" @@ -738,6 +844,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 pkg-mesosize" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/pkg-mesosize + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: plan-build" env: HAB_LICENSE: "accept-no-persist" @@ -752,6 +875,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 plan-build" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/plan-build + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: studio" env: HAB_LICENSE: "accept-no-persist" @@ -766,6 +906,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 studio" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/studio + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + - label: "[build] :linux: sup" env: HAB_LICENSE: "accept-no-persist" @@ -780,6 +937,23 @@ steps: automatic: limit: 1 + - label: "[build] :linux: aarch64 sup" + agents: + queue: docker-linux-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - .expeditor/scripts/verify/build_package.sh components/sup + expeditor: + executor: + docker: + privileged: true + retry: + automatic: + limit: 1 + ####################################################################### # Things that have no tests but should be built to make sure they # still build. - Linux Kernel 2 diff --git a/components/hab/habitat/plan.sh b/components/hab/habitat/plan.sh index 1c36321f71..e5d0787339 100644 --- a/components/hab/habitat/plan.sh +++ b/components/hab/habitat/plan.sh @@ -56,7 +56,7 @@ do_prepare() { # With the musl target, the ring crate is looking for aarch64-linux-musl-gcc, # but the core/musl package provides musl-gcc. This workaround is necessary until the appropriate changes are made to core/musl for aarch64. if [[ "${pkg_target%%-*}" == "aarch64" ]]; then - if [[ ! -r "$(pkg_path_for musl)/bin/musl-gcc" ]]; then + if [[ ! -r "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" ]]; then ln -sv "$(pkg_path_for musl)/bin/musl-gcc" "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" fi fi diff --git a/components/pkg-export-tar/habitat/plan.sh b/components/pkg-export-tar/habitat/plan.sh index 6518e15c93..7f6ca0ddd8 100644 --- a/components/pkg-export-tar/habitat/plan.sh +++ b/components/pkg-export-tar/habitat/plan.sh @@ -52,6 +52,14 @@ do_before() { do_prepare() { _common_prepare + # With the musl target, the ring crate is looking for aarch64-linux-musl-gcc, + # but the core/musl package provides musl-gcc. This workaround is necessary until the appropriate changes are made to core/musl for aarch64. + if [[ "${pkg_target%%-*}" == "aarch64" ]]; then + if [[ ! -r "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" ]]; then + ln -sv "$(pkg_path_for musl)/bin/musl-gcc" "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" + fi + fi + export rustc_target="${pkg_target%%-*}-unknown-linux-musl" build_line "Setting rustc_target=$rustc_target" From e44cd28928a4a5ccfcca6e00883b923ccda8cde1 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 12:26:07 +0530 Subject: [PATCH 10/55] try with default-privileged-aarch64 Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index a4a20cded1..7dd32475bc 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -627,7 +627,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" From 6ffe27319d784df718b987117979e9268f504cbe Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 12:36:50 +0530 Subject: [PATCH 11/55] target default-privileged-aarch64 queue Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 7dd32475bc..bba31bbf1e 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -662,7 +662,7 @@ steps: # This build currently fails due to a circular dependency in 'LTS-2024' and needs investigation. - label: "[build] :linux: aarch64 backline" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -693,7 +693,7 @@ steps: - label: "[build] :linux: aarch64 hab" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -717,7 +717,7 @@ steps: - label: "[build] :linux: aarch64 launcher" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -752,7 +752,7 @@ steps: # This build currently fails due to a dependency on 'pkg-export-container' - label: "[build] :linux: aarch64 pkg-cfize" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -784,7 +784,7 @@ steps: # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. - label: "[build] :linux: aarch64 pkg-export-containe" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -815,7 +815,7 @@ steps: - label: "[build] :linux: aarch64 pkg-export-tar" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -846,7 +846,7 @@ steps: - label: "[build] :linux: aarch64 pkg-mesosize" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -877,7 +877,7 @@ steps: - label: "[build] :linux: aarch64 plan-build" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -908,7 +908,7 @@ steps: - label: "[build] :linux: aarch64 studio" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -939,7 +939,7 @@ steps: - label: "[build] :linux: aarch64 sup" agents: - queue: docker-linux-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" From 4ac55a76ea59ea177a476d8aa3187874970a7874 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 13:32:47 +0530 Subject: [PATCH 12/55] try varied config Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index bba31bbf1e..e48b40eb21 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -627,7 +627,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue: default-privileged-aarch64 + queue: default-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -669,10 +669,6 @@ steps: BUILD_PKG_TARGET: "aarch64-linux" command: - .expeditor/scripts/verify/build_package.sh components/backline - expeditor: - executor: - docker: - privileged: true retry: automatic: limit: 1 From efc61d09569b9ac31b5b68415ab44f810c335809 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 13:34:34 +0530 Subject: [PATCH 13/55] default-arm64 Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index e48b40eb21..c78e999f9c 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -627,7 +627,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue: default-aarch64 + queue: default-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" From 1ca7adcd8c57aec21af7c81143064870cea518f0 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 13:52:02 +0530 Subject: [PATCH 14/55] try without privileged Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index c78e999f9c..4651efaa85 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -637,7 +637,6 @@ steps: expeditor: executor: docker: - privileged: true retry: automatic: limit: 1 From e8234bf19ca3a1c84c42ea6c18f37b7c16beb267 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 14:08:01 +0530 Subject: [PATCH 15/55] revert privileged Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 4651efaa85..c78e999f9c 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -637,6 +637,7 @@ steps: expeditor: executor: docker: + privileged: true retry: automatic: limit: 1 From eeac7cef10f09b24dd769cf5fac568167bc3cdb0 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 9 Sep 2024 19:50:59 +0530 Subject: [PATCH 16/55] fix shellcheck Signed-off-by: Phani Sajja --- .expeditor/scripts/shared.sh | 2 +- components/hab/habitat/plan.sh | 4 ++-- components/pkg-export-container/habitat/plan.sh | 4 ++-- components/pkg-export-tar/habitat/plan.sh | 4 ++-- components/sup/habitat/plan.sh | 4 ++-- components/sup/static/plan.sh | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.expeditor/scripts/shared.sh b/.expeditor/scripts/shared.sh index 02fd1127c8..cd57710617 100755 --- a/.expeditor/scripts/shared.sh +++ b/.expeditor/scripts/shared.sh @@ -27,7 +27,7 @@ curlbash_hab() { if [ "$pkg_target" = "aarch64-linux" ]; then install_args="$install_args -c aarch64-linux -v 1.6.706" fi - curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- $install_args + curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- "$install_args" case "${pkg_target}" in x86_64-linux | aarch64-linux | x86_64-linux-kernel2) hab_binary="/bin/hab" diff --git a/components/hab/habitat/plan.sh b/components/hab/habitat/plan.sh index e5d0787339..e9d25f2831 100644 --- a/components/hab/habitat/plan.sh +++ b/components/hab/habitat/plan.sh @@ -87,12 +87,12 @@ do_prepare() { do_build() { pushd "$SRC_PATH" > /dev/null || exit - cargo build ${build_type#--debug} --target=$rustc_target --verbose + cargo build ${build_type#--debug} --target="$rustc_target" --verbose popd > /dev/null || exit } do_install() { - install -v -D "$CARGO_TARGET_DIR"/$rustc_target/${build_type#--}/$bin \ + install -v -D "$CARGO_TARGET_DIR"/"$rustc_target"/${build_type#--}/$bin \ "$pkg_prefix"/bin/$bin } diff --git a/components/pkg-export-container/habitat/plan.sh b/components/pkg-export-container/habitat/plan.sh index f1efbce790..3fda523b9b 100644 --- a/components/pkg-export-container/habitat/plan.sh +++ b/components/pkg-export-container/habitat/plan.sh @@ -89,12 +89,12 @@ do_prepare() { do_build() { pushd "$PLAN_CONTEXT" || exit - cargo build ${build_type#--debug} --target=$rustc_target --verbose + cargo build ${build_type#--debug} --target="$rustc_target" --verbose popd || exit } do_install() { - install -v -D "$CARGO_TARGET_DIR"/$rustc_target/${build_type#--}/$bin \ + install -v -D "$CARGO_TARGET_DIR"/"$rustc_target"/${build_type#--}/$bin \ "$pkg_prefix"/bin/$bin } diff --git a/components/pkg-export-tar/habitat/plan.sh b/components/pkg-export-tar/habitat/plan.sh index 7f6ca0ddd8..8bb4093e46 100644 --- a/components/pkg-export-tar/habitat/plan.sh +++ b/components/pkg-export-tar/habitat/plan.sh @@ -82,11 +82,11 @@ do_prepare() { do_build() { pushd "$PLAN_CONTEXT" || exit - cargo build ${build_type#--debug} --target=$rustc_target --verbose + cargo build ${build_type#--debug} --target="$rustc_target" --verbose popd || exit } do_install() { - install -v -D "$CARGO_TARGET_DIR"/$rustc_target/${build_type#--}/$bin \ + install -v -D "$CARGO_TARGET_DIR"/"$rustc_target"/${build_type#--}/$bin \ "$pkg_prefix"/bin/$bin } diff --git a/components/sup/habitat/plan.sh b/components/sup/habitat/plan.sh index c0fbd5b49b..fb59115ce9 100644 --- a/components/sup/habitat/plan.sh +++ b/components/sup/habitat/plan.sh @@ -91,13 +91,13 @@ do_build() { export LIBRARY_PATH=$LIBZMQ_PREFIX/lib pushd "$SRC_PATH" > /dev/null || exit - cargo build ${build_type#--debug} --target=$rustc_target --verbose --no-default-features \ + cargo build ${build_type#--debug} --target="$rustc_target" --verbose --no-default-features \ --features apidocs popd > /dev/null || exit } do_install() { - install -v -D "$CARGO_TARGET_DIR"/$rustc_target/${build_type#--}/$bin \ + install -v -D "$CARGO_TARGET_DIR"/"$rustc_target"/${build_type#--}/$bin \ "$pkg_prefix"/bin/$bin } diff --git a/components/sup/static/plan.sh b/components/sup/static/plan.sh index 2330aa2a39..59dedae4ba 100644 --- a/components/sup/static/plan.sh +++ b/components/sup/static/plan.sh @@ -13,7 +13,7 @@ do_begin() { PLAN_CONTEXT=$(abspath ..) } -# shellcheck disable=2155 +# shellcheck disable=2155,SC2154 do_prepare() { _common_prepare From dea96d8c45b3d7aada0f4f597004d973b416ae0a Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 11 Sep 2024 10:55:31 +0530 Subject: [PATCH 17/55] trigger pipeline Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index c78e999f9c..bbb68d8bd2 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -627,7 +627,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue: default-arm64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -669,6 +669,10 @@ steps: BUILD_PKG_TARGET: "aarch64-linux" command: - .expeditor/scripts/verify/build_package.sh components/backline + expeditor: + executor: + docker: + privileged: true retry: automatic: limit: 1 @@ -696,6 +700,10 @@ steps: BUILD_PKG_TARGET: "aarch64-linux" command: - .expeditor/scripts/verify/build_package.sh components/hab + expeditor: + executor: + docker: + privileged: true - label: "[build] :linux: launcher" env: From 7da0dd3b5bf5e17a79bac7e11576f67a2064258c Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 11 Sep 2024 11:34:08 +0530 Subject: [PATCH 18/55] try without quotes Signed-off-by: Phani Sajja --- .expeditor/scripts/shared.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.expeditor/scripts/shared.sh b/.expeditor/scripts/shared.sh index cd57710617..666369f335 100755 --- a/.expeditor/scripts/shared.sh +++ b/.expeditor/scripts/shared.sh @@ -25,9 +25,10 @@ curlbash_hab() { # Once hab is released in the LTS channel, we can use either HAB_BLDR_CHANNEL or HAB_REFRESH_CHANNEL to install hab. install_args="-t $pkg_target" if [ "$pkg_target" = "aarch64-linux" ]; then - install_args="$install_args -c aarch64-linux -v 1.6.706" + install_args+="-c aarch64-linux -v 1.6.706" fi - curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- "$install_args" + echo "install_args: $install_args" + curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- ${install_args} case "${pkg_target}" in x86_64-linux | aarch64-linux | x86_64-linux-kernel2) hab_binary="/bin/hab" From 5727d666505042edbf859c71ebc2f1e4ff9f8f7c Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 11 Sep 2024 11:42:32 +0530 Subject: [PATCH 19/55] add missing space in cmd aqrgs Signed-off-by: Phani Sajja --- .expeditor/scripts/shared.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/scripts/shared.sh b/.expeditor/scripts/shared.sh index 666369f335..a8f434de58 100755 --- a/.expeditor/scripts/shared.sh +++ b/.expeditor/scripts/shared.sh @@ -25,7 +25,7 @@ curlbash_hab() { # Once hab is released in the LTS channel, we can use either HAB_BLDR_CHANNEL or HAB_REFRESH_CHANNEL to install hab. install_args="-t $pkg_target" if [ "$pkg_target" = "aarch64-linux" ]; then - install_args+="-c aarch64-linux -v 1.6.706" + install_args+=" -c aarch64-linux -v 1.6.706" fi echo "install_args: $install_args" curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- ${install_args} From 047bb91efbae65195003baa17619ee4cea099072 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Thu, 19 Sep 2024 13:52:19 +0530 Subject: [PATCH 20/55] try with docker-linux-amazon-2023-arm64 Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index bbb68d8bd2..e6e985fa2b 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -627,7 +627,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" From 5f90c5123a1a8545cb493af830c9dcffc2761e08 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Thu, 19 Sep 2024 14:41:40 +0530 Subject: [PATCH 21/55] try sudo Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index e6e985fa2b..3fd791f64f 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -633,11 +633,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh test-services/test-probe - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh test-services/test-probe retry: automatic: limit: 1 From dc2dcc24bf156f07c3205e478ac44bc0b18c8eae Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Thu, 19 Sep 2024 15:08:46 +0530 Subject: [PATCH 22/55] target docker-linux-amazon-2023-arm64 with sudo Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 83 ++++++++++------------------------ 1 file changed, 23 insertions(+), 60 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 3fd791f64f..819aded841 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -658,17 +658,13 @@ steps: # This build currently fails due to a circular dependency in 'LTS-2024' and needs investigation. - label: "[build] :linux: aarch64 backline" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/backline - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/backline retry: automatic: limit: 1 @@ -689,17 +685,16 @@ steps: - label: "[build] :linux: aarch64 hab" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/hab - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/hab + retry: + automatic: + limit: 1 - label: "[build] :linux: launcher" env: @@ -717,17 +712,13 @@ steps: - label: "[build] :linux: aarch64 launcher" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/launcher - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/launcher retry: automatic: limit: 1 @@ -752,17 +743,13 @@ steps: # This build currently fails due to a dependency on 'pkg-export-container' - label: "[build] :linux: aarch64 pkg-cfize" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/pkg-cfize - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-cfize retry: automatic: limit: 1 @@ -784,17 +771,13 @@ steps: # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. - label: "[build] :linux: aarch64 pkg-export-containe" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/pkg-export-containe - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-export-containe retry: automatic: limit: 1 @@ -815,17 +798,13 @@ steps: - label: "[build] :linux: aarch64 pkg-export-tar" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/pkg-export-tar - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-export-tar retry: automatic: limit: 1 @@ -846,17 +825,13 @@ steps: - label: "[build] :linux: aarch64 pkg-mesosize" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/pkg-mesosize - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-mesosize retry: automatic: limit: 1 @@ -877,17 +852,13 @@ steps: - label: "[build] :linux: aarch64 plan-build" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/plan-build - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/plan-build retry: automatic: limit: 1 @@ -908,17 +879,13 @@ steps: - label: "[build] :linux: aarch64 studio" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/studio - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/studio retry: automatic: limit: 1 @@ -939,17 +906,13 @@ steps: - label: "[build] :linux: aarch64 sup" agents: - queue: default-privileged-aarch64 + queue: docker-linux-amazon-2023-arm64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - .expeditor/scripts/verify/build_package.sh components/sup - expeditor: - executor: - docker: - privileged: true + - sudo -E .expeditor/scripts/verify/build_package.sh components/sup retry: automatic: limit: 1 From b479d1db38005743ea32edfcfec5b4052cf228b0 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 23 Sep 2024 22:32:09 +0530 Subject: [PATCH 23/55] fix component name Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 819aded841..1853c79692 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -769,7 +769,7 @@ steps: limit: 1 # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. - - label: "[build] :linux: aarch64 pkg-export-containe" + - label: "[build] :linux: aarch64 pkg-export-container" agents: queue: docker-linux-amazon-2023-arm64 env: @@ -777,7 +777,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-export-containe + - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-export-container retry: automatic: limit: 1 From 19cc60679ce1fb307ec36a182dfb20912cb6498d Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 25 Sep 2024 10:57:32 +0530 Subject: [PATCH 24/55] disable shellcheck SC2086 Signed-off-by: Phani Sajja --- .expeditor/scripts/shared.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.expeditor/scripts/shared.sh b/.expeditor/scripts/shared.sh index a8f434de58..7595f24a44 100755 --- a/.expeditor/scripts/shared.sh +++ b/.expeditor/scripts/shared.sh @@ -28,6 +28,7 @@ curlbash_hab() { install_args+=" -c aarch64-linux -v 1.6.706" fi echo "install_args: $install_args" + # shellcheck disable=SC2086 curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- ${install_args} case "${pkg_target}" in x86_64-linux | aarch64-linux | x86_64-linux-kernel2) From 05ba30dda4459be519613fb26d69baf5a64c4c55 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 4 Oct 2024 13:50:25 +0530 Subject: [PATCH 25/55] first cargo test Signed-off-by: Phani Sajja --- .../scripts/verify/run_cargo_test-aarch64.sh | 65 +++++++++++++++++++ .expeditor/verify.pipeline.yml | 14 ++++ 2 files changed, 79 insertions(+) create mode 100755 .expeditor/scripts/verify/run_cargo_test-aarch64.sh diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh new file mode 100755 index 0000000000..3b1952496f --- /dev/null +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -eou pipefail + +# shellcheck source=.expeditor/scripts/shared.sh +source .expeditor/scripts/verify/shared.sh + +if [[ ${1:-"--"} = "--" ]]; then + scope="habitat workspace" +else + component="$1" + shift + scope="$component" +fi + +toolchain=$(get_toolchain) + +hab pkg install core/zeromq +hab pkg install core/protobuf +hab pkg install core/rust/"$toolchain" +hab pkg install core/xz +hab pkg install core/coreutils +hab pkg install core/openssl + +export LIBZMQ_PREFIX +LIBZMQ_PREFIX=$(hab pkg path core/zeromq) +# now include zeromq and gcc so they exist in the runtime library path when cargo test is run +export LD_LIBRARY_PATH +LD_LIBRARY_PATH="$(hab pkg path core/gcc-base)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/xz)/lib:$(hab pkg path core/openssl)/lib" +export LIBRARY_PATH="$(hab pkg path core/xz)/lib" + +export PROTOC_NO_VENDOR=1 +export PROTOC +PROTOC=$(hab pkg path core/protobuf)/bin/protoc + +_oldPth=$PATH +_pth="$(hab pkg path core/coreutils)/bin:$(hab pkg path core/openssl)/bin" +eval "$(hab pkg env core/rust/"$toolchain"):$PATH" +export PATH="$PATH:$_pth:$_oldPth" + +export OPENSSL_DIR="$(hab pkg path core/openssl)/bin" +export OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib" + +# Set testing filesystem root +export FS_ROOT +FS_ROOT=$(mktemp -d /tmp/testing-fs-root-XXXXXX) + +export RUST_BACKTRACE=1 + +# Build the all the hab binaries so that we can run integration tests +if [[ "$scope" == "sup" ]]; then + cargo build +fi + +echo "--- Running cargo test with scope '$scope' and args '$*'" + +if [[ -n ${component:-} ]]; then + cd "components/$component" +fi + +# Always add `--quiet` to avoid the noise of compilation in test output. +# The invocation to this script can add `--format pretty` to the test runner +# args (that is, after --, like --nocapture and --test-threads) if the names +# of the tests being run is desired in the output. +cargo test --quiet "$@" diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 1853c79692..db3f70539b 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -111,6 +111,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 ignored" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test.sh -- --ignored + timeout_in_minutes: 20 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: builder-api-client" command: - .expeditor/scripts/verify/run_cargo_test.sh builder-api-client From 31f473f03ae3713b04a3a871c80ad2a8c880789d Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 4 Oct 2024 14:04:05 +0530 Subject: [PATCH 26/55] fix command Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index db3f70539b..5e531291e6 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -119,7 +119,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test.sh -- --ignored + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh -- --ignored timeout_in_minutes: 20 retry: automatic: From aceebb1958fafe74ae015f2090bfb16c830f0ba0 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 4 Oct 2024 15:30:19 +0530 Subject: [PATCH 27/55] add deps Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/run_cargo_test-aarch64.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index 3b1952496f..467ae130bf 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -21,6 +21,11 @@ hab pkg install core/rust/"$toolchain" hab pkg install core/xz hab pkg install core/coreutils hab pkg install core/openssl +hab pkg install core/perl +hab pkg install core/make + +export OPENSSL_DIR="$(hab pkg path core/openssl)/bin" +export OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib" export LIBZMQ_PREFIX LIBZMQ_PREFIX=$(hab pkg path core/zeromq) @@ -34,13 +39,10 @@ export PROTOC PROTOC=$(hab pkg path core/protobuf)/bin/protoc _oldPth=$PATH -_pth="$(hab pkg path core/coreutils)/bin:$(hab pkg path core/openssl)/bin" +_pth="$(hab pkg path core/coreutils)/bin:$(hab pkg path core/openssl)/bin:$(hab pkg path core/perl)/bin:$(hab pkg path core/make)/bin" eval "$(hab pkg env core/rust/"$toolchain"):$PATH" export PATH="$PATH:$_pth:$_oldPth" -export OPENSSL_DIR="$(hab pkg path core/openssl)/bin" -export OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib" - # Set testing filesystem root export FS_ROOT FS_ROOT=$(mktemp -d /tmp/testing-fs-root-XXXXXX) From e2e3a50a5ba0a0c2aeec88a0b444d2bb5f5ade37 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 4 Oct 2024 16:13:09 +0530 Subject: [PATCH 28/55] add all unit tests Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 217 +++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 5e531291e6..59bf9b731f 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -137,6 +137,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 builder-api-client" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh builder-api-client + timeout_in_minutes: 20 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: butterfly lock_as_mutex" command: - .expeditor/scripts/verify/run_cargo_test.sh butterfly --features "lock_as_mutex" -- --test-threads=1 --format=pretty @@ -147,6 +161,18 @@ steps: timeout_in_minutes: 20 soft_fail: true + - label: "[unit] :linux: aarch64 butterfly lock_as_mutex" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh butterfly --features "lock_as_mutex" -- --test-threads=1 --format=pretty + timeout_in_minutes: 20 + soft_fail: true + - label: "[unit] :linux: butterfly lock_as_rwlock" command: - .expeditor/scripts/verify/run_cargo_test.sh butterfly --features "lock_as_rwlock" -- --test-threads=1 --format=pretty @@ -159,6 +185,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 butterfly lock_as_rwlock" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh butterfly --features "lock_as_rwlock" -- --test-threads=1 --format=pretty + timeout_in_minutes: 20 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: common" command: - .expeditor/scripts/verify/run_cargo_test.sh common @@ -176,6 +216,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 common" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh common + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: core" command: - .expeditor/scripts/verify/run_cargo_test.sh core @@ -188,6 +242,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 core" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh core + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: core functional" command: - .expeditor/scripts/verify/run_cargo_test.sh core --features "functional" -- --format=pretty @@ -197,6 +265,17 @@ steps: privileged: true timeout_in_minutes: 20 + - label: "[unit] :linux: aarch64 core functional" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh core --features "functional" -- --format=pretty + timeout_in_minutes: 20 + - label: "[unit] :linux: http-client" command: - .expeditor/scripts/verify/run_cargo_test.sh http-client @@ -209,6 +288,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 http-client" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh http-client + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: hab" command: - .expeditor/scripts/verify/run_cargo_test.sh hab @@ -221,6 +314,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 hab" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh hab + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: launcher-client" command: - .expeditor/scripts/verify/run_cargo_test.sh launcher-client @@ -233,6 +340,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 launcher-client" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh launcher-client + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: launcher-protocol" command: - .expeditor/scripts/verify/run_cargo_test.sh launcher-protocol @@ -245,6 +366,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 launcher-protocol" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh launcher-protocol + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: pkg-export-container" command: - .expeditor/scripts/verify/run_cargo_test.sh pkg-export-container @@ -257,6 +392,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 pkg-export-container" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh pkg-export-container + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: pkg-export-tar" command: - .expeditor/scripts/verify/run_cargo_test.sh pkg-export-tar @@ -269,6 +418,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 pkg-export-tar" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh pkg-export-tar + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: sup lock_as_rwlock" command: - .expeditor/scripts/verify/run_cargo_test.sh sup --features "lock_as_rwlock" @@ -281,6 +444,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 sup lock_as_rwlock" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup --features "lock_as_rwlock" + timeout_in_minutes: 40 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: sup lock_as_mutex" command: - .expeditor/scripts/verify/run_cargo_test.sh sup --features "lock_as_mutex" @@ -291,6 +468,18 @@ steps: timeout_in_minutes: 40 soft_fail: true + - label: "[unit] :linux: aarch64 sup lock_as_mutex" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup --features "lock_as_mutex" + timeout_in_minutes: 40 + soft_fail: true + - label: "[unit] :linux: sup-client" command: - .expeditor/scripts/verify/run_cargo_test.sh sup-client @@ -303,6 +492,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 sup-client" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup-client + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: sup-protocol" command: - .expeditor/scripts/verify/run_cargo_test.sh sup-protocol @@ -315,6 +518,20 @@ steps: automatic: limit: 1 + - label: "[unit] :linux: aarch64 sup-protocol" + agents: + queue: docker-linux-amazon-2023-arm64 + env: + HAB_LICENSE: "accept-no-persist" + HAB_BLDR_CHANNEL: "LTS-2024" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup-protocol + timeout_in_minutes: 10 + retry: + automatic: + limit: 1 + - label: "[unit] :linux: studio" command: - .expeditor/scripts/verify/run_studio_test.sh From 0e99a637b12b79d25296604c1781284e3dfd1641 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 4 Oct 2024 16:18:19 +0530 Subject: [PATCH 29/55] fix shellcheck Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/run_cargo_test-aarch64.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index 467ae130bf..2ced32e081 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -24,15 +24,19 @@ hab pkg install core/openssl hab pkg install core/perl hab pkg install core/make -export OPENSSL_DIR="$(hab pkg path core/openssl)/bin" -export OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib" +export OPENSSL_DIR +OPENSSL_DIR="$(hab pkg path core/openssl)/bin" +export OPENSSL_LIB_DIR +OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib" export LIBZMQ_PREFIX LIBZMQ_PREFIX=$(hab pkg path core/zeromq) # now include zeromq and gcc so they exist in the runtime library path when cargo test is run export LD_LIBRARY_PATH LD_LIBRARY_PATH="$(hab pkg path core/gcc-base)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/xz)/lib:$(hab pkg path core/openssl)/lib" -export LIBRARY_PATH="$(hab pkg path core/xz)/lib" + +export LIBRARY_PATH +LIBRARY_PATH="$(hab pkg path core/xz)/lib" export PROTOC_NO_VENDOR=1 export PROTOC From e3e322537074d45bde031dbf3700dcb68cc47e85 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 7 Oct 2024 17:21:35 +0530 Subject: [PATCH 30/55] adjust script, fix test Signed-off-by: Phani Sajja --- .../scripts/verify/run_cargo_test-aarch64.sh | 16 ++++++++-------- components/common/src/templating/hooks.rs | 11 ----------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index 2ced32e081..85ce2d7d71 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -15,14 +15,14 @@ fi toolchain=$(get_toolchain) -hab pkg install core/zeromq -hab pkg install core/protobuf -hab pkg install core/rust/"$toolchain" -hab pkg install core/xz -hab pkg install core/coreutils -hab pkg install core/openssl -hab pkg install core/perl -hab pkg install core/make +sudo -E hab pkg install core/zeromq +sudo -E hab pkg install core/protobuf +sudo -E hab pkg install core/rust/"$toolchain" +sudo -E hab pkg install core/xz +sudo -E hab pkg install core/coreutils +sudo -E hab pkg install core/openssl +sudo -E hab pkg install core/perl +sudo -E hab pkg install core/make export OPENSSL_DIR OPENSSL_DIR="$(hab pkg path core/openssl)/bin" diff --git a/components/common/src/templating/hooks.rs b/components/common/src/templating/hooks.rs index c0abb0a717..d2ad20aa95 100644 --- a/components/common/src/templating/hooks.rs +++ b/components/common/src/templating/hooks.rs @@ -635,7 +635,6 @@ mod tests { test_helpers::*}; #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), all(target_os = "windows", target_arch = "x86_64"),)))] - use habitat_core::package::metadata::MetaFile; use habitat_core::{package::{PackageIdent, PackageInstall}, service::ServiceGroup}; @@ -866,16 +865,6 @@ echo "The message is Hola Mundo" PathBuf::from("/tmp"), PathBuf::from("/tmp")); - // Platforms without standard package support require all packages to be native packages - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), - all(target_os = "windows", target_arch = "x86_64"))))] - { - tokio::fs::create_dir_all(pkg_install.installed_path()).await - .unwrap(); - create_with_content(pkg_install.installed_path() - .join(MetaFile::PackageType.to_string()), - "native"); - } let pkg = Pkg::from_install(&pkg_install).await .expect("Could not create package!"); From 229bb41c371c4b9486e45e32e0719635ed3dfd51 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 7 Oct 2024 19:47:12 +0530 Subject: [PATCH 31/55] install hab Signed-off-by: Phani Sajja --- .../scripts/verify/run_cargo_test-aarch64.sh | 3 ++ .expeditor/verify.pipeline.yml | 34 +++++++++---------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index 85ce2d7d71..f248c7106f 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -5,6 +5,9 @@ set -eou pipefail # shellcheck source=.expeditor/scripts/shared.sh source .expeditor/scripts/verify/shared.sh +# Install hab +curlbash_hab "${BUILD_PKG_TARGET}" + if [[ ${1:-"--"} = "--" ]]; then scope="habitat workspace" else diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 59bf9b731f..8e101b6c8c 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -119,7 +119,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh -- --ignored + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh -- --ignored timeout_in_minutes: 20 retry: automatic: @@ -145,7 +145,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh builder-api-client + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh builder-api-client timeout_in_minutes: 20 retry: automatic: @@ -169,7 +169,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh butterfly --features "lock_as_mutex" -- --test-threads=1 --format=pretty + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh butterfly --features "lock_as_mutex" -- --test-threads=1 --format=pretty timeout_in_minutes: 20 soft_fail: true @@ -193,7 +193,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh butterfly --features "lock_as_rwlock" -- --test-threads=1 --format=pretty + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh butterfly --features "lock_as_rwlock" -- --test-threads=1 --format=pretty timeout_in_minutes: 20 retry: automatic: @@ -224,7 +224,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh common + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh common timeout_in_minutes: 10 retry: automatic: @@ -250,7 +250,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh core + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh core timeout_in_minutes: 10 retry: automatic: @@ -273,7 +273,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh core --features "functional" -- --format=pretty + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh core --features "functional" -- --format=pretty timeout_in_minutes: 20 - label: "[unit] :linux: http-client" @@ -296,7 +296,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh http-client + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh http-client timeout_in_minutes: 10 retry: automatic: @@ -322,7 +322,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh hab + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh hab timeout_in_minutes: 10 retry: automatic: @@ -348,7 +348,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh launcher-client + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh launcher-client timeout_in_minutes: 10 retry: automatic: @@ -374,7 +374,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh launcher-protocol + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh launcher-protocol timeout_in_minutes: 10 retry: automatic: @@ -400,7 +400,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh pkg-export-container + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh pkg-export-container timeout_in_minutes: 10 retry: automatic: @@ -426,7 +426,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh pkg-export-tar + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh pkg-export-tar timeout_in_minutes: 10 retry: automatic: @@ -452,7 +452,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup --features "lock_as_rwlock" + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup --features "lock_as_rwlock" timeout_in_minutes: 40 retry: automatic: @@ -476,7 +476,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup --features "lock_as_mutex" + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup --features "lock_as_mutex" timeout_in_minutes: 40 soft_fail: true @@ -500,7 +500,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup-client + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup-client timeout_in_minutes: 10 retry: automatic: @@ -526,7 +526,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup-protocol + - .expeditor/scripts/verify/run_cargo_test-aarch64.sh sup-protocol timeout_in_minutes: 10 retry: automatic: From 319cf48cb214ec9b229cce236af929ebb15af296 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 7 Oct 2024 20:46:27 +0530 Subject: [PATCH 32/55] install hab with curl Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/run_cargo_test-aarch64.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index f248c7106f..e3c6b4326a 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -6,7 +6,7 @@ set -eou pipefail source .expeditor/scripts/verify/shared.sh # Install hab -curlbash_hab "${BUILD_PKG_TARGET}" +curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t aarch64-linux -c aarch64-linux -v 1.6.706 if [[ ${1:-"--"} = "--" ]]; then scope="habitat workspace" From cf3f5c280531b27996d8d6747714962fe41ce490 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 7 Oct 2024 20:59:56 +0530 Subject: [PATCH 33/55] install hab Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/run_cargo_test-aarch64.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index e3c6b4326a..55ad0fc444 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -2,12 +2,12 @@ set -eou pipefail -# shellcheck source=.expeditor/scripts/shared.sh -source .expeditor/scripts/verify/shared.sh - # Install hab curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t aarch64-linux -c aarch64-linux -v 1.6.706 +# shellcheck source=.expeditor/scripts/shared.sh +source .expeditor/scripts/verify/shared.sh + if [[ ${1:-"--"} = "--" ]]; then scope="habitat workspace" else From 39598799953d67c1a0688be4641a71b040393262 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 7 Oct 2024 21:26:27 +0530 Subject: [PATCH 34/55] fix cfg Signed-off-by: Phani Sajja --- components/common/src/templating/hooks.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/components/common/src/templating/hooks.rs b/components/common/src/templating/hooks.rs index d2ad20aa95..a336f82da7 100644 --- a/components/common/src/templating/hooks.rs +++ b/components/common/src/templating/hooks.rs @@ -633,8 +633,11 @@ mod tests { context::RenderContext, package::Pkg, test_helpers::*}; - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), - all(target_os = "windows", target_arch = "x86_64"),)))] + #[cfg(not(any( + all(target_os = "linux", any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64"), + )))] + use habitat_core::package::metadata::MetaFile; use habitat_core::{package::{PackageIdent, PackageInstall}, service::ServiceGroup}; @@ -865,6 +868,18 @@ echo "The message is Hola Mundo" PathBuf::from("/tmp"), PathBuf::from("/tmp")); + // Platforms without standard package support require all packages to be native packages + #[cfg(not(any( + all(target_os = "linux", any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64") + )))] + { + tokio::fs::create_dir_all(pkg_install.installed_path()).await + .unwrap(); + create_with_content(pkg_install.installed_path() + .join(MetaFile::PackageType.to_string()), + "native"); + } let pkg = Pkg::from_install(&pkg_install).await .expect("Could not create package!"); From 761a0e199c8fc7ec22548628b51fcacfa9d01251 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 7 Oct 2024 21:40:45 +0530 Subject: [PATCH 35/55] fix fmt Signed-off-by: Phani Sajja --- components/common/src/templating/hooks.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/components/common/src/templating/hooks.rs b/components/common/src/templating/hooks.rs index a336f82da7..6ca15bd45d 100644 --- a/components/common/src/templating/hooks.rs +++ b/components/common/src/templating/hooks.rs @@ -633,10 +633,9 @@ mod tests { context::RenderContext, package::Pkg, test_helpers::*}; - #[cfg(not(any( - all(target_os = "linux", any(target_arch = "x86_64", target_arch = "aarch64")), - all(target_os = "windows", target_arch = "x86_64"), - )))] + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64"),)))] use habitat_core::package::metadata::MetaFile; use habitat_core::{package::{PackageIdent, PackageInstall}, @@ -869,10 +868,9 @@ echo "The message is Hola Mundo" PathBuf::from("/tmp")); // Platforms without standard package support require all packages to be native packages - #[cfg(not(any( - all(target_os = "linux", any(target_arch = "x86_64", target_arch = "aarch64")), - all(target_os = "windows", target_arch = "x86_64") - )))] + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64"))))] { tokio::fs::create_dir_all(pkg_install.installed_path()).await .unwrap(); From d0133f1f14107174e7f518557089cc502230c1d3 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 8 Oct 2024 16:56:48 +0530 Subject: [PATCH 36/55] updage agent Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 56 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 8e101b6c8c..5d1f54f879 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -113,7 +113,7 @@ steps: - label: "[unit] :linux: aarch64 ignored" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -139,7 +139,7 @@ steps: - label: "[unit] :linux: aarch64 builder-api-client" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -163,7 +163,7 @@ steps: - label: "[unit] :linux: aarch64 butterfly lock_as_mutex" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -187,7 +187,7 @@ steps: - label: "[unit] :linux: aarch64 butterfly lock_as_rwlock" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -218,7 +218,7 @@ steps: - label: "[unit] :linux: aarch64 common" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -244,7 +244,7 @@ steps: - label: "[unit] :linux: aarch64 core" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -267,7 +267,7 @@ steps: - label: "[unit] :linux: aarch64 core functional" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -290,7 +290,7 @@ steps: - label: "[unit] :linux: aarch64 http-client" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -316,7 +316,7 @@ steps: - label: "[unit] :linux: aarch64 hab" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -342,7 +342,7 @@ steps: - label: "[unit] :linux: aarch64 launcher-client" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -368,7 +368,7 @@ steps: - label: "[unit] :linux: aarch64 launcher-protocol" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -394,7 +394,7 @@ steps: - label: "[unit] :linux: aarch64 pkg-export-container" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -420,7 +420,7 @@ steps: - label: "[unit] :linux: aarch64 pkg-export-tar" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -446,7 +446,7 @@ steps: - label: "[unit] :linux: aarch64 sup lock_as_rwlock" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -470,7 +470,7 @@ steps: - label: "[unit] :linux: aarch64 sup lock_as_mutex" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -494,7 +494,7 @@ steps: - label: "[unit] :linux: aarch64 sup-client" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -520,7 +520,7 @@ steps: - label: "[unit] :linux: aarch64 sup-protocol" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -858,7 +858,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -889,7 +889,7 @@ steps: # This build currently fails due to a circular dependency in 'LTS-2024' and needs investigation. - label: "[build] :linux: aarch64 backline" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -916,7 +916,7 @@ steps: - label: "[build] :linux: aarch64 hab" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -943,7 +943,7 @@ steps: - label: "[build] :linux: aarch64 launcher" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -974,7 +974,7 @@ steps: # This build currently fails due to a dependency on 'pkg-export-container' - label: "[build] :linux: aarch64 pkg-cfize" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1002,7 +1002,7 @@ steps: # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. - label: "[build] :linux: aarch64 pkg-export-container" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1029,7 +1029,7 @@ steps: - label: "[build] :linux: aarch64 pkg-export-tar" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1056,7 +1056,7 @@ steps: - label: "[build] :linux: aarch64 pkg-mesosize" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1083,7 +1083,7 @@ steps: - label: "[build] :linux: aarch64 plan-build" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1110,7 +1110,7 @@ steps: - label: "[build] :linux: aarch64 studio" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1137,7 +1137,7 @@ steps: - label: "[build] :linux: aarch64 sup" agents: - queue: docker-linux-amazon-2023-arm64 + queue:default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" From 3d8c8cf2bf920812c20db20cbebe6b8cf013fb8d Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 8 Oct 2024 18:55:59 +0530 Subject: [PATCH 37/55] adds space Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 56 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 5d1f54f879..3726926c25 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -113,7 +113,7 @@ steps: - label: "[unit] :linux: aarch64 ignored" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -139,7 +139,7 @@ steps: - label: "[unit] :linux: aarch64 builder-api-client" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -163,7 +163,7 @@ steps: - label: "[unit] :linux: aarch64 butterfly lock_as_mutex" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -187,7 +187,7 @@ steps: - label: "[unit] :linux: aarch64 butterfly lock_as_rwlock" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -218,7 +218,7 @@ steps: - label: "[unit] :linux: aarch64 common" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -244,7 +244,7 @@ steps: - label: "[unit] :linux: aarch64 core" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -267,7 +267,7 @@ steps: - label: "[unit] :linux: aarch64 core functional" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -290,7 +290,7 @@ steps: - label: "[unit] :linux: aarch64 http-client" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -316,7 +316,7 @@ steps: - label: "[unit] :linux: aarch64 hab" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -342,7 +342,7 @@ steps: - label: "[unit] :linux: aarch64 launcher-client" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -368,7 +368,7 @@ steps: - label: "[unit] :linux: aarch64 launcher-protocol" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -394,7 +394,7 @@ steps: - label: "[unit] :linux: aarch64 pkg-export-container" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -420,7 +420,7 @@ steps: - label: "[unit] :linux: aarch64 pkg-export-tar" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -446,7 +446,7 @@ steps: - label: "[unit] :linux: aarch64 sup lock_as_rwlock" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -470,7 +470,7 @@ steps: - label: "[unit] :linux: aarch64 sup lock_as_mutex" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -494,7 +494,7 @@ steps: - label: "[unit] :linux: aarch64 sup-client" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -520,7 +520,7 @@ steps: - label: "[unit] :linux: aarch64 sup-protocol" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -858,7 +858,7 @@ steps: - label: "[build] :linux: aarch64 test-probe" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -889,7 +889,7 @@ steps: # This build currently fails due to a circular dependency in 'LTS-2024' and needs investigation. - label: "[build] :linux: aarch64 backline" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -916,7 +916,7 @@ steps: - label: "[build] :linux: aarch64 hab" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -943,7 +943,7 @@ steps: - label: "[build] :linux: aarch64 launcher" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -974,7 +974,7 @@ steps: # This build currently fails due to a dependency on 'pkg-export-container' - label: "[build] :linux: aarch64 pkg-cfize" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1002,7 +1002,7 @@ steps: # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. - label: "[build] :linux: aarch64 pkg-export-container" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1029,7 +1029,7 @@ steps: - label: "[build] :linux: aarch64 pkg-export-tar" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1056,7 +1056,7 @@ steps: - label: "[build] :linux: aarch64 pkg-mesosize" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1083,7 +1083,7 @@ steps: - label: "[build] :linux: aarch64 plan-build" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1110,7 +1110,7 @@ steps: - label: "[build] :linux: aarch64 studio" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" @@ -1137,7 +1137,7 @@ steps: - label: "[build] :linux: aarch64 sup" agents: - queue:default-privileged-aarch64 + queue: default-privileged-aarch64 env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" From ff3f02face012e78ff6d1f95c0b6e16d15da2c8f Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Thu, 17 Oct 2024 11:16:18 +0530 Subject: [PATCH 38/55] should be an error Signed-off-by: Phani Sajja --- components/core/src/os/net.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/core/src/os/net.rs b/components/core/src/os/net.rs index 22e5395444..1d68f04245 100644 --- a/components/core/src/os/net.rs +++ b/components/core/src/os/net.rs @@ -65,6 +65,4 @@ fn test_fqdn_lookup() { fn test_fqdn_lookup_err() { let fqdn = lookup_fqdn(""); assert!(fqdn.is_err(), "Should be an Err()"); - assert_eq!(format!("{}", fqdn.unwrap_err()), - "failed to lookup address information: Name or service not known"); } From e8da464090afccaa05fe91c6a6663b6086598db3 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Thu, 17 Oct 2024 23:18:15 +0530 Subject: [PATCH 39/55] aarch64-linux has standard pkgs Signed-off-by: Phani Sajja --- components/common/src/templating.rs | 3 ++- components/common/src/templating/config.rs | 3 ++- components/sup/src/manager/service/hooks.rs | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/components/common/src/templating.rs b/components/common/src/templating.rs index 2f1bfa4483..bc6158a239 100644 --- a/components/common/src/templating.rs +++ b/components/common/src/templating.rs @@ -420,7 +420,8 @@ mod test { let pkg_install = PackageInstall::new_from_parts(pg_id, root.clone(), root.clone(), root.clone()); // Platforms without standard package support require all packages to be native packages - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), all(target_os = "windows", target_arch = "x86_64"))))] { tokio::fs::create_dir_all(pkg_install.installed_path()).await diff --git a/components/common/src/templating/config.rs b/components/common/src/templating/config.rs index bfb43e29eb..65d45bca37 100644 --- a/components/common/src/templating/config.rs +++ b/components/common/src/templating/config.rs @@ -1151,7 +1151,8 @@ mod test { "config message is {{cfg.message}}"); // Platforms without standard package support require all packages to be native packages - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), all(target_os = "windows", target_arch = "x86_64"))))] { create_with_content(pkg_dir.join(MetaFile::PackageType.to_string()), "native"); diff --git a/components/sup/src/manager/service/hooks.rs b/components/sup/src/manager/service/hooks.rs index 385d374fa4..e9646286f0 100644 --- a/components/sup/src/manager/service/hooks.rs +++ b/components/sup/src/manager/service/hooks.rs @@ -797,8 +797,8 @@ mod tests { PathBuf::from("/tmp"), PathBuf::from("/tmp")); // Platforms without standard package support require all packages to be native packages - // TODO: This is currently also needed on aarch64-linux until we publish official packages - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), all(target_os = "windows", target_arch = "x86_64"))))] { tokio::fs::create_dir_all(pkg_install.installed_path()).await From 798661fcdcbabc298666a0c44971dd85e01a22a0 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Thu, 17 Oct 2024 23:39:37 +0530 Subject: [PATCH 40/55] fix unused import Signed-off-by: Phani Sajja --- components/common/src/templating.rs | 3 ++- components/common/src/templating/config.rs | 3 ++- components/sup/src/manager/service/hooks.rs | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/components/common/src/templating.rs b/components/common/src/templating.rs index bc6158a239..7b44c1a6ee 100644 --- a/components/common/src/templating.rs +++ b/components/common/src/templating.rs @@ -178,7 +178,8 @@ mod test { FS_ROOT_PATH}, package::PackageIdent}, templating::test_helpers::*}; - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), all(target_os = "windows", target_arch = "x86_64"),)))] use habitat_core::package::metadata::MetaFile; use std::{collections::BTreeMap, diff --git a/components/common/src/templating/config.rs b/components/common/src/templating/config.rs index 65d45bca37..57ad3f8708 100644 --- a/components/common/src/templating/config.rs +++ b/components/common/src/templating/config.rs @@ -614,7 +614,8 @@ mod test { PackageInstall}}, templating::{context::RenderContext, test_helpers::*}}; - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), all(target_os = "windows", target_arch = "x86_64"),)))] use hcore::package::metadata::MetaFile; use std::{env, diff --git a/components/sup/src/manager/service/hooks.rs b/components/sup/src/manager/service/hooks.rs index e9646286f0..1433c5c419 100644 --- a/components/sup/src/manager/service/hooks.rs +++ b/components/sup/src/manager/service/hooks.rs @@ -732,8 +732,9 @@ mod tests { types::{GossipListenAddr, HttpListenAddr, ListenCtlAddr}}; - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), - all(target_os = "windows", target_arch = "x86_64"))))] + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64"),)))] use habitat_core::package::metadata::MetaFile; use habitat_core::{crypto::keys::KeyCache, fs::CACHE_KEY_PATH, From 973758cf506691ccaf2e75819ec85a60e4d66ae3 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 18 Oct 2024 00:10:44 +0530 Subject: [PATCH 41/55] remove cfize, mesomise and fix container plan Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 27 ------------------- .../pkg-export-container/habitat/plan.sh | 8 ++++++ 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 3726926c25..66bf7c79e4 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -971,19 +971,6 @@ steps: # automatic: # limit: 1 - # This build currently fails due to a dependency on 'pkg-export-container' - - label: "[build] :linux: aarch64 pkg-cfize" - agents: - queue: default-privileged-aarch64 - env: - HAB_LICENSE: "accept-no-persist" - HAB_BLDR_CHANNEL: "LTS-2024" - BUILD_PKG_TARGET: "aarch64-linux" - command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-cfize - retry: - automatic: - limit: 1 - label: "[build] :linux: pkg-export-container" env: @@ -999,7 +986,6 @@ steps: automatic: limit: 1 - # This build currently fails due to a dependency on 'core/buildah', which is not available in 'LTS-2024'. - label: "[build] :linux: aarch64 pkg-export-container" agents: queue: default-privileged-aarch64 @@ -1054,19 +1040,6 @@ steps: automatic: limit: 1 - - label: "[build] :linux: aarch64 pkg-mesosize" - agents: - queue: default-privileged-aarch64 - env: - HAB_LICENSE: "accept-no-persist" - HAB_BLDR_CHANNEL: "LTS-2024" - BUILD_PKG_TARGET: "aarch64-linux" - command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-mesosize - retry: - automatic: - limit: 1 - - label: "[build] :linux: plan-build" env: HAB_LICENSE: "accept-no-persist" diff --git a/components/pkg-export-container/habitat/plan.sh b/components/pkg-export-container/habitat/plan.sh index 3fda523b9b..d9e22a27c1 100644 --- a/components/pkg-export-container/habitat/plan.sh +++ b/components/pkg-export-container/habitat/plan.sh @@ -58,6 +58,14 @@ do_before() { do_prepare() { _common_prepare + # With the musl target, the ring crate is looking for aarch64-linux-musl-gcc, + # but the core/musl package provides musl-gcc. This workaround is necessary until the appropriate changes are made to core/musl for aarch64. + if [[ "${pkg_target%%-*}" == "aarch64" ]]; then + if [[ ! -r "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" ]]; then + ln -sv "$(pkg_path_for musl)/bin/musl-gcc" "$(pkg_path_for musl)/bin/aarch64-linux-musl-gcc" + fi + fi + export rustc_target="${pkg_target%%-*}-unknown-linux-musl" build_line "Setting rustc_target=$rustc_target" From 4b121e87b9dbd5a340ae0d083847be8b8f8c3ce2 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 18 Oct 2024 14:51:41 +0530 Subject: [PATCH 42/55] set interpreter for aarch64 tests Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/run_cargo_test-aarch64.sh | 4 ++++ components/sup/tests/utils/fs.rs | 9 +++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index 55ad0fc444..0d0088f4b3 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -67,6 +67,10 @@ if [[ -n ${component:-} ]]; then cd "components/$component" fi +# We do not have any packages in the stable channel for aarch64 and probably never will. +# Set the HAB_INTERPRETER_IDENT to point to LTS-2024 to proceed with the tests. +export HAB_INTERPRETER_IDENT="core/busybox-static/1.36.1/20240805133911" + # Always add `--quiet` to avoid the noise of compilation in test output. # The invocation to this script can add `--format pretty` to the test runner # args (that is, after --, like --nocapture and --test-threads) if the names diff --git a/components/sup/tests/utils/fs.rs b/components/sup/tests/utils/fs.rs index e81c9b8ef7..a8f866b2a2 100644 --- a/components/sup/tests/utils/fs.rs +++ b/components/sup/tests/utils/fs.rs @@ -365,12 +365,9 @@ async fn write_default_metafiles(hab_root: &HabRoot, // Write metafiles to convert the package to a native package on platforms without package // support - // TODO: We intentionally use native packages on aarch64-linux platform as the interpreter - // packages core/busybox-static are not yet available publicly on builder. This will cause - // The package to fail when the supervisor attempts to load it. We should use standard - // packages on aarch64-linux once the interpreter package is made publicly available. - #[cfg(not(any(all(target_os = "linux", any(target_arch = "x86_64")), - all(target_os = "windows", target_arch = "x86_64"))))] + #[cfg(not(any(all(target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64"))))] { let pkg_type_metafile = hab_root.pkg_type_path(pkg_origin, pkg_name); write_metafile(pkg_type_metafile, "native").await?; From aaa5bdd44e5778f4a77623a60898b9520c823c1e Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 18 Oct 2024 14:58:16 +0530 Subject: [PATCH 43/55] fix rustfmt Signed-off-by: Phani Sajja --- components/sup/tests/utils/fs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/sup/tests/utils/fs.rs b/components/sup/tests/utils/fs.rs index a8f866b2a2..6d15ad639b 100644 --- a/components/sup/tests/utils/fs.rs +++ b/components/sup/tests/utils/fs.rs @@ -366,8 +366,8 @@ async fn write_default_metafiles(hab_root: &HabRoot, // Write metafiles to convert the package to a native package on platforms without package // support #[cfg(not(any(all(target_os = "linux", - any(target_arch = "x86_64", target_arch = "aarch64")), - all(target_os = "windows", target_arch = "x86_64"))))] + any(target_arch = "x86_64", target_arch = "aarch64")), + all(target_os = "windows", target_arch = "x86_64"))))] { let pkg_type_metafile = hab_root.pkg_type_path(pkg_origin, pkg_name); write_metafile(pkg_type_metafile, "native").await?; From 2a26e83f46ac60f29d5725e3f0fa78e7adea4733 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 25 Oct 2024 17:34:01 +0530 Subject: [PATCH 44/55] aarch64 packages are updated against LTS-2024 Signed-off-by: Phani Sajja --- .expeditor/scripts/shared.sh | 10 +----- .../scripts/verify/build_package-aarch64.sh | 26 ++++++++++++++ .expeditor/scripts/verify/build_package.sh | 5 --- .../scripts/verify/run_cargo_test-aarch64.sh | 4 +-- .expeditor/verify.pipeline.yml | 34 ++++++++++++++----- 5 files changed, 54 insertions(+), 25 deletions(-) create mode 100755 .expeditor/scripts/verify/build_package-aarch64.sh diff --git a/.expeditor/scripts/shared.sh b/.expeditor/scripts/shared.sh index 7595f24a44..1857a26988 100755 --- a/.expeditor/scripts/shared.sh +++ b/.expeditor/scripts/shared.sh @@ -21,15 +21,7 @@ curlbash_hab() { rm -rf /hab/pkgs/core/hab/0.82.0 fi - # Use the temporarily uploaded Habitat package from the 'aarch64-linux' channel for aarch64. - # Once hab is released in the LTS channel, we can use either HAB_BLDR_CHANNEL or HAB_REFRESH_CHANNEL to install hab. - install_args="-t $pkg_target" - if [ "$pkg_target" = "aarch64-linux" ]; then - install_args+=" -c aarch64-linux -v 1.6.706" - fi - echo "install_args: $install_args" - # shellcheck disable=SC2086 - curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- ${install_args} + curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$pkg_target" case "${pkg_target}" in x86_64-linux | aarch64-linux | x86_64-linux-kernel2) hab_binary="/bin/hab" diff --git a/.expeditor/scripts/verify/build_package-aarch64.sh b/.expeditor/scripts/verify/build_package-aarch64.sh new file mode 100755 index 0000000000..bbf318c4c9 --- /dev/null +++ b/.expeditor/scripts/verify/build_package-aarch64.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eou pipefail + +package_path=${1?package_path argument required} + +# Install hab from a temporarily uploaded aarch64 package +curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t $BUILD_PKG_TARGET -c $HAB_FALLBACK_CHANNEL -v 1.6.1178 + +# Since we are only verifying we don't have build failures, make everything +# temp! +export HAB_ORIGIN +HAB_ORIGIN=throwaway +# let's make a selfcontained tempdir for this job +export JOB_TEMP_ROOT +JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX) +export HAB_CACHE_KEY_PATH +HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" + +echo "--- :key: Generating fake origin key" +hab origin key generate +echo "--- :hab: Running hab pkg build for $package_path" +# Install the temporarily built hab-studio. +# Once hab is released in the LTS channel, this step may no longer be required. +hab pkg install core/hab-studio -c $HAB_FALLBACK_CHANNEL +hab pkg build "$package_path" diff --git a/.expeditor/scripts/verify/build_package.sh b/.expeditor/scripts/verify/build_package.sh index c37e3a4d2d..61e81d6cd2 100755 --- a/.expeditor/scripts/verify/build_package.sh +++ b/.expeditor/scripts/verify/build_package.sh @@ -21,10 +21,5 @@ HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" echo "--- :key: Generating fake origin key" ${hab_binary} origin key generate -# Install the hab-studio from the 'aarch64-linux' channel. -# Once hab is released in the LTS channel, we can use either HAB_BLDR_CHANNEL or HAB_REFRESH_CHANNEL to install the studio. -if [ "$BUILD_PKG_TARGET" = "aarch64-linux" ]; then - ${hab_binary} pkg install core/hab-studio/1.6.706 --channel aarch64-linux -fi echo "--- :hab: Running hab pkg build for $package_path" ${hab_binary} pkg build "$package_path" diff --git a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh index 0d0088f4b3..90ac5de6c6 100755 --- a/.expeditor/scripts/verify/run_cargo_test-aarch64.sh +++ b/.expeditor/scripts/verify/run_cargo_test-aarch64.sh @@ -2,8 +2,8 @@ set -eou pipefail -# Install hab -curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t aarch64-linux -c aarch64-linux -v 1.6.706 +# Install hab from a temporarily uploaded aarch64 package +curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t aarch64-linux -c lts24-aarch64-linux -v 1.6.1178 # shellcheck source=.expeditor/scripts/shared.sh source .expeditor/scripts/verify/shared.sh diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 66bf7c79e4..cdb1755c32 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -862,9 +862,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh test-services/test-probe + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh test-services/test-probe retry: automatic: limit: 1 @@ -893,9 +895,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/backline + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/backline retry: automatic: limit: 1 @@ -920,9 +924,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/hab + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/hab retry: automatic: limit: 1 @@ -947,9 +953,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/launcher + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/launcher retry: automatic: limit: 1 @@ -994,7 +1002,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-export-container + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/pkg-export-container retry: automatic: limit: 1 @@ -1019,9 +1027,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/pkg-export-tar + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/pkg-export-tar retry: automatic: limit: 1 @@ -1060,9 +1070,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/plan-build + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/plan-build retry: automatic: limit: 1 @@ -1087,9 +1099,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/studio + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/studio retry: automatic: limit: 1 @@ -1114,9 +1128,11 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - - sudo -E .expeditor/scripts/verify/build_package.sh components/sup + - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/sup retry: automatic: limit: 1 From a53ce9c8a27ad76cd0f62239ddf3737a360d26f9 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Fri, 25 Oct 2024 18:15:11 +0530 Subject: [PATCH 45/55] fix shellcheck Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/build_package-aarch64.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.expeditor/scripts/verify/build_package-aarch64.sh b/.expeditor/scripts/verify/build_package-aarch64.sh index bbf318c4c9..420a856b33 100755 --- a/.expeditor/scripts/verify/build_package-aarch64.sh +++ b/.expeditor/scripts/verify/build_package-aarch64.sh @@ -5,7 +5,7 @@ set -eou pipefail package_path=${1?package_path argument required} # Install hab from a temporarily uploaded aarch64 package -curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t $BUILD_PKG_TARGET -c $HAB_FALLBACK_CHANNEL -v 1.6.1178 +curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | sudo bash -s -- -t "$BUILD_PKG_TARGET" -c "$HAB_FALLBACK_CHANNEL" -v 1.6.1178 # Since we are only verifying we don't have build failures, make everything # temp! @@ -22,5 +22,5 @@ hab origin key generate echo "--- :hab: Running hab pkg build for $package_path" # Install the temporarily built hab-studio. # Once hab is released in the LTS channel, this step may no longer be required. -hab pkg install core/hab-studio -c $HAB_FALLBACK_CHANNEL +hab pkg install core/hab-studio -c "$HAB_FALLBACK_CHANNEL" hab pkg build "$package_path" From 339d7e93e6c78b9bf51a699888e2aabcfc4db62f Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 28 Oct 2024 13:08:10 +0530 Subject: [PATCH 46/55] set missing fallback channel Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index cdb1755c32..a9a35814a8 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -1000,6 +1000,8 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/pkg-export-container From d4e011b5569d85993273dbe9bb8ac3c68b4c2a12 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 28 Oct 2024 13:31:16 +0530 Subject: [PATCH 47/55] add aarch64 to the release pipeline Signed-off-by: Phani Sajja --- .expeditor/release_habitat.pipeline.yml | 82 +++++++++++++++++++ .../package_and_upload_binary.sh | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/.expeditor/release_habitat.pipeline.yml b/.expeditor/release_habitat.pipeline.yml index d121e12111..71d60f4979 100644 --- a/.expeditor/release_habitat.pipeline.yml +++ b/.expeditor/release_habitat.pipeline.yml @@ -38,6 +38,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh hab + - label: "[:linux: :two: build hab]" command: - .expeditor/scripts/release_habitat/build_component.sh hab @@ -111,6 +120,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab-plan-build]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh plan-build + - label: "[:linux: :two: build hab-plan-build]" command: - .expeditor/scripts/release_habitat/build_component.sh plan-build @@ -146,6 +164,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab-backline]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh backline + - label: "[:linux: :two: build hab-backline]" command: - .expeditor/scripts/release_habitat/build_component.sh backline @@ -170,6 +197,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab-studio]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh studio + - label: "[:linux: :two: build hab-studio]" command: - .expeditor/scripts/release_habitat/build_component.sh studio @@ -208,6 +244,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 launcher]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh launcher + - label: "[:linux: :two: build launcher]" command: - .expeditor/scripts/release_habitat/build_component.sh launcher @@ -241,6 +286,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab-sup]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh sup + - label: "[:linux: :two: build hab-sup]" command: - .expeditor/scripts/release_habitat/build_component.sh sup @@ -274,6 +328,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab-pkg-export-container]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-container + - label: "[:windows: build hab-pkg-export-container]" command: - powershell .expeditor/scripts/release_habitat/build_component.ps1 pkg-export-container @@ -295,6 +358,15 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: build aarch64 hab-pkg-export-tar]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar + - label: "[:linux: :two: build hab-pkg-export-tar]" command: - .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar @@ -352,6 +424,16 @@ steps: environment: - BUILD_PKG_TARGET=x86_64-linux + - label: "[:linux: upload hab aarch64 binary]" + agents: + queue: default-privileged-aarch64 + env: + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + BUILD_PKG_TARGET: "aarch64-linux" + command: + - sudo -E .expeditor/scripts/release_habitat/package_and_upload_binary.sh + if: build.creator.name == 'Chef Expeditor' || build.env("UPLOAD_AND_PROMOTE") == 'true' + - label: "[:linux: :two: upload hab binary]" command: - .expeditor/scripts/release_habitat/package_and_upload_binary.sh diff --git a/.expeditor/scripts/release_habitat/package_and_upload_binary.sh b/.expeditor/scripts/release_habitat/package_and_upload_binary.sh index 31511e6d8d..9274689cfd 100755 --- a/.expeditor/scripts/release_habitat/package_and_upload_binary.sh +++ b/.expeditor/scripts/release_habitat/package_and_upload_binary.sh @@ -14,7 +14,7 @@ channel=$(get_release_channel) echo "--- Channel: $channel - bldr url: $HAB_BLDR_URL" declare -g hab_binary -install_release_channel_hab_binary "x86_64-linux" +install_release_channel_hab_binary "$BUILD_PKG_TARGET" import_gpg_keys From f5eb00557079f4aa72e4ad10b21f144ac6af8ee5 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Mon, 28 Oct 2024 18:28:47 +0530 Subject: [PATCH 48/55] temporarily prefer local hab* pkgs Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index a9a35814a8..aea94486af 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -897,6 +897,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" HAB_REFRESH_CHANNEL: "LTS-2024" HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_PREFER_LOCAL_CHEF_DEPS: "true" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/backline @@ -1074,6 +1075,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" HAB_REFRESH_CHANNEL: "LTS-2024" HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_PREFER_LOCAL_CHEF_DEPS: "true" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/plan-build @@ -1103,6 +1105,7 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" HAB_REFRESH_CHANNEL: "LTS-2024" HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_PREFER_LOCAL_CHEF_DEPS: "true" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/studio From 1385d03b3c074439b01bd6e93314d4968faa5815 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 29 Oct 2024 18:30:34 +0530 Subject: [PATCH 49/55] use core origin in verity Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/build_package-aarch64.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.expeditor/scripts/verify/build_package-aarch64.sh b/.expeditor/scripts/verify/build_package-aarch64.sh index 420a856b33..4d3211530d 100755 --- a/.expeditor/scripts/verify/build_package-aarch64.sh +++ b/.expeditor/scripts/verify/build_package-aarch64.sh @@ -10,15 +10,15 @@ curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/in # Since we are only verifying we don't have build failures, make everything # temp! export HAB_ORIGIN -HAB_ORIGIN=throwaway +HAB_ORIGIN=core # let's make a selfcontained tempdir for this job export JOB_TEMP_ROOT JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX) export HAB_CACHE_KEY_PATH HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" -echo "--- :key: Generating fake origin key" -hab origin key generate +echo "--- :key: Generating temporary origin key" +hab origin key generate "$HAB_ORIGIN" echo "--- :hab: Running hab pkg build for $package_path" # Install the temporarily built hab-studio. # Once hab is released in the LTS channel, this step may no longer be required. From 53dff918e384226480cf231691b3755fa970aff2 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 29 Oct 2024 18:45:39 +0530 Subject: [PATCH 50/55] prefer local installed Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/build_package-aarch64.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.expeditor/scripts/verify/build_package-aarch64.sh b/.expeditor/scripts/verify/build_package-aarch64.sh index 4d3211530d..13b523c9bf 100755 --- a/.expeditor/scripts/verify/build_package-aarch64.sh +++ b/.expeditor/scripts/verify/build_package-aarch64.sh @@ -17,6 +17,9 @@ JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX) export HAB_CACHE_KEY_PATH HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" +export HAB_FALLBACK_CHANNEL +export HAB_PREFER_LOCAL_CHEF_DEPS="true" + echo "--- :key: Generating temporary origin key" hab origin key generate "$HAB_ORIGIN" echo "--- :hab: Running hab pkg build for $package_path" From f934eb7bdeaeadca91a6fc909b4cdea61c3df800 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 29 Oct 2024 19:00:28 +0530 Subject: [PATCH 51/55] another try Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index aea94486af..70624ccf80 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -895,8 +895,8 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" - HAB_REFRESH_CHANNEL: "LTS-2024" - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_REFRESH_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" HAB_PREFER_LOCAL_CHEF_DEPS: "true" BUILD_PKG_TARGET: "aarch64-linux" command: From c102d2446269a1aa998103649048bd197a827ad3 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 30 Oct 2024 13:40:56 +0530 Subject: [PATCH 52/55] set studio fallback channel Signed-off-by: Phani Sajja --- .expeditor/scripts/verify/build_package-aarch64.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.expeditor/scripts/verify/build_package-aarch64.sh b/.expeditor/scripts/verify/build_package-aarch64.sh index 13b523c9bf..6012ffa00a 100755 --- a/.expeditor/scripts/verify/build_package-aarch64.sh +++ b/.expeditor/scripts/verify/build_package-aarch64.sh @@ -17,7 +17,8 @@ JOB_TEMP_ROOT=$(mktemp -d /tmp/job-root-XXXXXX) export HAB_CACHE_KEY_PATH HAB_CACHE_KEY_PATH="$JOB_TEMP_ROOT/keys" -export HAB_FALLBACK_CHANNEL +export HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL +HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL="$HAB_FALLBACK_CHANNEL" export HAB_PREFER_LOCAL_CHEF_DEPS="true" echo "--- :key: Generating temporary origin key" From 82fc2623d7a7a312c836b4af25923c7c05bf7636 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 30 Oct 2024 17:23:57 +0530 Subject: [PATCH 53/55] remove unneeded target Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 70624ccf80..041a6ffa20 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -9,15 +9,6 @@ steps: # Linting! ####################################################################### - - label: "[unit] :linux: ARM Platform Support" - command: - - .expeditor/scripts/verify/run_native_cargo_test.sh - agents: - queue: docker-linux-arm64 - plugins: - - docker#v3.13.0: - image: "rust:1.62.1-buster" - - label: "[lint] :linux: :bash: Shellcheck" command: - .expeditor/scripts/verify/shellcheck.sh From c9befb7e2a2bc85381c96bac6c834dae59439d31 Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Wed, 30 Oct 2024 17:50:04 +0530 Subject: [PATCH 54/55] correct fallback channel Signed-off-by: Phani Sajja --- .expeditor/verify.pipeline.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 041a6ffa20..91abd2696c 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -886,9 +886,8 @@ steps: env: HAB_LICENSE: "accept-no-persist" HAB_BLDR_CHANNEL: "LTS-2024" - HAB_REFRESH_CHANNEL: "lts24-aarch64-linux" - HAB_FALLBACK_CHANNEL: "LTS-2024" - HAB_PREFER_LOCAL_CHEF_DEPS: "true" + HAB_REFRESH_CHANNEL: "LTS-2024" + HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/backline @@ -1066,7 +1065,6 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" HAB_REFRESH_CHANNEL: "LTS-2024" HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" - HAB_PREFER_LOCAL_CHEF_DEPS: "true" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/plan-build @@ -1096,7 +1094,6 @@ steps: HAB_BLDR_CHANNEL: "LTS-2024" HAB_REFRESH_CHANNEL: "LTS-2024" HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" - HAB_PREFER_LOCAL_CHEF_DEPS: "true" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/verify/build_package-aarch64.sh components/studio From 5866c8c7fc2dc23e9625be5ae648eb7e576bf01e Mon Sep 17 00:00:00 2001 From: Phani Sajja Date: Tue, 12 Nov 2024 21:21:01 +0530 Subject: [PATCH 55/55] update fallback channel and remove arm binary upload Signed-off-by: Phani Sajja --- .expeditor/release_habitat.pipeline.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.expeditor/release_habitat.pipeline.yml b/.expeditor/release_habitat.pipeline.yml index 71d60f4979..66bdce4ced 100644 --- a/.expeditor/release_habitat.pipeline.yml +++ b/.expeditor/release_habitat.pipeline.yml @@ -22,12 +22,6 @@ steps: # Release! ####################################################################### - - label: "[:linux: build hab (ARM)]" - command: - - .expeditor/scripts/release_habitat/package_and_upload_arm_binary.sh - agents: - queue: docker-linux-arm64 - - label: "[:linux: build hab]" command: - .expeditor/scripts/release_habitat/build_component.sh hab @@ -42,7 +36,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh hab @@ -124,7 +118,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh plan-build @@ -168,7 +162,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh backline @@ -201,7 +195,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh studio @@ -248,7 +242,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh launcher @@ -290,7 +284,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh sup @@ -332,7 +326,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-container @@ -362,7 +356,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/build_component.sh pkg-export-tar @@ -428,7 +422,7 @@ steps: agents: queue: default-privileged-aarch64 env: - HAB_FALLBACK_CHANNEL: "lts24-aarch64-linux" + HAB_FALLBACK_CHANNEL: "LTS-2024" BUILD_PKG_TARGET: "aarch64-linux" command: - sudo -E .expeditor/scripts/release_habitat/package_and_upload_binary.sh