From 226403da0bd492d811eb8d685685bc70086da250 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Fri, 29 May 2020 19:35:06 +0200 Subject: [PATCH 01/22] Fix 64 bit ARM builders --- deb-arm/Dockerfile | 12 ++++++++++-- rpm-arm/Dockerfile | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/deb-arm/Dockerfile b/deb-arm/Dockerfile index 5cb971e9..463b8812 100644 --- a/deb-arm/Dockerfile +++ b/deb-arm/Dockerfile @@ -18,7 +18,6 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION -ENV GIMME_ARCH $TARGET_ARCH RUN apt-get update && apt-get install -y fakeroot curl git procps bzip2 \ @@ -44,7 +43,16 @@ RUN pip install invoke distro==1.4.0 awscli==1.16.240 # Gimme RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme RUN chmod +x /bin/gimme -RUN gimme $GIMME_GO_VERSION + +# GIMME_ARCH = GOARCH, so must be a valid entry from `goarchlist` here: +# https://github.com/golang/go/blob/master/src/go/build/syslist.go +# Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 +RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ + $GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION \ + else \ + $GIMME_ARCH=arm gimme $GIMME_GO_VERSION \ + fi + COPY ./gobin.sh /etc/profile.d/ # CMake. Pre-built using the build-cmake.sh script, to speed-up docker build. diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index adda8d2f..7d54aebe 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -18,7 +18,6 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION -ENV GIMME_ARCH $TARGET_ARCH # configure yum and rpm for running on non-armv7l architectures RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then echo "armhfp" > /etc/yum/vars/basearch && \ @@ -49,7 +48,16 @@ RUN pip install invoke distro==1.4.0 awscli==1.16.240 # Gimme RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme RUN chmod +x /bin/gimme -RUN gimme $GIMME_GO_VERSION + +# GIMME_ARCH = GOARCH, so must be a valid entry from `goarchlist` here: +# https://github.com/golang/go/blob/master/src/go/build/syslist.go +# Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 +RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ + $GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION \ + else \ + $GIMME_ARCH=arm gimme $GIMME_GO_VERSION \ + fi + COPY ./gobin.sh /etc/profile.d/ # CMake. Pre-built using the build-cmake.sh script, to speed-up docker build. From 2aa2c0aff9823135ab49ce0ed08975cb3c726516 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Fri, 29 May 2020 20:49:34 +0200 Subject: [PATCH 02/22] fix syntax error --- deb-arm/Dockerfile | 4 ++-- rpm-arm/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deb-arm/Dockerfile b/deb-arm/Dockerfile index 463b8812..e1519c5e 100644 --- a/deb-arm/Dockerfile +++ b/deb-arm/Dockerfile @@ -48,9 +48,9 @@ RUN chmod +x /bin/gimme # https://github.com/golang/go/blob/master/src/go/build/syslist.go # Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ - $GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION \ + GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION ; \ else \ - $GIMME_ARCH=arm gimme $GIMME_GO_VERSION \ + GIMME_ARCH=arm gimme $GIMME_GO_VERSION ; \ fi COPY ./gobin.sh /etc/profile.d/ diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 7d54aebe..85bf4c35 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -53,9 +53,9 @@ RUN chmod +x /bin/gimme # https://github.com/golang/go/blob/master/src/go/build/syslist.go # Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ - $GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION \ + GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION ; \ else \ - $GIMME_ARCH=arm gimme $GIMME_GO_VERSION \ + GIMME_ARCH=arm gimme $GIMME_GO_VERSION ; \ fi COPY ./gobin.sh /etc/profile.d/ From 49bb254768288f2738cd8d85529a2e899df161e3 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Wed, 10 Jun 2020 15:05:19 +0200 Subject: [PATCH 03/22] Add GIMME_ARCH in entrypoint --- entrypoint.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 0b358300..d5f523b7 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,6 +5,11 @@ source /root/.bashrc conda activate ddpy3 || true # We use system python3 on some images, allow this to fail +if [ "$TARGET_ARCH" = "arm64v8" ] ; then + export GIMME_ARCH=arm64 +else + export GIMME_ARCH=arm +fi eval "$(gimme)" exec "$@" From 3c326983aa5704359a8d2190ad5a1db8c52ad70b Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Wed, 10 Jun 2020 16:35:25 +0200 Subject: [PATCH 04/22] Set GIMME_ARCH only for Arm builds --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index d5f523b7..fbda1de0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -7,7 +7,7 @@ conda activate ddpy3 || true # We use system python3 on some images, allow this if [ "$TARGET_ARCH" = "arm64v8" ] ; then export GIMME_ARCH=arm64 -else +elif [ "$TARGET_ARCH" = "arm32v7" ] ; then export GIMME_ARCH=arm fi eval "$(gimme)" From 3f986df06614f3e7b41022e7f2a9e08e8dd428ac Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Wed, 10 Jun 2020 17:14:32 +0200 Subject: [PATCH 05/22] Pass TARGET_ARCH to destination image --- deb-arm/Dockerfile | 2 +- rpm-arm/Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deb-arm/Dockerfile b/deb-arm/Dockerfile index e1519c5e..e3876b42 100644 --- a/deb-arm/Dockerfile +++ b/deb-arm/Dockerfile @@ -18,7 +18,7 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION - +ENV TARGET_ARCH $TARGET_ARCH RUN apt-get update && apt-get install -y fakeroot curl git procps bzip2 \ python2.7-dev build-essential pkg-config tar libsystemd-dev libkrb5-dev \ diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 85bf4c35..54388d61 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -18,6 +18,7 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION +ENV TARGET_ARCH $TARGET_ARCH # configure yum and rpm for running on non-armv7l architectures RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then echo "armhfp" > /etc/yum/vars/basearch && \ From 1e3965ca90136403c841f0f63917e2890e892f26 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Thu, 11 Jun 2020 16:01:23 +0200 Subject: [PATCH 06/22] Test fake armv7l LD_PRELOAD trick --- rpm-arm/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 54388d61..4bf1a7c8 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -33,6 +33,17 @@ RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMa curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ libtool autoconf policycoreutils-python +# This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l +RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then \ + git clone https://github.com/robotpy/fakearmv7l ; \ + cd fakearmv7l ; \ + make ; \ + make install ; \ + cd .. ; \ + rm -rf ./fakearmv7l ; \ + export LD_PRELOAD="/usr/local/lib/libfakearmv7l.so" ; \ + fi + # RVM COPY ./rvm/gpg-keys /gpg-keys RUN gpg --import /gpg-keys/* From 3e58734dbd72a7db1cd93df3ce42691ad25d169d Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Fri, 12 Jun 2020 12:47:53 +0200 Subject: [PATCH 07/22] Use Datadog fork of fakearmv7l --- rpm-arm/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 4bf1a7c8..7a4ee8ea 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -34,14 +34,14 @@ RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMa libtool autoconf policycoreutils-python # This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l +# The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then \ - git clone https://github.com/robotpy/fakearmv7l ; \ + git clone https://github.com/DataDog/fakearmv7l ; \ cd fakearmv7l ; \ make ; \ make install ; \ cd .. ; \ rm -rf ./fakearmv7l ; \ - export LD_PRELOAD="/usr/local/lib/libfakearmv7l.so" ; \ fi # RVM From 3b294e20c34692d9613562a8371dbf2c9feb0bed Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Fri, 12 Jun 2020 17:42:32 +0200 Subject: [PATCH 08/22] Replace TARGET_ARCH -> DD_TARGET_ARCH because it can conflict with some software Add missing dependency for RPM --- .gitlab-ci.yml | 20 ++++++++++---------- deb-arm/Dockerfile | 10 +++++----- entrypoint.sh | 4 ++-- rpm-arm/Dockerfile | 18 +++++++++--------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3078ea29..653c73bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,13 +37,13 @@ build_rpm_x64: image: arm64v8/docker:18.04 variables: BASE_IMAGE: arm64v8/ubuntu:16.04 - TARGET_ARCH: arm64v8 + DD_TARGET_ARCH: arm64v8 script: - apk add --update python py-pip - pip install awscli - aws --version - eval "$(aws ecr get-login --region us-east-1 --no-include-email --registry-ids 486234852809)" - - docker build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg TARGET_ARCH=$TARGET_ARCH --tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} --file $DOCKERFILE . + - docker build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} --file $DOCKERFILE . - docker push 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} # For testing purposes - docker tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID @@ -61,7 +61,7 @@ build_deb_armhf: DOCKERFILE: deb-arm/Dockerfile IMAGE: deb_armhf BASE_IMAGE: arm32v7/ubuntu:16.04 - TARGET_ARCH: arm32v7 + DD_TARGET_ARCH: arm32v7 build_rpm_arm64: extends: .armbuild @@ -76,7 +76,7 @@ build_rpm_armhf: DOCKERFILE: rpm-arm/Dockerfile IMAGE: rpm_armhf BASE_IMAGE: arm32v7/centos:7 - TARGET_ARCH: arm32v7 + DD_TARGET_ARCH: arm32v7 build_system-probe_x64: extends: .build @@ -168,8 +168,8 @@ test_system-probe_arm64: - $SHORT_CI_COMMIT_SHA = $($CI_COMMIT_SHA.Substring(0,7)) - $SRC_TAG = "v$CI_PIPELINE_ID-$SHORT_CI_COMMIT_SHA" - $SRC_IMAGE = "486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/${IMAGE}:${SRC_TAG}" - - echo "build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg TARGET_ARCH=$TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ." - - powershell -Command "docker build --no-cache --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg TARGET_ARCH=$TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ." + - echo "build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ." + - powershell -Command "docker build --no-cache --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --build-arg WINDOWS_VERSION=$WINDOWS_VERSION --tag $SRC_IMAGE --file $DOCKERFILE ." - docker push $SRC_IMAGE after_script: - docker image prune -f @@ -182,7 +182,7 @@ build_windows_1809_x64: DOCKERFILE: windows/Dockerfile IMAGE: windows_1809_x64 BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 - TARGET_ARCH: x64 + DD_TARGET_ARCH: x64 WINDOWS_VERSION: 1809 resource_group: windows_x64 @@ -193,7 +193,7 @@ build_windows_1809_x86: DOCKERFILE: windows/Dockerfile IMAGE: windows_1809_x86 BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200114-windowsservercore-ltsc2019 - TARGET_ARCH: x86 + DD_TARGET_ARCH: x86 WINDOWS_VERSION: 1809 resource_group: windows_x86 @@ -204,7 +204,7 @@ build_windows_1909_x64: DOCKERFILE: windows/Dockerfile IMAGE: windows_1909_x64 BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909 - TARGET_ARCH: x64 + DD_TARGET_ARCH: x64 WINDOWS_VERSION: 1909 resource_group: windows_x64 @@ -215,7 +215,7 @@ build_windows_1909_x86: DOCKERFILE: windows/Dockerfile IMAGE: windows_1909_x86 BASE_IMAGE: mcr.microsoft.com/dotnet/framework/runtime:4.8-20200310-windowsservercore-1909 - TARGET_ARCH: x86 + DD_TARGET_ARCH: x86 WINDOWS_VERSION: 1909 resource_group: windows_x86 diff --git a/deb-arm/Dockerfile b/deb-arm/Dockerfile index e3876b42..ef5cde4d 100644 --- a/deb-arm/Dockerfile +++ b/deb-arm/Dockerfile @@ -9,7 +9,7 @@ ARG DD_PIP_VERSION=19.1 ARG DD_SETUPTOOLS_VERSION=41.0.1 ARG CMAKE_VERSION=3.14.4 ARG CLANG_VERSION=8.0.0 -ARG TARGET_ARCH=arm64v8 +ARG DD_TARGET_ARCH=arm64v8 # Environment ENV GOPATH /go @@ -18,7 +18,7 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION -ENV TARGET_ARCH $TARGET_ARCH +ENV DD_TARGET_ARCH $DD_TARGET_ARCH RUN apt-get update && apt-get install -y fakeroot curl git procps bzip2 \ python2.7-dev build-essential pkg-config tar libsystemd-dev libkrb5-dev \ @@ -47,7 +47,7 @@ RUN chmod +x /bin/gimme # GIMME_ARCH = GOARCH, so must be a valid entry from `goarchlist` here: # https://github.com/golang/go/blob/master/src/go/build/syslist.go # Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 -RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then \ GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION ; \ else \ GIMME_ARCH=arm gimme $GIMME_GO_VERSION ; \ @@ -56,14 +56,14 @@ RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ COPY ./gobin.sh /etc/profile.d/ # CMake. Pre-built using the build-cmake.sh script, to speed-up docker build. -RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then curl -sL -O https://dd-agent-omnibus.s3.amazonaws.com/cmake-${CMAKE_VERSION}-ubuntu-aarch64.tar.xz && \ +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then curl -sL -O https://dd-agent-omnibus.s3.amazonaws.com/cmake-${CMAKE_VERSION}-ubuntu-aarch64.tar.xz && \ tar xf cmake-${CMAKE_VERSION}-ubuntu-aarch64.tar.xz --no-same-owner -kC / && \ rm cmake-${CMAKE_VERSION}-ubuntu-aarch64.tar.xz ; fi ENV PATH="/opt/cmake/bin:$PATH" # Install clang and llvm version 8. Pre-built because building takes ~4 hours. # This was built from sources on centos 7, using the build-clang.sh script -RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then curl -sL -o clang_llvm.tar.xz https://dd-agent-omnibus.s3.amazonaws.com/clang%2Bllvm-${CLANG_VERSION}-aarch64-linux.tar.xz && \ +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then curl -sL -o clang_llvm.tar.xz https://dd-agent-omnibus.s3.amazonaws.com/clang%2Bllvm-${CLANG_VERSION}-aarch64-linux.tar.xz && \ tar xf clang_llvm.tar.xz --no-same-owner -kC / && \ rm clang_llvm.tar.xz ; fi ENV PATH="/opt/clang/bin:$PATH" diff --git a/entrypoint.sh b/entrypoint.sh index fbda1de0..af6900b5 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,9 +5,9 @@ source /root/.bashrc conda activate ddpy3 || true # We use system python3 on some images, allow this to fail -if [ "$TARGET_ARCH" = "arm64v8" ] ; then +if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then export GIMME_ARCH=arm64 -elif [ "$TARGET_ARCH" = "arm32v7" ] ; then +elif [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then export GIMME_ARCH=arm fi eval "$(gimme)" diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 7a4ee8ea..678e2c13 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -9,7 +9,7 @@ ARG DD_PIP_VERSION=19.1 ARG DD_SETUPTOOLS_VERSION=41.0.1 ARG CMAKE_VERSION=3.14.4 ARG CLANG_VERSION=8.0.0 -ARG TARGET_ARCH=arm64v8 +ARG DD_TARGET_ARCH=arm64v8 # Environment ENV GOPATH /go @@ -18,24 +18,24 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION -ENV TARGET_ARCH $TARGET_ARCH +ENV DD_TARGET_ARCH $DD_TARGET_ARCH # configure yum and rpm for running on non-armv7l architectures -RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then echo "armhfp" > /etc/yum/vars/basearch && \ +RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then echo "armhfp" > /etc/yum/vars/basearch && \ echo "armv7hl" > /etc/yum/vars/arch && \ echo "armv7hl-redhat-linux-gnu" > /etc/rpm/platform ; fi # Resolves a nasty NOKEY warning that appears when using yum -RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ +RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-AltArch-Arm32 ; fi RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ - libtool autoconf policycoreutils-python + libtool autoconf policycoreutils-python libstdc++6 # This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" -RUN if [ "$TARGET_ARCH" = "arm32v7" ] ; then \ +RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then \ git clone https://github.com/DataDog/fakearmv7l ; \ cd fakearmv7l ; \ make ; \ @@ -64,7 +64,7 @@ RUN chmod +x /bin/gimme # GIMME_ARCH = GOARCH, so must be a valid entry from `goarchlist` here: # https://github.com/golang/go/blob/master/src/go/build/syslist.go # Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 -RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then \ GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION ; \ else \ GIMME_ARCH=arm gimme $GIMME_GO_VERSION ; \ @@ -73,14 +73,14 @@ RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then \ COPY ./gobin.sh /etc/profile.d/ # CMake. Pre-built using the build-cmake.sh script, to speed-up docker build. -RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then curl -sL -O https://dd-agent-omnibus.s3.amazonaws.com/cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz && \ +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then curl -sL -O https://dd-agent-omnibus.s3.amazonaws.com/cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz && \ tar xf cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz --no-same-owner -kC / && \ rm cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz ; fi ENV PATH="/opt/cmake/bin:$PATH" # Install clang and llvm version 8. Pre-built because building takes ~4 hours. # This was built from sources on centos 7, using the build-clang.sh script -RUN if [ "$TARGET_ARCH" = "arm64v8" ] ; then curl -sL -o clang_llvm.tar.xz https://dd-agent-omnibus.s3.amazonaws.com/clang%2Bllvm-${CLANG_VERSION}-aarch64-linux.tar.xz && \ +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then curl -sL -o clang_llvm.tar.xz https://dd-agent-omnibus.s3.amazonaws.com/clang%2Bllvm-${CLANG_VERSION}-aarch64-linux.tar.xz && \ tar xf clang_llvm.tar.xz --no-same-owner -kC / && \ rm clang_llvm.tar.xz ; fi ENV PATH="/opt/clang/bin:$PATH" From f0c61c4362dce1a5e1106e4d34409a441edd8719 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Fri, 12 Jun 2020 18:37:52 +0200 Subject: [PATCH 09/22] Test fix RPM ARM builds --- rpm-arm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 678e2c13..33309b3f 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -31,7 +31,7 @@ RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then rpm --import /etc/pki/rpm-gpg/RP RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ - libtool autoconf policycoreutils-python libstdc++6 + libtool autoconf policycoreutils-python gcc-4.9 g++-4.9 # This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" From a85d6bfcbf03f6eff3f24632a96af0332bc72499 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Sat, 13 Jun 2020 01:13:43 +0200 Subject: [PATCH 10/22] Add newer gcc --- rpm-arm/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 33309b3f..2a78f4df 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -31,7 +31,11 @@ RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then rpm --import /etc/pki/rpm-gpg/RP RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ - libtool autoconf policycoreutils-python gcc-4.9 g++-4.9 + libtool autoconf policycoreutils-python + +RUN yum -y install centos-release-scl-rh && \ + yum -y install devtoolset-9 && \ + echo source scl_source enable devtoolset-9 >> /root/.bashrc # This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" From 3cc8dd5bc4e1a52f5dac9ca55846dd654e3f980e Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Sat, 13 Jun 2020 01:25:02 +0200 Subject: [PATCH 11/22] Add conditional for ARM 64 --- rpm-arm/Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 2a78f4df..8e2a123a 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -33,9 +33,12 @@ RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMa curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ libtool autoconf policycoreutils-python -RUN yum -y install centos-release-scl-rh && \ - yum -y install devtoolset-9 && \ - echo source scl_source enable devtoolset-9 >> /root/.bashrc +# For ARM64, also install modern GCC +RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then \ + yum -y install centos-release-scl-rh && \ + yum -y install devtoolset-9 && \ + echo source scl_source enable devtoolset-9 >> /root/.bashrc ; \ + fi # This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" From 11c88357d3251d49439be16dc073d44df961c5d1 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Mon, 15 Jun 2020 16:40:20 +0200 Subject: [PATCH 12/22] Separate RPM ARM64 / ARM32 --- .gitlab-ci.yml | 4 +-- rpm-arm/Dockerfile | 46 ++++++------------------------ rpm-arm64/Dockerfile | 67 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 40 deletions(-) create mode 100644 rpm-arm64/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 653c73bd..92727780 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,9 +66,9 @@ build_deb_armhf: build_rpm_arm64: extends: .armbuild variables: - DOCKERFILE: rpm-arm/Dockerfile + DOCKERFILE: rpm-arm64/Dockerfile IMAGE: rpm_arm64 - BASE_IMAGE: arm64v8/centos:7 + BASE_IMAGE: amazonlinux:2.0.20181114 build_rpm_armhf: extends: .armbuild diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 8e2a123a..89aea3fb 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -1,6 +1,4 @@ -## Valid archs are -# amd64, arm32v7, arm64v8, i386, ppc64le -ARG BASE_IMAGE=arm64v8/centos:7 +ARG BASE_IMAGE=arm32v7/centos:7 FROM ${BASE_IMAGE} # Build Args @@ -9,7 +7,6 @@ ARG DD_PIP_VERSION=19.1 ARG DD_SETUPTOOLS_VERSION=41.0.1 ARG CMAKE_VERSION=3.14.4 ARG CLANG_VERSION=8.0.0 -ARG DD_TARGET_ARCH=arm64v8 # Environment ENV GOPATH /go @@ -18,38 +15,28 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION -ENV DD_TARGET_ARCH $DD_TARGET_ARCH # configure yum and rpm for running on non-armv7l architectures -RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then echo "armhfp" > /etc/yum/vars/basearch && \ +RUN echo "armhfp" > /etc/yum/vars/basearch && \ echo "armv7hl" > /etc/yum/vars/arch && \ - echo "armv7hl-redhat-linux-gnu" > /etc/rpm/platform ; fi + echo "armv7hl-redhat-linux-gnu" > /etc/rpm/platform # Resolves a nasty NOKEY warning that appears when using yum -RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ - rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-AltArch-Arm32 ; fi +RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \ + rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-AltArch-Arm32 RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ libtool autoconf policycoreutils-python -# For ARM64, also install modern GCC -RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then \ - yum -y install centos-release-scl-rh && \ - yum -y install devtoolset-9 && \ - echo source scl_source enable devtoolset-9 >> /root/.bashrc ; \ - fi - # This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" -RUN if [ "$DD_TARGET_ARCH" = "arm32v7" ] ; then \ - git clone https://github.com/DataDog/fakearmv7l ; \ +RUN git clone https://github.com/DataDog/fakearmv7l ; \ cd fakearmv7l ; \ make ; \ make install ; \ cd .. ; \ - rm -rf ./fakearmv7l ; \ - fi + rm -rf ./fakearmv7l ; # RVM COPY ./rvm/gpg-keys /gpg-keys @@ -71,27 +58,10 @@ RUN chmod +x /bin/gimme # GIMME_ARCH = GOARCH, so must be a valid entry from `goarchlist` here: # https://github.com/golang/go/blob/master/src/go/build/syslist.go # Also see https://github.com/travis-ci/gimme/blob/master/gimme#L880 -RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then \ - GIMME_ARCH=arm64 gimme $GIMME_GO_VERSION ; \ - else \ - GIMME_ARCH=arm gimme $GIMME_GO_VERSION ; \ - fi +RUN GIMME_ARCH=arm gimme $GIMME_GO_VERSION COPY ./gobin.sh /etc/profile.d/ -# CMake. Pre-built using the build-cmake.sh script, to speed-up docker build. -RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then curl -sL -O https://dd-agent-omnibus.s3.amazonaws.com/cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz && \ - tar xf cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz --no-same-owner -kC / && \ - rm cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz ; fi -ENV PATH="/opt/cmake/bin:$PATH" - -# Install clang and llvm version 8. Pre-built because building takes ~4 hours. -# This was built from sources on centos 7, using the build-clang.sh script -RUN if [ "$DD_TARGET_ARCH" = "arm64v8" ] ; then curl -sL -o clang_llvm.tar.xz https://dd-agent-omnibus.s3.amazonaws.com/clang%2Bllvm-${CLANG_VERSION}-aarch64-linux.tar.xz && \ - tar xf clang_llvm.tar.xz --no-same-owner -kC / && \ - rm clang_llvm.tar.xz ; fi -ENV PATH="/opt/clang/bin:$PATH" - # Entrypoint COPY ./entrypoint.sh / RUN chmod +x /entrypoint.sh diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile new file mode 100644 index 00000000..d145966c --- /dev/null +++ b/rpm-arm64/Dockerfile @@ -0,0 +1,67 @@ +ARG BASE_IMAGE=amazonlinux:2.0.20181114 +FROM ${BASE_IMAGE} + +# Build Args +ARG GIMME_GO_VERSION=1.12.9 +ARG DD_PIP_VERSION=19.1 +ARG DD_SETUPTOOLS_VERSION=41.0.1 +ARG CMAKE_VERSION=3.14.4 +ARG CLANG_VERSION=8.0.0 + +# Environment +ENV GOPATH /go +ENV GIMME_GO_VERSION $GIMME_GO_VERSION +ENV DD_PIP_VERSION $DD_PIP_VERSION +ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION +ENV CMAKE_VERSION $CMAKE_VERSION +ENV CLANG_VERSION $CLANG_VERSION + + +RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ + curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig + +# RVM +COPY ./rvm/gpg-keys /gpg-keys +RUN gpg --import /gpg-keys/* +RUN rm -rf /gpg-keys +RUN curl -sSL https://get.rvm.io | bash -s stable +RUN /bin/bash -l -c "rvm requirements" +RUN /bin/bash -l -c "rvm install 2.3 && rvm cleanup all" +RUN /bin/bash -l -c "gem install bundler --no-document" + +# Pip & Invoke +RUN curl "https://bootstrap.pypa.io/get-pip.py" | python2.7 - pip==${DD_PIP_VERSION} setuptools==${DD_SETUPTOOLS_VERSION} +RUN pip install invoke awscli==1.16.240 + + + +# Gimme +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme +RUN gimme $GIMME_GO_VERSION +COPY ./gobin.sh /etc/profile.d/ + +# CMake. Pre-built using the build-cmake.sh script, to speed-up docker build. +RUN curl -sL -O https://dd-agent-omnibus.s3.amazonaws.com/cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz && \ + tar xf cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz --no-same-owner -kC / && \ + rm cmake-${CMAKE_VERSION}-amzn-aarch64.tar.xz +ENV PATH="/opt/cmake/bin:$PATH" + +# Install clang and llvm version 8. Pre-built because building takes ~4 hours. +# This was built from sources on centos 7, using the build-clang.sh script +RUN curl -sL -o clang_llvm.tar.xz https://dd-agent-omnibus.s3.amazonaws.com/clang%2Bllvm-${CLANG_VERSION}-aarch64-linux.tar.xz && \ + tar xf clang_llvm.tar.xz --no-same-owner -kC / && \ + rm clang_llvm.tar.xz +ENV PATH="/opt/clang/bin:$PATH" + +# Entrypoint +COPY ./entrypoint.sh / +RUN chmod +x /entrypoint.sh + +# create the agent build folder within $GOPATH +RUN mkdir -p /go/src/github.com/DataDog/datadog-agent + +# Force umask to 0022 +RUN echo "umask 0022" >> /root/.bashrc + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file From ec5097b4eab0f9286d825408b9183453012225b7 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Mon, 15 Jun 2020 16:53:12 +0200 Subject: [PATCH 13/22] Ensures processes in build image use fakearmv7l --- rpm-arm/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 89aea3fb..1a63beaa 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -36,7 +36,8 @@ RUN git clone https://github.com/DataDog/fakearmv7l ; \ make ; \ make install ; \ cd .. ; \ - rm -rf ./fakearmv7l ; + rm -rf ./fakearmv7l ; \ + export LD_PRELOAD="/usr/local/lib/libfakearmv7l.so" # RVM COPY ./rvm/gpg-keys /gpg-keys From 794aa048fccc3cbcdffdf6aea8038bb0795c3987 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 11:30:54 +0200 Subject: [PATCH 14/22] Keep DD_TARGET_ARCH env variable --- rpm-arm/Dockerfile | 2 ++ rpm-arm64/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 1a63beaa..b0d3d455 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -7,6 +7,7 @@ ARG DD_PIP_VERSION=19.1 ARG DD_SETUPTOOLS_VERSION=41.0.1 ARG CMAKE_VERSION=3.14.4 ARG CLANG_VERSION=8.0.0 +ARG DD_TARGET_ARCH=arm32v7 # Environment ENV GOPATH /go @@ -15,6 +16,7 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION +ENV DD_TARGET_ARCH $DD_TARGET_ARCH # configure yum and rpm for running on non-armv7l architectures RUN echo "armhfp" > /etc/yum/vars/basearch && \ diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile index d145966c..beaea3f9 100644 --- a/rpm-arm64/Dockerfile +++ b/rpm-arm64/Dockerfile @@ -7,6 +7,7 @@ ARG DD_PIP_VERSION=19.1 ARG DD_SETUPTOOLS_VERSION=41.0.1 ARG CMAKE_VERSION=3.14.4 ARG CLANG_VERSION=8.0.0 +ARG DD_TARGET_ARCH=arm64v8 # Environment ENV GOPATH /go @@ -15,7 +16,7 @@ ENV DD_PIP_VERSION $DD_PIP_VERSION ENV DD_SETUPTOOLS_VERSION $DD_SETUPTOOLS_VERSION ENV CMAKE_VERSION $CMAKE_VERSION ENV CLANG_VERSION $CLANG_VERSION - +ENV DD_TARGET_ARCH $DD_TARGET_ARCH RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig From ff9a57570b4748f641618bf6121ef7a60b1f007f Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 11:07:15 +0200 Subject: [PATCH 15/22] Update rpm-arm/Dockerfile Co-authored-by: Pablo Baeyens --- rpm-arm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index b0d3d455..9e5ce3bc 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -31,7 +31,7 @@ RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMa curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ libtool autoconf policycoreutils-python -# This trick preloads a dynamic libary that will replace the uname syscall to pretend the machine is armv7l +# This trick preloads a dynamic library that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" RUN git clone https://github.com/DataDog/fakearmv7l ; \ cd fakearmv7l ; \ From 8fdd2096beb0d249dfcc248351948333fe24ebb8 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 12:14:53 +0200 Subject: [PATCH 16/22] Test passing variables from gitlab-ci directly --- .gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92727780..98a97510 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,11 @@ stages: variables: WINDOWS_RELEASE_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1909_x64:v2348149-ba6640d + GIMME_GO_VERSION: 1.13.8 + DD_PIP_VERSION: 19.1 + DD_SETUPTOOLS_VERSION: 41.0.1 + CMAKE_VERSION: 3.14.4 + CLANG_VERSION: 8.0.0 .build: stage: build @@ -43,7 +48,15 @@ build_rpm_x64: - pip install awscli - aws --version - eval "$(aws ecr get-login --region us-east-1 --no-include-email --registry-ids 486234852809)" - - docker build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} --file $DOCKERFILE . + - docker build --build-arg BASE_IMAGE=$BASE_IMAGE + --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH + --build-arg GIMME_GO_VERSION=$GIMME_GO_VERSION + --build-arg DD_PIP_VERSION=$DD_PIP_VERSION + --build-arg DD_SETUPTOOLS_VERSION=$DD_SETUPTOOLS_VERSION + --build-arg CMAKE_VERSION=$CMAKE_VERSION + --build-arg CLANG_VERSION=$CLANG_VERSION + --tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} + --file $DOCKERFILE . - docker push 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} # For testing purposes - docker tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID From c21e5b6ee63fa98aec4c8b150f3c3572091d25b5 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 12:23:54 +0200 Subject: [PATCH 17/22] Revert "Test passing variables from gitlab-ci directly" This reverts commit 8fdd2096beb0d249dfcc248351948333fe24ebb8. --- .gitlab-ci.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98a97510..92727780 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,11 +5,6 @@ stages: variables: WINDOWS_RELEASE_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-builders/windows_1909_x64:v2348149-ba6640d - GIMME_GO_VERSION: 1.13.8 - DD_PIP_VERSION: 19.1 - DD_SETUPTOOLS_VERSION: 41.0.1 - CMAKE_VERSION: 3.14.4 - CLANG_VERSION: 8.0.0 .build: stage: build @@ -48,15 +43,7 @@ build_rpm_x64: - pip install awscli - aws --version - eval "$(aws ecr get-login --region us-east-1 --no-include-email --registry-ids 486234852809)" - - docker build --build-arg BASE_IMAGE=$BASE_IMAGE - --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH - --build-arg GIMME_GO_VERSION=$GIMME_GO_VERSION - --build-arg DD_PIP_VERSION=$DD_PIP_VERSION - --build-arg DD_SETUPTOOLS_VERSION=$DD_SETUPTOOLS_VERSION - --build-arg CMAKE_VERSION=$CMAKE_VERSION - --build-arg CLANG_VERSION=$CLANG_VERSION - --tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} - --file $DOCKERFILE . + - docker build --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg DD_TARGET_ARCH=$DD_TARGET_ARCH --tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} --file $DOCKERFILE . - docker push 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} # For testing purposes - docker tag 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID-${CI_COMMIT_SHA:0:7} 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/$IMAGE:v$CI_PIPELINE_ID From fa0c26d93c66bb1eb92f82c94fb432b8ce695bc7 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 12:25:01 +0200 Subject: [PATCH 18/22] Update GO_VERSION --- rpm-arm64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile index beaea3f9..83a76388 100644 --- a/rpm-arm64/Dockerfile +++ b/rpm-arm64/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE=amazonlinux:2.0.20181114 FROM ${BASE_IMAGE} # Build Args -ARG GIMME_GO_VERSION=1.12.9 +ARG GIMME_GO_VERSION=1.13.8 ARG DD_PIP_VERSION=19.1 ARG DD_SETUPTOOLS_VERSION=41.0.1 ARG CMAKE_VERSION=3.14.4 From d6457889398c5aa0400b28eea2bf3f6ba0dae4bb Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 14:40:42 +0200 Subject: [PATCH 19/22] Add systemd dependency to RPM ARM64 --- rpm-arm64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile index 83a76388..b1ba314b 100644 --- a/rpm-arm64/Dockerfile +++ b/rpm-arm64/Dockerfile @@ -19,7 +19,7 @@ ENV CLANG_VERSION $CLANG_VERSION ENV DD_TARGET_ARCH $DD_TARGET_ARCH RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ - curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig + curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig libsystemd-dev # RVM COPY ./rvm/gpg-keys /gpg-keys From 93e3fed508bf0c6b391b672aa2c78d59edd8aef8 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 15:13:39 +0200 Subject: [PATCH 20/22] Use correct RPM package for systemd --- rpm-arm64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile index b1ba314b..08ad0348 100644 --- a/rpm-arm64/Dockerfile +++ b/rpm-arm64/Dockerfile @@ -19,7 +19,7 @@ ENV CLANG_VERSION $CLANG_VERSION ENV DD_TARGET_ARCH $DD_TARGET_ARCH RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ - curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig libsystemd-dev + curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig systemd-devel # RVM COPY ./rvm/gpg-keys /gpg-keys From 8f0a16a9b91dc542e32f13f9a3ab4dafa5bc9a36 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Tue, 16 Jun 2020 16:31:20 +0200 Subject: [PATCH 21/22] Revert RPM ARM64 to this version https://github.com/DataDog/datadog-agent-buildimages/blob/0439a409cfb0bf3dcd6ff301b591e05c47f60ac0/rpm-arm64/Dockerfile --- rpm-arm64/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rpm-arm64/Dockerfile b/rpm-arm64/Dockerfile index 08ad0348..616965f5 100644 --- a/rpm-arm64/Dockerfile +++ b/rpm-arm64/Dockerfile @@ -19,7 +19,8 @@ ENV CLANG_VERSION $CLANG_VERSION ENV DD_TARGET_ARCH $DD_TARGET_ARCH RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMaker ncurses-compat-libs git procps \ - curl-devel expat-devel gettext-devel openssl-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig systemd-devel + curl-devel expat-devel gettext-devel openssl-devel systemd-devel zlib-devel bzip2 glibc-static python-devel tar pkgconfig \ + libtool autoconf policycoreutils-python # RVM COPY ./rvm/gpg-keys /gpg-keys @@ -32,9 +33,7 @@ RUN /bin/bash -l -c "gem install bundler --no-document" # Pip & Invoke RUN curl "https://bootstrap.pypa.io/get-pip.py" | python2.7 - pip==${DD_PIP_VERSION} setuptools==${DD_SETUPTOOLS_VERSION} -RUN pip install invoke awscli==1.16.240 - - +RUN pip install invoke distro==1.4.0 awscli==1.16.240 # Gimme RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme From 46e965a72c59dbaefe9f233bb7c2db54214e2f88 Mon Sep 17 00:00:00 2001 From: Julien Lebot Date: Wed, 17 Jun 2020 11:43:12 +0200 Subject: [PATCH 22/22] Update rpm-arm/Dockerfile Co-authored-by: Pablo Baeyens --- rpm-arm/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpm-arm/Dockerfile b/rpm-arm/Dockerfile index 9e5ce3bc..90a636c7 100644 --- a/rpm-arm/Dockerfile +++ b/rpm-arm/Dockerfile @@ -33,7 +33,7 @@ RUN yum groupinstall -y development && yum -y install which perl-ExtUtils-MakeMa # This trick preloads a dynamic library that will replace the uname syscall to pretend the machine is armv7l # The fakearmv7l library gets installed in "/usr/local/lib/libfakearmv7l.so" -RUN git clone https://github.com/DataDog/fakearmv7l ; \ +RUN git clone --depth 1 https://github.com/DataDog/fakearmv7l ; \ cd fakearmv7l ; \ make ; \ make install ; \