Merge pull request #1915 from DataDog/ivoanjo/arm64-ci

Bump Ruby 3.0 image to 3.0.3, Ruby 3.1 image to 3.1.1, and add support for local testing/development on arm64

.circleci/config.yml

@@ -487,7 +487,7 @@ job_configuration:
   - &config-3_1
     <<: *filters_all_branches_and_tags
     ruby_version: '3.1'
-    image: ivoanjo/docker-library:ddtrace_rb_3_1_0_preview1
+    image: ivoanjo/docker-library:ddtrace_rb_3_1_1
     resource_class_to_use: xlarge # Temporary, see https://github.com/DataDog/dd-trace-rb/issues/1791
     # ADD NEW RUBIES HERE
   # JRuby

.circleci/images/primary/Dockerfile-3.0.1 → .circleci/images/primary/Dockerfile-3.0.3

@@ -1,5 +1,6 @@
-# https://hub.docker.com/layers/ruby/library/ruby/3.0.1-buster/images/sha256-d5c828809239010c8549eeaf1f6da84f67bee7f61353e6a4a52159bf3f397aa6
-FROM ruby@sha256:d5c828809239010c8549eeaf1f6da84f67bee7f61353e6a4a52159bf3f397aa6
+# This image supports multiple platforms, see README for build instructions
+
+FROM ruby:3.0.3-bullseye
 
 # Make apt non-interactive
 RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \
@@ -15,7 +16,7 @@ RUN set -ex; \
             git mercurial xvfb \
             locales sudo openssh-client ca-certificates tar gzip parallel \
             net-tools netcat unzip zip bzip2 gnupg curl wget \
-            tzdata rsync vim less; \
+            tzdata rsync vim less jq; \
         rm -rf /var/lib/apt/lists/*;
 
 # Set timezone to UTC by default
@@ -26,16 +27,10 @@ RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 
-# Install jq
-RUN JQ_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/jq-latest" \
-  && curl --silent --show-error --location --fail --retry 3 --output /usr/bin/jq $JQ_URL \
-  && chmod +x /usr/bin/jq \
-  && jq --version
-
 # Install Docker
 RUN set -ex \
-  && export DOCKER_VERSION=$(curl --silent --fail --retry 3 https://download.docker.com/linux/static/stable/x86_64/ | grep -o -e 'docker-[.0-9]*-ce\.tgz' | sort -r | head -n 1) \
-  && DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/${DOCKER_VERSION}" \
+  && export DOCKER_VERSION=$(curl --silent --fail --retry 3 https://download.docker.com/linux/static/stable/$(arch)/ | grep -o -e 'docker-[.0-9]*-ce\.tgz' | sort -r | head -n 1) \
+  && DOCKER_URL="https://download.docker.com/linux/static/stable/$(arch)/${DOCKER_VERSION}" \
   && echo Docker URL: $DOCKER_URL \
   && curl --silent --show-error --location --fail --retry 3 --output /tmp/docker.tgz "${DOCKER_URL}" \
   && ls -lha /tmp/docker.tgz \
@@ -46,17 +41,13 @@ RUN set -ex \
   && (docker version || true)
 
 # Install Docker Compose
-RUN COMPOSE_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/docker-compose-latest" \
+RUN COMPOSE_URL="https://github.com/linuxserver/docker-docker-compose/releases/download/1.29.2-ls51/docker-compose-$(dpkg --print-architecture)" \
   && curl --silent --show-error --location --fail --retry 3 --output /usr/bin/docker-compose $COMPOSE_URL \
   && chmod +x /usr/bin/docker-compose \
   && docker-compose version
 
 # Install Dockerize
-RUN DOCKERIZE_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/dockerize-latest.tar.gz" \
-  && curl --silent --show-error --location --fail --retry 3 --output /tmp/dockerize-linux-amd64.tar.gz $DOCKERIZE_URL \
-  && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64.tar.gz \
-  && rm -rf /tmp/dockerize-linux-amd64.tar.gz \
-  && dockerize --version
+RUN curl -sfL $(curl -s https://api.github.com/repos/powerman/dockerize/releases/latest | grep -i /dockerize-$(uname -s)-$(dpkg --print-architecture)\" | cut -d\" -f4) | install /dev/stdin /usr/local/bin/dockerize
 
 # Install RubyGems
 RUN gem update --system

.circleci/images/primary/Dockerfile-3.1.0-preview1 → .circleci/images/primary/Dockerfile-3.1.1

@@ -1,5 +1,6 @@
-# https://hub.docker.com/layers/ruby/library/ruby/3.1.0-preview1-bullseye/images/sha256-19f9abe757cd2a99262aded1177cedba0613cefc3c6181ce670b2259270a59d7
-FROM ruby@sha256:19f9abe757cd2a99262aded1177cedba0613cefc3c6181ce670b2259270a59d7
+# This image supports multiple platforms, see README for build instructions
+
+FROM ruby:3.1.1-bullseye
 
 # Make apt non-interactive
 RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \
@@ -15,7 +16,7 @@ RUN set -ex; \
             git mercurial xvfb \
             locales sudo openssh-client ca-certificates tar gzip parallel \
             net-tools netcat unzip zip bzip2 gnupg curl wget \
-            tzdata rsync vim less; \
+            tzdata rsync vim less jq; \
         rm -rf /var/lib/apt/lists/*;
 
 # Set timezone to UTC by default
@@ -26,16 +27,10 @@ RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 
-# Install jq
-RUN JQ_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/jq-latest" \
-  && curl --silent --show-error --location --fail --retry 3 --output /usr/bin/jq $JQ_URL \
-  && chmod +x /usr/bin/jq \
-  && jq --version
-
 # Install Docker
 RUN set -ex \
-  && export DOCKER_VERSION=$(curl --silent --fail --retry 3 https://download.docker.com/linux/static/stable/x86_64/ | grep -o -e 'docker-[.0-9]*-ce\.tgz' | sort -r | head -n 1) \
-  && DOCKER_URL="https://download.docker.com/linux/static/stable/x86_64/${DOCKER_VERSION}" \
+  && export DOCKER_VERSION=$(curl --silent --fail --retry 3 https://download.docker.com/linux/static/stable/$(arch)/ | grep -o -e 'docker-[.0-9]*-ce\.tgz' | sort -r | head -n 1) \
+  && DOCKER_URL="https://download.docker.com/linux/static/stable/$(arch)/${DOCKER_VERSION}" \
   && echo Docker URL: $DOCKER_URL \
   && curl --silent --show-error --location --fail --retry 3 --output /tmp/docker.tgz "${DOCKER_URL}" \
   && ls -lha /tmp/docker.tgz \
@@ -46,17 +41,13 @@ RUN set -ex \
   && (docker version || true)
 
 # Install Docker Compose
-RUN COMPOSE_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/docker-compose-latest" \
+RUN COMPOSE_URL="https://github.com/linuxserver/docker-docker-compose/releases/download/1.29.2-ls51/docker-compose-$(dpkg --print-architecture)" \
   && curl --silent --show-error --location --fail --retry 3 --output /usr/bin/docker-compose $COMPOSE_URL \
   && chmod +x /usr/bin/docker-compose \
   && docker-compose version
 
 # Install Dockerize
-RUN DOCKERIZE_URL="https://circle-downloads.s3.amazonaws.com/circleci-images/cache/linux-amd64/dockerize-latest.tar.gz" \
-  && curl --silent --show-error --location --fail --retry 3 --output /tmp/dockerize-linux-amd64.tar.gz $DOCKERIZE_URL \
-  && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64.tar.gz \
-  && rm -rf /tmp/dockerize-linux-amd64.tar.gz \
-  && dockerize --version
+RUN curl -sfL $(curl -s https://api.github.com/repos/powerman/dockerize/releases/latest | grep -i /dockerize-$(uname -s)-$(dpkg --print-architecture)\" | cut -d\" -f4) | install /dev/stdin /usr/local/bin/dockerize
 
 # Install RubyGems
 RUN gem update --system

.circleci/images/primary/README.md

@@ -0,0 +1,21 @@
+# Dockerfiles used for testing and development
+
+This folder contains Dockerfiles for all versions of Ruby used in the testing and development of dd-trace-rb.
+
+## Multiplatform builds
+
+Images marked with a "# This image supports multiple platforms" are able to be built for both amd64 (x86_64) and
+arm64 (aarch64) Linux.
+
+Here's an example of building the Ruby 3.1 image:
+
+```bash
+# To build and push the image (update tag as needed):
+$ docker buildx build . --platform linux/amd64,linux/arm64/v8 -f Dockerfile-3.1.1 -t ivoanjo/docker-library:ddtrace_rb_3_1_1 --push
+
+# The tag created will automatically contain both architectures. To additionally create an
+# architecture-specific tag, I needed to do the following (sha is the specific image)
+$ docker pull ivoanjo/docker-library:ddtrace_rb_3_1_1@sha256:56402a1c5e522b669965db4600f1a4fa035f6e3597d098ec808e77192c4238fd
+$ docker tag ivoanjo/docker-library@sha256:56402a1c5e522b669965db4600f1a4fa035f6e3597d098ec808e77192c4238fd ivoanjo/docker-library:ddtrace_rb_3_1_1_amd64
+$ docker push ivoanjo/docker-library:ddtrace_rb_3_1_1_amd64
+```

Gemfile

@@ -87,7 +87,7 @@ end
 # previously-passing codebase start failing. Thus, we need to lock to a specific
 # version and bump it from time to time.
 # Also, there's no support for windows
-if RUBY_VERSION >= '2.4.0' && !Gem.win_platform?
+if RUBY_VERSION >= '2.4.0' && (RUBY_PLATFORM =~ /^x86_64-(darwin|linux)/)
   gem 'sorbet', '= 0.5.9672'
   gem 'spoom', '~> 1.1'
 end

docker-compose.yml

@@ -226,7 +226,7 @@ services:
       - bundle-2.7:/usr/local/bundle
       - "ddagent_var_run:${TEST_DDAGENT_VAR_RUN}"
   tracer-3.0:
-    image: ivoanjo/docker-library:ddtrace_rb_3_0_1
+    image: ivoanjo/docker-library:ddtrace_rb_3_0_3
     platform: linux/x86_64
     command: /bin/bash
     depends_on:
@@ -254,8 +254,36 @@ services:
       - .:/app
       - bundle-3.0:/usr/local/bundle
       - "ddagent_var_run:${TEST_DDAGENT_VAR_RUN}"
+  tracer-3.0-arm64:
+    image: ivoanjo/docker-library:ddtrace_rb_3_0_3_arm64
+    command: /bin/bash
+    depends_on:
+      - ddagent
+      - elasticsearch
+      - memcached
+      - mongodb
+      - mysql
+      - postgres
+      - redis
+    env_file: ./.env
+    environment:
+      - BUNDLE_GEMFILE=/app/Gemfile
+      - DD_AGENT_HOST=ddagent
+      - TEST_DATADOG_INTEGRATION=1
+      - TEST_ELASTICSEARCH_HOST=elasticsearch
+      - TEST_MEMCACHED_HOST=memcached
+      - TEST_MONGODB_HOST=mongodb
+      - TEST_MYSQL_HOST=mysql
+      - TEST_POSTGRES_HOST=postgres
+      - TEST_REDIS_HOST=redis
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+      - bundle-3.0-arm64:/usr/local/bundle
+      - "ddagent_var_run:${TEST_DDAGENT_VAR_RUN}"
   tracer-3.1:
-    image: ivoanjo/docker-library:ddtrace_rb_3_1_0_preview1
+    image: ivoanjo/docker-library:ddtrace_rb_3_1_1
     platform: linux/x86_64
     command: /bin/bash
     depends_on:
@@ -283,6 +311,34 @@ services:
       - .:/app
       - bundle-3.1:/usr/local/bundle
       - "ddagent_var_run:${TEST_DDAGENT_VAR_RUN}"
+  tracer-3.1-arm64:
+    image: ivoanjo/docker-library:ddtrace_rb_3_1_1_arm64
+    command: /bin/bash
+    depends_on:
+      - ddagent
+      - elasticsearch
+      - memcached
+      - mongodb
+      - mysql
+      - postgres
+      - redis
+    env_file: ./.env
+    environment:
+      - BUNDLE_GEMFILE=/app/Gemfile
+      - DD_AGENT_HOST=ddagent
+      - TEST_DATADOG_INTEGRATION=1
+      - TEST_ELASTICSEARCH_HOST=elasticsearch
+      - TEST_MEMCACHED_HOST=memcached
+      - TEST_MONGODB_HOST=mongodb
+      - TEST_MYSQL_HOST=mysql
+      - TEST_POSTGRES_HOST=postgres
+      - TEST_REDIS_HOST=redis
+    stdin_open: true
+    tty: true
+    volumes:
+      - .:/app
+      - bundle-3.1-arm64:/usr/local/bundle
+      - "ddagent_var_run:${TEST_DDAGENT_VAR_RUN}"
   # ADD NEW RUBIES HERE
   # JRuby
   tracer-jruby-9.2.8.0:
@@ -472,7 +528,9 @@ volumes:
   bundle-2.6:
   bundle-2.7:
   bundle-3.0:
+  bundle-3.0-arm64:
   bundle-3.1:
+  bundle-3.1-arm64:
   # ADD NEW RUBIES HERE
   bundle-jruby-9.2.8.0:
   bundle-jruby-9.2-latest:

spec/datadog/ci/test_spec.rb

@@ -185,7 +185,7 @@
           span_op.get_tag(Datadog::CI::Ext::Test::TAG_OS_ARCHITECTURE)
         end
 
-        it { is_expected.to eq('x86_64').or eq('i686').or start_with('arm') }
+        it { is_expected.to eq('x86_64').or eq('i686').or eq('aarch64').or start_with('arm') }
       end
 
       context 'for the OS platform' do

spec/datadog/core/diagnostics/environment_logger_spec.rb

@@ -46,7 +46,7 @@
           'health_metrics_enabled' => false,
           'lang' => 'ruby',
           'lang_version' => match(/[23]\./),
-          'os_name' => include('x86_64'),
+          'os_name' => (include('x86_64').or include('i686').or include('aarch64').or include('arm')),
           'partial_flushing_enabled' => false,
           'priority_sampling_enabled' => false,
           'runtime_metrics_enabled' => false,
@@ -140,7 +140,7 @@
           integrations_loaded: nil,
           lang: 'ruby',
           lang_version: match(/[23]\./),
-          os_name: include('x86_64'),
+          os_name: (include('x86_64').or include('i686').or include('aarch64').or include('arm')),
           partial_flushing_enabled: false,
           priority_sampling_enabled: false,
           runtime_metrics_enabled: false,