Improve update gemfiles

.github/workflows/update-gemfiles.yml

@@ -1,83 +1,99 @@
 name: Update Gemfiles
 
-# This action cannot be skipped altogether because it is a mandatory status check.
-# Instead we conditionally skip it at job level, instead of workflow level.
 on:
-  # Execute on `push` and not `pull_request` because `pull_request`
-  # always compares if the `paths` have changed compared to the PR base.
-  # This is an issue because it means that all commits to the branch
-  # will trigger the gemfile update process, which is unnecessary and expensive.
-  #
-  # By executing on `push`, GitHub compares `paths` with the parent commit,
-  # meaning the gemfile update process will only execute on the exact commit
-  # that changes any of the `paths`.
-  #
-  # Because this process is slow and expensive, and we commit the gemfile changes back
-  # to the branch, we have an additional filter to only execute this action on branches
-  # attached to a PR.
-  #
-  # We could do the inverse: execute this action on `pull_request`, and additionally check
-  # if `paths` was changed compared to the parent commit, but this proved more complicated.
-  push
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to be updated'
+        required: true
+  # To be removed once the workflow is fully automated
+  push:
+    branches:
+      - tonycthsu/automate-update-gemfiles
 
 # Ensure obsolete job is cancelled if another commit is pushed to the same branch.
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  check:
-    name: Update Gemfiles
-    runs-on: ubuntu-22.04
+  # TODO: In order to fully automate this workflow for each PR,
+  # have a reliable way to precheck job to understand whether it need to be updated
+  lock:
+    runs-on: ubuntu-latest
     permissions:
-      contents: write
       pull-requests: write
+      contents: write
+    strategy:
+      fail-fast: false
+      matrix:
+        engine:
+          # ADD NEW RUBIES HERE
+          - name: ruby
+            version: '3.4'
+          - name: ruby
+            version: '3.3'
+          - name: ruby
+            version: '3.2'
+          - name: ruby
+            version: '3.1'
+          - name: ruby
+            version: '3.0'
+          - name: ruby
+            version: '2.7'
+          - name: ruby
+            version: '2.6'
+          - name: ruby
+            version: '2.5'
+          - name: jruby
+            version: '9.4'
+          - name: jruby
+            version: '9.3'
+          - name: jruby
+            version: '9.2'
+    container:
+      image: ghcr.io/datadog/images-rb/engines/${{ matrix.engine.name }}:${{ matrix.engine.version }}
+      env:
+        BUNDLE_WITHOUT: check
     steps:
-      # Only execute if there's a PR attached to this branch.
-      # Because we execute on `push`, we have to double check here if this is part of a PR.
-      - name: Check if this branch is attached to a Pull Request
-        uses: 8bitjonny/gh-get-current-pr@2215326c76d51bfa3f2af0a470f32677f6c0cae9 # v2.2.0
-        id: pr
-        with:
-          filterOutClosed: true # Don't trigger on commits with closed PRs, including merges into `master`.
-      - if: steps.pr.outputs.pr_found == 'true'
+      - name: Checkout code
         uses: actions/checkout@v4
-      # And also, only execute if files that can affect gemfiles are modified.
-      - if: steps.pr.outputs.pr_found == 'true'
-        uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
-        id: filter
+      - run: |
+          ruby -v
+          gem -v
+          bundler -v
+
+      - run: bundle install
+
+      # TODO: Migrate away from `appraisal`
+      - run: bundle exec appraisal generate
+
+      - run: rake dependency
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
         with:
-          base: ${{ github.ref_name }}
-          filters: |
-            gemfile:
-              # Files that declare the dependency tree
-              - Gemfile
-              - Appraisals
-              - datadog.gemspec
-              # Files that control gemfile generation
-              - tasks/appraisal.rake
-              - .github/workflows/update-gemfiles.yml
-              # The gem version is present in all lock files
-              - lib/datadog/version.rb
-              # In case the generated files were updated manually or in a merge commit
-              - appraisal/**
-              - gemfiles/**
-      - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true'
-        uses: ruby/setup-ruby@v1
+          name: gemfiles-${{ github.run_id }}-${{ matrix.engine.name }}-${{ matrix.engine.version }}
+          path: gemfiles/${{ matrix.engine.name }}_${{ matrix.engine.version }}*
+
+  commit:
+    needs: lock
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
         with:
-          ruby-version: '3.2'
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-      - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true'
-        name: Ensure gemfiles/*.gemfile.lock match gem definition
-        run: bundle exec rake appraisal:lock
-      - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true'
-        name: Add all supported platforms to gemfiles/*.gemfile.lock
-        run: bundle exec rake appraisal:platform
-      - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true'
-        name: Remove obsolete gemfiles/*
-        run: bundle exec rake appraisal:clean
-      - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true'
-        name: Commit gemfiles changes, if any, back to the branch
-        uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.16.0
+          path: gemfiles
+          pattern: gemfiles-${{ github.run_id }}-*
+          merge-multiple: true
+
+      - run: git diff
+
+      - uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit_message: Update gemfiles/*
+          file_pattern: 'gemfiles/*'
+          commit_message: "[🤖]: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

Appraisals

@@ -52,7 +52,7 @@ def build_coverage_matrix(integration, range, gem: nil, min: nil, meta: {})
 
   if min
     appraise "#{integration}-min" do
-      gem gem, "= #{n}"
+      gem gem, "= #{min}"
       meta.each { |k, v| gem k, v }
     end
   end

appraisal/jruby-9.2.rb

@@ -189,7 +189,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/jruby-9.3.rb

@@ -162,7 +162,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/jruby-9.4.rb

@@ -66,7 +66,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-2.5.rb

@@ -209,7 +209,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-2.6.rb

@@ -162,7 +162,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-2.7.rb

@@ -162,7 +162,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-3.0.rb

@@ -75,7 +75,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-3.1.rb

@@ -75,7 +75,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-3.2.rb

@@ -75,7 +75,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-3.3.rb

@@ -75,7 +75,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

appraisal/ruby-3.4.rb

@@ -75,7 +75,7 @@
   gem 'typhoeus'
 end
 
-build_coverage_matrix('stripe', 7..12)
+build_coverage_matrix('stripe', 7..12, min: '5.15.0')
 build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby')
 build_coverage_matrix('elasticsearch', 7..8)
 

tasks/appraisal_conversion.rb

@@ -0,0 +1,49 @@
+require 'pathname'
+
+# This module translates our custom mapping between appraisal and bundler.
+#
+# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`.
+module AppraisalConversion
+  module_function
+
+  @gemfile_dir = 'gemfiles'
+  @definition_dir = 'appraisal'
+
+  def to_bundle_gemfile(group)
+    gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_')
+    path = root_path.join(gemfile_dir, gemfile)
+
+    if path.exist?
+      path.to_s
+    else
+      raise "Gemfile not found at #{path}"
+    end
+  end
+
+  def definition
+    path = root_path.join(@definition_dir, "#{runtime_identifier}.rb")
+
+    if path.exist?
+      path.to_s
+    else
+      raise "Definition not found at #{path}"
+    end
+  end
+
+  def runtime_identifier
+    major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments
+    "#{RUBY_ENGINE}-#{major}.#{minor}"
+  end
+
+  def gemfile_pattern
+    root_path + gemfile_dir + "#{runtime_identifier.tr('-', '_')}_*.gemfile"
+  end
+
+  def gemfile_dir
+    @gemfile_dir
+  end
+
+  def root_path
+    Pathname.pwd
+  end
+end

tasks/dependency.rake

@@ -0,0 +1,24 @@
+require 'open3'
+
+require_relative 'appraisal_conversion'
+
+task :dependency => %w[dependency:lock]
+namespace :dependency do
+  # rubocop:disable Style/MultilineBlockChain
+  Dir.glob(AppraisalConversion.gemfile_pattern).each do |gemfile|
+    # desc "Lock the dependencies for #{gemfile}"
+    task gemfile do
+      Bundler.with_unbundled_env do
+        command = +'bundle lock'
+        command << ' --add-platform x86_64-linux aarch64-linux' unless RUBY_PLATFORM == 'java'
+        output, = Open3.capture2e({ 'BUNDLE_GEMFILE' => gemfile.to_s }, command)
+
+        puts output
+      end
+    end
+  end.tap do |gemfiles|
+    desc "Lock the dependencies for #{AppraisalConversion.runtime_identifier}"
+    multitask :lock => gemfiles
+  end
+  # rubocop:enable Style/MultilineBlockChain
+end

tasks/edge.rake

@@ -1,47 +1,6 @@
-# frozen_string_literal: true
-
 require 'open3'
-require 'pathname'
-
-# This module translates our custom mapping between appraisal and bundler.
-#
-# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`.
-module AppraisalConversion
-  module_function
-
-  @gemfile_dir = 'gemfiles'
-  @definition_dir = 'appraisal'
-
-  def to_bundle_gemfile(group)
-    gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_')
-    path = root_path.join(@gemfile_dir, gemfile)
-
-    if path.exist?
-      path.to_s
-    else
-      raise "Gemfile not found at #{path}"
-    end
-  end
-
-  def definition
-    path = root_path.join(@definition_dir, "#{runtime_identifier}.rb")
 
-    if path.exist?
-      path.to_s
-    else
-      raise "Definition not found at #{path}"
-    end
-  end
-
-  def runtime_identifier
-    major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments
-    "#{RUBY_ENGINE}-#{major}.#{minor}"
-  end
-
-  def root_path
-    Pathname.pwd
-  end
-end
+require_relative 'appraisal_conversion'
 
 # rubocop:disable Metrics/BlockLength
 namespace :edge do