From ac99f5a993e15d25e64ab056cac9df4bae7d4a7f Mon Sep 17 00:00:00 2001 From: Tony Hsu Date: Wed, 18 Sep 2024 11:44:09 +0200 Subject: [PATCH] Improve update gemfiles --- .github/workflows/update-gemfiles.yml | 150 ++++++++++++++------------ Appraisals | 2 +- appraisal/jruby-9.2.rb | 2 +- appraisal/jruby-9.3.rb | 2 +- appraisal/jruby-9.4.rb | 2 +- appraisal/ruby-2.5.rb | 2 +- appraisal/ruby-2.6.rb | 2 +- appraisal/ruby-2.7.rb | 2 +- appraisal/ruby-3.0.rb | 2 +- appraisal/ruby-3.1.rb | 2 +- appraisal/ruby-3.2.rb | 2 +- appraisal/ruby-3.3.rb | 2 +- appraisal/ruby-3.4.rb | 2 +- tasks/appraisal_conversion.rb | 49 +++++++++ tasks/dependency.rake | 25 +++++ tasks/edge.rake | 43 +------- 16 files changed, 168 insertions(+), 123 deletions(-) create mode 100644 tasks/appraisal_conversion.rb create mode 100644 tasks/dependency.rake diff --git a/.github/workflows/update-gemfiles.yml b/.github/workflows/update-gemfiles.yml index 078a15ccd3d..9aa5b55c991 100644 --- a/.github/workflows/update-gemfiles.yml +++ b/.github/workflows/update-gemfiles.yml @@ -1,83 +1,95 @@ name: Update Gemfiles -# This action cannot be skipped altogether because it is a mandatory status check. -# Instead we conditionally skip it at job level, instead of workflow level. on: - # Execute on `push` and not `pull_request` because `pull_request` - # always compares if the `paths` have changed compared to the PR base. - # This is an issue because it means that all commits to the branch - # will trigger the gemfile update process, which is unnecessary and expensive. - # - # By executing on `push`, GitHub compares `paths` with the parent commit, - # meaning the gemfile update process will only execute on the exact commit - # that changes any of the `paths`. - # - # Because this process is slow and expensive, and we commit the gemfile changes back - # to the branch, we have an additional filter to only execute this action on branches - # attached to a PR. - # - # We could do the inverse: execute this action on `pull_request`, and additionally check - # if `paths` was changed compared to the parent commit, but this proved more complicated. - push + workflow_dispatch: + inputs: + branch: + description: 'Branch to be updated' + required: true + # To be removed once the workflow is fully automated + push: + branches: + - tonycthsu/automate-update-gemfiles # Ensure obsolete job is cancelled if another commit is pushed to the same branch. concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - check: - name: Update Gemfiles - runs-on: ubuntu-22.04 - permissions: - contents: write - pull-requests: write + # TODO: In order to fully automate this workflow for each PR, + # have a reliable way to precheck job to understand whether it need to be updated + lock: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + engine: + # ADD NEW RUBIES HERE + - name: ruby + version: '3.4' + - name: ruby + version: '3.3' + - name: ruby + version: '3.2' + - name: ruby + version: '3.1' + - name: ruby + version: '3.0' + - name: ruby + version: '2.7' + - name: ruby + version: '2.6' + - name: ruby + version: '2.5' + - name: jruby + version: '9.4' + - name: jruby + version: '9.3' + - name: jruby + version: '9.2' + container: + image: ghcr.io/datadog/images-rb/engines/${{ matrix.engine.name }}:${{ matrix.engine.version }} + env: + BUNDLE_WITHOUT: check steps: - # Only execute if there's a PR attached to this branch. - # Because we execute on `push`, we have to double check here if this is part of a PR. - - name: Check if this branch is attached to a Pull Request - uses: 8bitjonny/gh-get-current-pr@2215326c76d51bfa3f2af0a470f32677f6c0cae9 # v2.2.0 - id: pr - with: - filterOutClosed: true # Don't trigger on commits with closed PRs, including merges into `master`. - - if: steps.pr.outputs.pr_found == 'true' + - name: Checkout code uses: actions/checkout@v4 - # And also, only execute if files that can affect gemfiles are modified. - - if: steps.pr.outputs.pr_found == 'true' - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: filter + - run: ls -al + - run: | + ruby -v + gem -v + bundler -v + + - run: bundle install + + # TODO: Migrate away from `appraisal` + - run: bundle exec appraisal generate + - run: bundle exec rake dependency + + - name: Upload artifact + uses: actions/upload-artifact@v4 with: - base: ${{ github.ref_name }} - filters: | - gemfile: - # Files that declare the dependency tree - - Gemfile - - Appraisals - - datadog.gemspec - # Files that control gemfile generation - - tasks/appraisal.rake - - .github/workflows/update-gemfiles.yml - # The gem version is present in all lock files - - lib/datadog/version.rb - # In case the generated files were updated manually or in a merge commit - - appraisal/** - - gemfiles/** - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - uses: ruby/setup-ruby@v1 + name: gemfiles-${{ github.run_id }}-${{ matrix.engine.name }}-${{ matrix.engine.version }} + path: gemfiles/${{ matrix.engine.name }}_${{ matrix.engine.version }}* + + commit: + needs: lock + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 with: - ruby-version: '3.2' - bundler-cache: true # runs 'bundle install' and caches installed gems automatically - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Ensure gemfiles/*.gemfile.lock match gem definition - run: bundle exec rake appraisal:lock - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Add all supported platforms to gemfiles/*.gemfile.lock - run: bundle exec rake appraisal:platform - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Remove obsolete gemfiles/* - run: bundle exec rake appraisal:clean - - if: steps.pr.outputs.pr_found == 'true' && steps.filter.outputs.gemfile == 'true' - name: Commit gemfiles changes, if any, back to the branch - uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.16.0 + path: gemfiles + pattern: gemfiles-${{ github.run_id }}-* + merge-multiple: true + + - run: git diff + + - uses: stefanzweifel/git-auto-commit-action@v5 with: - commit_message: Update gemfiles/* + file_pattern: 'gemfiles/*' + commit_message: "[🤖]: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/Appraisals b/Appraisals index b312f409b3e..ef649f43103 100644 --- a/Appraisals +++ b/Appraisals @@ -52,7 +52,7 @@ def build_coverage_matrix(integration, range, gem: nil, min: nil, meta: {}) if min appraise "#{integration}-min" do - gem gem, "= #{n}" + gem gem, "= #{min}" meta.each { |k, v| gem k, v } end end diff --git a/appraisal/jruby-9.2.rb b/appraisal/jruby-9.2.rb index cd3a5619f49..ed8c523b69f 100644 --- a/appraisal/jruby-9.2.rb +++ b/appraisal/jruby-9.2.rb @@ -189,7 +189,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/jruby-9.3.rb b/appraisal/jruby-9.3.rb index 3a98ded6920..6556d4f81bb 100644 --- a/appraisal/jruby-9.3.rb +++ b/appraisal/jruby-9.3.rb @@ -162,7 +162,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/jruby-9.4.rb b/appraisal/jruby-9.4.rb index 1968306e239..2bf7c0d5698 100644 --- a/appraisal/jruby-9.4.rb +++ b/appraisal/jruby-9.4.rb @@ -66,7 +66,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-2.5.rb b/appraisal/ruby-2.5.rb index 2e8d2cf9772..d0ff148a8d1 100644 --- a/appraisal/ruby-2.5.rb +++ b/appraisal/ruby-2.5.rb @@ -209,7 +209,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-2.6.rb b/appraisal/ruby-2.6.rb index 312a76cdba9..45d196a33b0 100644 --- a/appraisal/ruby-2.6.rb +++ b/appraisal/ruby-2.6.rb @@ -162,7 +162,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-2.7.rb b/appraisal/ruby-2.7.rb index 4e02ac9adc2..e7c203513b7 100644 --- a/appraisal/ruby-2.7.rb +++ b/appraisal/ruby-2.7.rb @@ -162,7 +162,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.0.rb b/appraisal/ruby-3.0.rb index 7dcd32e7a3f..c0f2d58b3fc 100644 --- a/appraisal/ruby-3.0.rb +++ b/appraisal/ruby-3.0.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.1.rb b/appraisal/ruby-3.1.rb index 7dcd32e7a3f..c0f2d58b3fc 100644 --- a/appraisal/ruby-3.1.rb +++ b/appraisal/ruby-3.1.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.2.rb b/appraisal/ruby-3.2.rb index 7dcd32e7a3f..c0f2d58b3fc 100644 --- a/appraisal/ruby-3.2.rb +++ b/appraisal/ruby-3.2.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.3.rb b/appraisal/ruby-3.3.rb index 6bdd3958304..c740488fb1a 100644 --- a/appraisal/ruby-3.3.rb +++ b/appraisal/ruby-3.3.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/appraisal/ruby-3.4.rb b/appraisal/ruby-3.4.rb index e38331b9dba..97baa215598 100644 --- a/appraisal/ruby-3.4.rb +++ b/appraisal/ruby-3.4.rb @@ -75,7 +75,7 @@ gem 'typhoeus' end -build_coverage_matrix('stripe', 7..12) +build_coverage_matrix('stripe', 7..12, min: '5.15.0') build_coverage_matrix('opensearch', 2..3, gem: 'opensearch-ruby') build_coverage_matrix('elasticsearch', 7..8) diff --git a/tasks/appraisal_conversion.rb b/tasks/appraisal_conversion.rb new file mode 100644 index 00000000000..685c544911f --- /dev/null +++ b/tasks/appraisal_conversion.rb @@ -0,0 +1,49 @@ +require 'pathname' + +# This module translates our custom mapping between appraisal and bundler. +# +# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`. +module AppraisalConversion + module_function + + @gemfile_dir = 'gemfiles' + @definition_dir = 'appraisal' + + def to_bundle_gemfile(group) + gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_') + path = root_path.join(gemfile_dir, gemfile) + + if path.exist? + path.to_s + else + raise "Gemfile not found at #{path}" + end + end + + def definition + path = root_path.join(@definition_dir, "#{runtime_identifier}.rb") + + if path.exist? + path.to_s + else + raise "Definition not found at #{path}" + end + end + + def runtime_identifier + major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments + "#{RUBY_ENGINE}-#{major}.#{minor}" + end + + def gemfile_pattern + root_path + gemfile_dir + "#{runtime_identifier.tr('-', '_')}_*.gemfile" + end + + def gemfile_dir + @gemfile_dir + end + + def root_path + Pathname.pwd + end +end diff --git a/tasks/dependency.rake b/tasks/dependency.rake new file mode 100644 index 00000000000..4bfc7e749a6 --- /dev/null +++ b/tasks/dependency.rake @@ -0,0 +1,25 @@ +require 'open3' + +require_relative 'appraisal_conversion' + +task :dep => :dependency +task :dependency => %w[dependency:lock] +namespace :dependency do + # rubocop:disable Style/MultilineBlockChain + Dir.glob(AppraisalConversion.gemfile_pattern).each do |gemfile| + # desc "Lock the dependencies for #{gemfile}" + task gemfile do + Bundler.with_unbundled_env do + command = +'bundle lock' + command << ' --add-platform x86_64-linux aarch64-linux' unless RUBY_PLATFORM == 'java' + output, = Open3.capture2e({ 'BUNDLE_GEMFILE' => gemfile.to_s }, command) + + puts output + end + end + end.tap do |gemfiles| + desc "Lock the dependencies for #{AppraisalConversion.runtime_identifier}" + multitask :lock => gemfiles + end + # rubocop:enable Style/MultilineBlockChain +end diff --git a/tasks/edge.rake b/tasks/edge.rake index aee5e675150..5ee9766ccff 100644 --- a/tasks/edge.rake +++ b/tasks/edge.rake @@ -1,47 +1,6 @@ -# frozen_string_literal: true - require 'open3' -require 'pathname' - -# This module translates our custom mapping between appraisal and bundler. -# -# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`. -module AppraisalConversion - module_function - - @gemfile_dir = 'gemfiles' - @definition_dir = 'appraisal' - - def to_bundle_gemfile(group) - gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_') - path = root_path.join(@gemfile_dir, gemfile) - - if path.exist? - path.to_s - else - raise "Gemfile not found at #{path}" - end - end - - def definition - path = root_path.join(@definition_dir, "#{runtime_identifier}.rb") - if path.exist? - path.to_s - else - raise "Definition not found at #{path}" - end - end - - def runtime_identifier - major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments - "#{RUBY_ENGINE}-#{major}.#{minor}" - end - - def root_path - Pathname.pwd - end -end +require_relative 'appraisal_conversion' # rubocop:disable Metrics/BlockLength namespace :edge do