From 80e5120c03ac3d1d72f4c99d44dd332a44f6b78c Mon Sep 17 00:00:00 2001 From: Tony Hsu Date: Fri, 22 Nov 2024 10:23:04 +0100 Subject: [PATCH] Add Github Actions tests --- .github/workflows/test.yml | 91 ++++++++++++++++++ Rakefile | 30 +++--- tasks/appraisal_conversion.rb | 49 ++++++++++ tasks/github.rake | 170 ++++++++++++++++++++++++++++++++++ 4 files changed, 325 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 tasks/appraisal_conversion.rb create mode 100644 tasks/github.rake diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..65f0421c --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,91 @@ +# Please do NOT manually edit this file. +# This file is generated by 'bundle exec rake github:actions:test_template' +--- +name: Unit Tests +'on': +- push +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: "${{ github.ref != 'refs/heads/master' }}" +jobs: + compute_tasks: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + engine: + - name: ruby + version: '3.3' + alias: ruby-33 + - name: ruby + version: '3.2' + alias: ruby-32 + container: + image: ghcr.io/datadog/images-rb/engines/${{ matrix.engine.name }}:${{ matrix.engine.version }} + outputs: + ruby-33-matrix: "${{ steps.set-matrix.outputs.ruby-33 }}" + ruby-32-matrix: "${{ steps.set-matrix.outputs.ruby-32 }}" + steps: + - uses: actions/checkout@v4 + - run: bundle install + - id: set-matrix + run: | + matrix_json=$(bundle exec rake github:generate_matrix) + # Debug output + echo "Generated JSON:" + echo "$matrix_json" + # Set the output + echo "${{ matrix.engine.alias }}=$(echo "$matrix_json")" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v4 + with: + name: bundled-lock-${{ github.run_id }}-${{ matrix.engine.alias }} + retention-days: 1 + path: 'Gemfile.lock + + ' + test-ruby-33: + name: 'ruby-3.3: ${{ matrix.task }} (${{ matrix.group }})' + needs: + - compute_tasks + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.compute_tasks.outputs.ruby-33-matrix) }}" + container: + image: ghcr.io/datadog/images-rb/engines/ruby:3.3 + env: + BUNDLE_GEMFILE: "${{ matrix.gemfile }}" + steps: + - uses: actions/checkout@v4 + - name: Configure Git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/download-artifact@v4 + with: + name: bundled-lock-${{ github.run_id }}-ruby-33 + - run: bundle install && bundle exec rake compile_ext + - name: Test ${{ matrix.task }} with ${{ matrix.gemfile }} + run: bundle exec rake spec:${{ matrix.task }} + test-ruby-32: + name: 'ruby-3.2: ${{ matrix.task }} (${{ matrix.group }})' + needs: + - compute_tasks + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: "${{ fromJSON(needs.compute_tasks.outputs.ruby-32-matrix) }}" + container: + image: ghcr.io/datadog/images-rb/engines/ruby:3.2 + env: + BUNDLE_GEMFILE: "${{ matrix.gemfile }}" + steps: + - uses: actions/checkout@v4 + - name: Configure Git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/download-artifact@v4 + with: + name: bundled-lock-${{ github.run_id }}-ruby-32 + - run: bundle install && bundle exec rake compile_ext + - name: Test ${{ matrix.task }} with ${{ matrix.gemfile }} + run: bundle exec rake spec:${{ matrix.task }} diff --git a/Rakefile b/Rakefile index b2106288..6692daad 100644 --- a/Rakefile +++ b/Rakefile @@ -20,21 +20,6 @@ if Gem.loaded_specs.key? "ruby_memcheck" ) end -RSpec::Core::RakeTask.new(:spec) - -Dir.glob("tasks/*.rake").each { |r| import r } - -YARD::Rake::YardocTask.new(:docs) do |t| - # Options defined in `.yardopts` are read first, then merged with - # options defined here. - # - # It's recommended to define options in `.yardopts` instead of here, - # as `.yardopts` can be read by external YARD tools, like the - # hot-reload YARD server `yard server --reload`. - - t.options += ["--title", "datadog-ci #{Datadog::CI::VERSION} documentation"] -end - # ADD NEW RUBIES HERE TEST_METADATA = { "main" => { @@ -84,6 +69,21 @@ TEST_METADATA = { } } +RSpec::Core::RakeTask.new(:spec) + +Dir.glob("tasks/*.rake").each { |r| import r } + +YARD::Rake::YardocTask.new(:docs) do |t| + # Options defined in `.yardopts` are read first, then merged with + # options defined here. + # + # It's recommended to define options in `.yardopts` instead of here, + # as `.yardopts` can be read by external YARD tools, like the + # hot-reload YARD server `yard server --reload`. + + t.options += ["--title", "datadog-ci #{Datadog::CI::VERSION} documentation"] +end + namespace :test do task all: TEST_METADATA.map { |k, _| "test:#{k}" } diff --git a/tasks/appraisal_conversion.rb b/tasks/appraisal_conversion.rb new file mode 100644 index 00000000..685c5449 --- /dev/null +++ b/tasks/appraisal_conversion.rb @@ -0,0 +1,49 @@ +require 'pathname' + +# This module translates our custom mapping between appraisal and bundler. +# +# It cannot be included into `Appraisal` file, because it was invoked via `instance_eval`. +module AppraisalConversion + module_function + + @gemfile_dir = 'gemfiles' + @definition_dir = 'appraisal' + + def to_bundle_gemfile(group) + gemfile = "#{runtime_identifier}_#{group}.gemfile".tr('-', '_') + path = root_path.join(gemfile_dir, gemfile) + + if path.exist? + path.to_s + else + raise "Gemfile not found at #{path}" + end + end + + def definition + path = root_path.join(@definition_dir, "#{runtime_identifier}.rb") + + if path.exist? + path.to_s + else + raise "Definition not found at #{path}" + end + end + + def runtime_identifier + major, minor, = Gem::Version.new(RUBY_ENGINE_VERSION).segments + "#{RUBY_ENGINE}-#{major}.#{minor}" + end + + def gemfile_pattern + root_path + gemfile_dir + "#{runtime_identifier.tr('-', '_')}_*.gemfile" + end + + def gemfile_dir + @gemfile_dir + end + + def root_path + Pathname.pwd + end +end diff --git a/tasks/github.rake b/tasks/github.rake new file mode 100644 index 00000000..c143495e --- /dev/null +++ b/tasks/github.rake @@ -0,0 +1,170 @@ +require 'json' +require "psych" +require 'ostruct' +require_relative 'appraisal_conversion' + +# rubocop:disable Metrics/BlockLength +namespace :github do + namespace :actions do + task :test_template do |t| + ubuntu = "ubuntu-22.04" + + runtimes = [ + "ruby:3.3", + "ruby:3.2", + # "ruby:3.1", + # "ruby:3.0", + # "ruby:2.7", + # "jruby:9.4", + ].map do |runtime| + engine, version = runtime.split(':') + runtime_alias = "#{engine}-#{version.gsub('.', '')}" + + OpenStruct.new( + "engine" => engine, + "version" => version, + "alias" => runtime_alias, + "image" => "ghcr.io/datadog/images-rb/engines/#{engine}:#{version}" + ) + end + + test_jobs = runtimes.map do |runtime| + { + "test-#{runtime.alias}" => { + "name" => "#{runtime.engine}-#{runtime.version}: ${{ matrix.task }} (${{ matrix.group }})", + "needs" => ["compute_tasks"], + "runs-on" => ubuntu, + "strategy" => { + "fail-fast" => false, + "matrix" => { + "include" => "${{ fromJSON(needs.compute_tasks.outputs.#{runtime.alias}-matrix) }}" + } + }, + "container" => { + "image" => runtime.image, + "env" => { + "BUNDLE_GEMFILE" => "${{ matrix.gemfile }}" + } + }, + "steps" => [ + { "uses" => "actions/checkout@v4" }, + { + "name" => "Configure Git", + "run" => 'git config --global --add safe.directory "$GITHUB_WORKSPACE"' + }, + { + "uses" => "actions/download-artifact@v4", + "with" => { + "name" => "bundled-lock-${{ github.run_id }}-#{runtime.alias}", + } + }, + { "run" => "bundle install && bundle exec rake compile_ext" }, + { + "name" => "Test ${{ matrix.task }} with ${{ matrix.gemfile }}", + "run" => "bundle exec rake spec:${{ matrix.task }}" + } + ] + } + } + end + + compute_tasks = { + "runs-on" => ubuntu, + "strategy" => { + "fail-fast" => false, + "matrix" => { + "engine" => runtimes.map do |runtime| + { "name" => runtime.engine, "version" => runtime.version, "alias" => runtime.alias } + end + } + }, + "container" =>{ + "image" => "ghcr.io/datadog/images-rb/engines/${{ matrix.engine.name }}:${{ matrix.engine.version }}" + }, + "outputs" => runtimes.each_with_object({}) do |runtime, hash| + hash["#{runtime.alias}-matrix"] = "${{ steps.set-matrix.outputs.#{runtime.alias} }}" + end, + "steps" => [ + { "uses" => "actions/checkout@v4" }, + { "run" => "bundle install" }, + { + "id" => "set-matrix", + "run" => <<~BASH + matrix_json=$(bundle exec rake github:generate_matrix) + # Debug output + echo "Generated JSON:" + echo "$matrix_json" + # Set the output + echo "${{ matrix.engine.alias }}=$(echo "$matrix_json")" >> $GITHUB_OUTPUT + BASH + }, + { + "uses" => "actions/upload-artifact@v4", + "with" => { + "name" => "bundled-lock-${{ github.run_id }}-${{ matrix.engine.alias }}", + "retention-days" => 1, + "path" => <<~STRING + Gemfile.lock + STRING + } + }, + ] + } + + base = { + "name" => 'Unit Tests', + "on" => ['push'], + "concurrency" => { + "group" => '${{ github.workflow }}-${{ github.ref }}', + "cancel-in-progress" => '${{ github.ref != \'refs/heads/master\' }}' + }, + "jobs" => { + "compute_tasks" => compute_tasks, + **test_jobs.reduce(&:merge) + } + } + + # `Psych.dump` directly creates anchors, but Github Actions does not support anchors for YAML, + # convert to JSON first to avoid anchors + json = JSON.dump(base) + yaml = Psych.safe_load(json) + + string = +"" + string << <<~EOS + # Please do NOT manually edit this file. + # This file is generated by 'bundle exec rake #{t.name}' + EOS + string << Psych.dump(yaml, line_width: 120) + File.binwrite(".github/workflows/test.yml", string) + end + end + + task :generate_matrix do + matrix = TEST_METADATA + + ruby_version = RUBY_VERSION[0..2] + array = [] + matrix.each do |key, spec_metadata| + spec_metadata.each do |group, rubies| + matched = if RUBY_PLATFORM == 'java' + rubies.include?("✅ #{ruby_version}") && rubies.include?('✅ jruby') + else + rubies.include?("✅ #{ruby_version}") + end + + if matched + gemfile = AppraisalConversion.to_bundle_gemfile(group) rescue "Gemfile" + + array << { + group: group, + gemfile: gemfile, + task: key + } + end + end + end + + puts JSON.dump(array) + end +end +# rubocop:enable Metrics/BlockLength