diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6a644b4..d662617 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,8 +19,6 @@ jobs: - 2.5 - 2.4 - 2.3 - - 2.2 - - 2.1.9 libre2: - version: "20150501" soname: 0 @@ -55,41 +53,32 @@ jobs: with: ruby-version: "${{ matrix.ruby }}" bundler-cache: true + - run: bundle exec rake compile -- --enable-system-libraries - run: bundle exec rake - legacy: - name: Legacy Ruby ${{ matrix.ruby }} - libre2 ABI version ${{ matrix.libre2.soname }} + vendor: + name: Ruby ${{ matrix.ruby }} - vendored libre2 and abseil runs-on: ubuntu-20.04 - container: - image: "mudge/re2-ci:${{ matrix.ruby }}" - options: "--add-host rubygems.org:151.101.129.227 --add-host api.rubygems.org:151.101.129.227" strategy: matrix: ruby: - - 1.8 - - 1.9.1 - - '2.0' - libre2: - - version: "20150501" - soname: 0 - - version: "20200302" - soname: 1 - - version: "20200303" - soname: 6 - - version: "20200501" - soname: 7 - - version: "20200706" - soname: 8 - - version: "20201101" - soname: 9 - - version: "20221201" - soname: 10 + - '3.2' + - '3.1' + - '3.0' + - 2.7 + - 2.6 + - 2.5 + - 2.4 + - 2.3 steps: - uses: actions/checkout@v3 - - name: Download and install specific release of libre2 + - name: Install build dependencies run: | - curl -Lo libre2-dev.deb https://github.com/mudge/re2-ci/releases/download/v1/libre2-dev_${{ matrix.libre2.version }}_amd64.deb - dpkg -i libre2-dev.deb + sudo apt-get install -y build-essential cmake + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - name: Configure Bundler for Ruby dependencies run: bundle config --local path vendor/bundle - name: Generate Gemfile.lock @@ -100,4 +89,5 @@ jobs: path: vendor/bundle key: gems-v1-${{ runner.os }}-${{ matrix.ruby }}-${{ hashFiles('Gemfile.lock') }} - run: bundle install --jobs 4 + - run: bundle exec rake compile -- --disable-system-libraries - run: bundle exec rake diff --git a/Rakefile b/Rakefile index 957a0c2..e942972 100644 --- a/Rakefile +++ b/Rakefile @@ -1,10 +1,121 @@ +# frozen_string_literal: true + require 'rake/extensiontask' require 'rspec/core/rake_task' +require 'rake_compiler_dock' + +CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], + FileList['**/extconf.h'], + FileList['**/Makefile'], + FileList['pkg/'] + +CLOBBER.include FileList['**/tmp'], + FileList['**/*.log'], + FileList['doc/**'], + FileList['tmp/'] +CLOBBER.add("ports/*").exclude(%r{ports/archives$}) + +RE2_GEM_SPEC = Gem::Specification.load('re2.gemspec') + +Gem::PackageTask.new(RE2_GEM_SPEC) do |p| + p.need_zip = false + p.need_tar = false +end -Rake::ExtensionTask.new('re2') +CROSS_RUBY_VERSIONS = %w[3.2.0 3.1.0 3.0.0 2.7.0].join(':') +CROSS_RUBY_PLATFORMS = %w[ + aarch64-linux + arm-linux + arm64-darwin + x64-mingw-ucrt + x86-linux + x86-mingw32 + x86_64-darwin + x86_64-linux +].freeze + +ENV['RUBY_CC_VERSION'] = CROSS_RUBY_VERSIONS + +Rake::ExtensionTask.new('re2', RE2_GEM_SPEC) do |e| + e.cross_compile = true + e.cross_config_options << '--enable-cross-build' + e.config_options << '--disable-system-libraries' + e.cross_platform = CROSS_RUBY_PLATFORMS + e.cross_compiling do |spec| + spec.files.reject! { |path| File.fnmatch?('ports/*', path) } + spec.dependencies.reject! { |dep| dep.name == 'mini_portile2' } + end +end RSpec::Core::RakeTask.new(:spec) +namespace 'gem' do + def gem_builder(platform) + # use Task#invoke because the pkg/*gem task is defined at runtime + Rake::Task["native:#{platform}"].invoke + Rake::Task["pkg/#{RE2_GEM_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke + end + + CROSS_RUBY_PLATFORMS.each do |platform| + # The Linux x86 image (ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-linux) + # is based on CentOS 7 and has two versions of cmake installed: + # a 2.8 version in /usr/bin and a 3.25 in /usr/local/bin. The latter is needed by abseil. + cmake = + case platform + when 'x86_64-linux', 'x86-linux' + '/usr/local/bin/cmake' + else + 'cmake' + end + + desc "build native gem for #{platform} platform" + task platform do + RakeCompilerDock.sh <<~SCRIPT, platform: platform, verbose: true + gem install bundler --no-document && + bundle && + bundle exec rake gem:#{platform}:builder CMAKE=#{cmake} + SCRIPT + end + + namespace platform do + desc "build native gem for #{platform} platform (guest container)" + task 'builder' do + gem_builder(platform) + end + end + end + + desc 'build all native gems' + task 'native' => CROSS_RUBY_PLATFORMS +end + +def add_file_to_gem(relative_source_path) + dest_path = File.join(gem_build_path, relative_source_path) + dest_dir = File.dirname(dest_path) + + mkdir_p dest_dir unless Dir.exist?(dest_dir) + rm_f dest_path if File.exist?(dest_path) + safe_ln relative_source_path, dest_path + + RE2_GEM_SPEC.files << relative_source_path +end + +def gem_build_path + File.join 'pkg', RE2_GEM_SPEC.full_name +end + +def add_vendored_libraries + dependencies = YAML.load_file(File.join(File.dirname(__FILE__), 'dependencies.yml')) + abseil_archive = File.join('ports', 'archives', "#{dependencies['abseil']['version']}.tar.gz") + libre2_archive = File.join('ports', 'archives', "re2-#{dependencies['libre2']['version']}.tar.gz") + + add_file_to_gem(abseil_archive) + add_file_to_gem(libre2_archive) +end + +task gem_build_path do + add_vendored_libraries +end + task :spec => :compile task :default => :spec - diff --git a/dependencies.yml b/dependencies.yml new file mode 100644 index 0000000..1043178 --- /dev/null +++ b/dependencies.yml @@ -0,0 +1,9 @@ +libre2: + version: "2023-07-01" + sha256: "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4" + # sha-256 hash provided in https://github.com/google/re2/releases/download/2023-07-01/re2-2023-07-01.tar.gz + +abseil: + version: "20230125.3" + sha256: 5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36 + # sha-256 hash provided in https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz diff --git a/ext/re2/extconf.rb b/ext/re2/extconf.rb index 9887232..f1f4bea 100644 --- a/ext/re2/extconf.rb +++ b/ext/re2/extconf.rb @@ -6,6 +6,157 @@ require 'mkmf' +PACKAGE_ROOT_DIR = File.expand_path(File.join(File.dirname(__FILE__), '..', '..')) + +REQUIRED_MINI_PORTILE_VERSION = "~> 2.8.2" # keep this version in sync with the one in the gemspec + +RE2_HELP_MESSAGE = <<~HELP + USAGE: ruby #{$0} [options] + + Flags that are always valid: + + --enable-system-libraries + Use system libraries instead of building and using the packaged libraries. + + --disable-system-libraries + Use the packaged libraries, and ignore the system libraries. This is the default. + + + Flags only used when using system libraries: + + Related to re2 library: + + --with-re2-dir=DIRECTORY + Look for re2 headers and library in DIRECTORY. + + Environment variables used: + + CC + Use this path to invoke the compiler instead of `RbConfig::CONFIG['CC']` + + CPPFLAGS + If this string is accepted by the C preprocessor, add it to the flags passed to the C preprocessor + + CFLAGS + If this string is accepted by the compiler, add it to the flags passed to the compiler + + LDFLAGS + If this string is accepted by the linker, add it to the flags passed to the linker + + LIBS + Add this string to the flags passed to the linker +HELP + +# +# utility functions +# +def config_system_libraries? + enable_config("system-libraries", ENV.key?('RE2_USE_SYSTEM_LIBRARIES')) +end + +def concat_flags(*args) + args.compact.join(" ") +end + +def do_help + print(RE2_HELP_MESSAGE) + exit!(0) +end + +def darwin? + RbConfig::CONFIG["target_os"].include?("darwin") +end + +def windows? + # Use =~ instead of match? to preserve Ruby 2.3 compatibility + RbConfig::CONFIG["target_os"] =~ /mingw|mswin/ +end + +def freebsd? + RbConfig::CONFIG["target_os"].include?("freebsd") +end + +def target_host + # We use 'host' to set compiler prefix for cross-compiling. Prefer host_alias over host. And + # prefer i686 (what external dev tools use) to i386 (what ruby's configure.ac emits). + host = RbConfig::CONFIG["host_alias"].empty? ? RbConfig::CONFIG["host"] : RbConfig::CONFIG["host_alias"] + host.gsub(/i386/, "i686") +end + +def find_compiler(compilers) + compilers.find { |binary| find_executable(binary) } +end + +# configure automatically searches for the right compiler based on the +# `--host` parameter. However, we don't have that feature with +# cmake. Search for the right compiler for the target architecture using +# some basic heruistics. +# See https://github.com/flavorjones/mini_portile/issues/128. +def find_c_and_cxx_compilers(host) + c_compiler = ENV["CC"] + cxx_compiler = ENV["CXX"] + + if darwin? + c_compiler ||= 'clang' + cxx_compiler ||='clang++' + else + c_compiler ||= 'gcc' + cxx_compiler ||= 'g++' + end + + c_platform_compiler = "#{host}-#{c_compiler}" + cxx_platform_compiler = "#{host}-#{cxx_compiler}" + c_compiler = find_compiler([c_platform_compiler, c_compiler]) + cxx_compiler = find_compiler([cxx_platform_compiler, cxx_compiler]) + + [c_compiler, cxx_compiler] +end + +def cmake_system_name + if darwin? + 'Darwin' + elsif windows? + 'Windows' + elsif freebsd? + 'FreeBSD' + else + 'Linux' + end +end + +def cmake_compile_flags(host) + c_compiler, cxx_compiler = find_c_and_cxx_compilers(host) + + # needed to ensure cross-compilation with CMake targets the right CPU and compilers + [ + "-DCMAKE_SYSTEM_PROCESSOR=#{RbConfig::CONFIG['target_cpu']}", + "-DCMAKE_SYSTEM_NAME=#{cmake_system_name}", + "-DCMAKE_C_COMPILER=#{c_compiler}", + "-DCMAKE_CXX_COMPILER=#{cxx_compiler}" + ] +end + +# By default, mini_portile2 might add an unnecessary option: +# https://github.com/flavorjones/mini_portile/blob/5084a2aeab12076f534cf0cabc81a4d5f84b5c25/lib/mini_portile2/mini_portile_cmake.rb#L17 +# See https://github.com/flavorjones/mini_portile/issues/127. +def delete_cmake_generator_option!(options) + indices = [] + + options.each_with_index do |element, index| + if element == '-G' && index + 1 < options.length + indices << index + indices << index + 1 + end + end + + indices.reverse_each { |index| options.delete_at(index) } +end + +# +# main +# +do_help if arg_config('--help') + if ENV["CC"] RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"] RbConfig::CONFIG["CC"] = ENV["CC"] @@ -16,70 +167,57 @@ RbConfig::CONFIG["CXX"] = ENV["CXX"] end -header_dirs = [ - "/usr/local/include", - "/opt/homebrew/include", - "/usr/include" -] - -lib_dirs = [ - "/usr/local/lib", - "/opt/homebrew/lib", - "/usr/lib" -] - -dir_config("re2", header_dirs, lib_dirs) - -$CFLAGS << " -Wall -Wextra -funroll-loops" +def build_extension + $CFLAGS << " -Wall -Wextra -funroll-loops" -# Pass -x c++ to force gcc to compile the test program -# as C++ (as it will end in .c by default). -compile_options = "-x c++" + # Pass -x c++ to force gcc to compile the test program + # as C++ (as it will end in .c by default). + compile_options = "-x c++" -have_library("stdc++") -have_header("stdint.h") -have_func("rb_str_sublen") + have_library("stdc++") + have_header("stdint.h") + have_func("rb_str_sublen") -unless have_library("re2") - abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" -end + unless have_library("re2") + abort "You must have re2 installed and specified with --with-re2-dir, please see https://github.com/google/re2/wiki/Install" + end -minimal_program = < int main() { return 0; } SRC -re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do - !try_compile(minimal_program, compile_options) -end + re2_requires_version_flag = checking_for("re2 that requires explicit C++ version flag") do + !try_compile(minimal_program, compile_options) + end -if re2_requires_version_flag - # Recent versions of re2 depend directly on abseil, which requires a - # compiler with C++14 support (see - # https://github.com/abseil/abseil-cpp/issues/1127 and - # https://github.com/abseil/abseil-cpp/issues/1431). However, the - # `std=c++14` flag doesn't appear to suffice; we need at least - # `std=c++17`. - abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| - checking_for("re2 that compiles with #{std} standard") do - if try_compile(minimal_program, compile_options + " -std=#{std}") - compile_options << " -std=#{std}" - $CPPFLAGS << " -std=#{std}" + if re2_requires_version_flag + # Recent versions of re2 depend directly on abseil, which requires a + # compiler with C++14 support (see + # https://github.com/abseil/abseil-cpp/issues/1127 and + # https://github.com/abseil/abseil-cpp/issues/1431). However, the + # `std=c++14` flag doesn't appear to suffice; we need at least + # `std=c++17`. + abort "Cannot compile re2 with your compiler: recent versions require C++14 support." unless %w[c++20 c++17 c++11 c++0x].any? do |std| + checking_for("re2 that compiles with #{std} standard") do + if try_compile(minimal_program, compile_options + " -std=#{std}") + compile_options << " -std=#{std}" + $CPPFLAGS << " -std=#{std}" - true + true + end end end end -end -# Determine which version of re2 the user has installed. -# Revision d9f8806c004d added an `endpos` argument to the -# generic Match() function. -# -# To test for this, try to compile a simple program that uses -# the newer form of Match() and set a flag if it is successful. -checking_for("RE2::Match() with endpos argument") do - test_re2_match_signature = < int main() { @@ -91,13 +229,13 @@ } SRC - if try_compile(test_re2_match_signature, compile_options) - $defs.push("-DHAVE_ENDPOS_ARGUMENT") + if try_compile(test_re2_match_signature, compile_options) + $defs.push("-DHAVE_ENDPOS_ARGUMENT") + end end -end -checking_for("RE2::Set::Match() with error information") do - test_re2_set_match_signature = < #include #include @@ -115,9 +253,126 @@ } SRC - if try_compile(test_re2_set_match_signature, compile_options) - $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + if try_compile(test_re2_set_match_signature, compile_options) + $defs.push("-DHAVE_ERROR_INFO_ARGUMENT") + end end end +def process_recipe(name, version) + require "rubygems" + gem("mini_portile2", REQUIRED_MINI_PORTILE_VERSION) # gemspec is not respected at install time + require "mini_portile2" + message("Using mini_portile version #{MiniPortile::VERSION}\n") + + MiniPortileCMake.new(name, version).tap do |recipe| + recipe.host = target_host + recipe.target = File.join(PACKAGE_ROOT_DIR, "ports") + + recipe.configure_options += [ + # abseil needs a C++14 compiler + '-DCMAKE_CXX_STANDARD=17', + # needed for building the C extension shared library with -fPIC + '-DCMAKE_POSITION_INDEPENDENT_CODE=ON', + # ensures pkg-config and installed libraries will be in lib, not lib64 + '-DCMAKE_INSTALL_LIBDIR=lib' + ] + recipe.configure_options += cmake_compile_flags(recipe.host) + delete_cmake_generator_option!(recipe.configure_options) + + yield recipe + + checkpoint = "#{recipe.target}/#{recipe.name}-#{recipe.version}-#{recipe.host}.installed" + + if File.exist?(checkpoint) + message("Building re2 with a packaged version of #{name}-#{version}.\n") + else + message(<<~EOM) + ---------- IMPORTANT NOTICE ---------- + Building re2 with a packaged version of #{name}-#{version}. + Configuration options: #{recipe.configure_options.shelljoin} + EOM + + unless recipe.patch_files.empty? + message("The following patches are being applied:\n") + + recipe.patch_files.each do |patch| + message(" - %s\n" % File.basename(patch)) + end + end + + recipe.cook + + FileUtils.touch(checkpoint) + end + + recipe.activate + end +end + +def build_with_system_libraries + header_dirs = [ + "/usr/local/include", + "/opt/homebrew/include", + "/usr/include" + ] + + lib_dirs = [ + "/usr/local/lib", + "/opt/homebrew/lib", + "/usr/lib" + ] + + dir_config("re2", header_dirs, lib_dirs) + + build_extension +end + +def build_with_vendored_libraries + message "Building re2 using packaged libraries.\n" + + require 'yaml' + dependencies = YAML.load_file(File.join(PACKAGE_ROOT_DIR, 'dependencies.yml')) + + abseil_recipe = process_recipe('abseil', dependencies['abseil']['version']) do |recipe| + recipe.files = [{ + url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/#{recipe.version}.tar.gz", + sha256: dependencies['abseil']['sha256'] + }] + recipe.configure_options += ['-DABSL_PROPAGATE_CXX_STD=ON'] + end + + re2_recipe = process_recipe('libre2', dependencies['libre2']['version']) do |recipe| + recipe.files = [{ + url: "https://github.com/google/re2/releases/download/#{recipe.version}/re2-#{recipe.version}.tar.gz", + sha256: dependencies['libre2']['sha256'] + }] + recipe.configure_options += ["-DCMAKE_PREFIX_PATH=#{abseil_recipe.path}", '-DCMAKE_CXX_FLAGS=-DNDEBUG'] + end + + pkg_config_paths = [ + "#{abseil_recipe.path}/lib/pkgconfig", + "#{re2_recipe.path}/lib/pkgconfig" + ].join(':') + + pkg_config_paths = "#{ENV['PKG_CONFIG_PATH']}:#{pkg_config_paths}" if ENV['PKG_CONFIG_PATH'] + + ENV['PKG_CONFIG_PATH'] = pkg_config_paths + pc_file = File.join(re2_recipe.path, 'lib', 'pkgconfig', 're2.pc') + + raise 'Please install the `pkg-config` utility!' unless pkg_config('re2') + + # See https://bugs.ruby-lang.org/issues/18490, broken in Ruby 3.1 but fixed in Ruby 3.2. + flags = xpopen(['pkg-config', '--libs', '--static', pc_file], err: %i[child out], &:read) + flags.split.each { |flag| append_ldflags(flag) } if $?.success? + + build_extension +end + +if config_system_libraries? + build_with_system_libraries +else + build_with_vendored_libraries +end + create_makefile("re2") diff --git a/lib/re2.rb b/lib/re2.rb index 5870b47..9c903ff 100644 --- a/lib/re2.rb +++ b/lib/re2.rb @@ -3,5 +3,11 @@ # # Copyright (c) 2010-2014, Paul Mucur (http://mudge.name) # Released under the BSD Licence, please see LICENSE.txt -require "re2.so" +begin + ::RUBY_VERSION =~ /(\d+\.\d+)/ + require_relative "#{Regexp.last_match(1)}/re2.so" +rescue LoadError + require 're2.so' +end + require "re2/scanner" diff --git a/re2.gemspec b/re2.gemspec index e64b5a6..873ce5e 100644 --- a/re2.gemspec +++ b/re2.gemspec @@ -27,6 +27,8 @@ Gem::Specification.new do |s| "spec/re2/set_spec.rb", "spec/re2/scanner_spec.rb" ] - s.add_development_dependency("rake-compiler", "~> 0.9") + s.add_development_dependency "rake-compiler", "~> 1.2.1" + s.add_development_dependency "rake-compiler-dock", "~> 1.3.0" s.add_development_dependency("rspec", "~> 3.2") + s.add_runtime_dependency("mini_portile2", "~> 2.8.2") # keep version in sync with extconf.rb end