diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml
index 01fdb85..b866c20 100644
--- a/.expeditor/verify.pipeline.yml
+++ b/.expeditor/verify.pipeline.yml
@@ -17,14 +17,6 @@ steps:
       docker:
         image: ruby:2.7-buster
 
-- label: run-specs-ruby-2.4
-  command:
-    - .expeditor/run_linux_tests.sh rspec
-  expeditor:
-    executor:
-      docker:
-        image: ruby:2.4-buster
-
 - label: run-specs-ruby-2.5
   command:
     - .expeditor/run_linux_tests.sh rspec
diff --git a/Gemfile b/Gemfile
index 2e6d11a..2837e89 100644
--- a/Gemfile
+++ b/Gemfile
@@ -5,6 +5,11 @@ gemspec
 
 gem "mixlib-shellout", "< 3.2.3"
 
+if Gem.ruby_version.to_s.start_with?("2.5")
+  # 16.7.23 required ruby 2.6+
+  gem "chef-utils", "< 16.7.23" # TODO: remove when we drop ruby 2.5
+end
+
 group :docs do
   gem "yard"
   gem "github-markup"
diff --git a/win32-certstore.gemspec b/win32-certstore.gemspec
index 3f3b2ba..d71fe37 100644
--- a/win32-certstore.gemspec
+++ b/win32-certstore.gemspec
@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.summary       = "Ruby library for accessing the certificate stores on Windows."
   spec.homepage      = "https://github.com/chef/win32-certstore"
 
-  spec.required_ruby_version = ">= 2.3"
+  spec.required_ruby_version = ">= 2.5"
 
   spec.files         = Dir["LICENSE", "lib/**/*"]
   spec.require_paths = ["lib"]