diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index a5f15c30..e6dfd423 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -113,3 +113,9 @@ steps:
       DOCKER_IMAGE: "jruby"
       RUBY_VERSION: "9.3"
       BUNDLE_GEMFILE: "/work/Gemfile-legacy"
+
+  - command: "auto/run-specs"
+    label: "rspec (jruby-9.4)"
+    env:
+      DOCKER_IMAGE: "jruby"
+      RUBY_VERSION: "9.4"
diff --git a/Gemfile b/Gemfile
index 7a404a26..734a56ea 100644
--- a/Gemfile
+++ b/Gemfile
@@ -8,8 +8,5 @@ gem "sorbet", "0.5.10821"
 gem "tapioca", "0.11.6", require: false
 gem 'parlour'
 
-# This is an optional dependency at runtime, but load it in development so tapioca generates rbi files
-gem "ascii85_native"
-
 # Required by yard. Part of stdlib in older rubies, but on modern rubies it's a gem
 gem "webrick"