diff --git a/Gemfile b/Gemfile
index 743879d..253066a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -5,6 +5,7 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in sigstore.gemspec
 gemspec
 
+gem "base64", "~> 0.2.0" # Until https://github.com/vcr/vcr/commit/5c9230b43b6a51dec78941d16bf8e2954042964c is released
 gem "rake", "~> 13.0"
 gem "rubocop", "~> 1.21"
 gem "rubocop-rake", "~> 0.6.0"
diff --git a/Gemfile.lock b/Gemfile.lock
index cea0bde..6fb1bdc 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -10,6 +10,7 @@ GEM
     addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
+    base64 (0.2.0)
     bigdecimal (3.1.6)
     bigdecimal (3.1.6-java)
     crack (1.0.0)
@@ -84,6 +85,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  base64 (~> 0.2.0)
   rake (~> 13.0)
   rubocop (~> 1.21)
   rubocop-rake (~> 0.6.0)
@@ -97,6 +99,7 @@ DEPENDENCIES
 CHECKSUMS
   addressable (2.8.6) sha256=798f6af3556641a7619bad1dce04cdb6eb44b0216a991b0396ea7339276f2b47
   ast (2.4.2) sha256=1e280232e6a33754cde542bc5ef85520b74db2aac73ec14acef453784447cc12
+  base64 (0.2.0) sha256=0f25e9b21a02a0cc0cea8ef92b2041035d39350946e8789c562b2d1a3da01507
   bigdecimal (3.1.6) sha256=bcbc27d449cf8ed1b1814d21308f49c9d22ce73e33fff0d228e38799c02eab01
   bigdecimal (3.1.6-java) sha256=2ef0e13a578e2411123254273f8b34c47ff9d45de91a6f64465fb217de8d5d04
   crack (1.0.0) sha256=c83aefdb428cdc7b66c7f287e488c796f055c0839e6e545fec2c7047743c4a49
diff --git a/bin/conformance-entrypoint b/bin/conformance-entrypoint
index d1c58c5..03220c1 100755
--- a/bin/conformance-entrypoint
+++ b/bin/conformance-entrypoint
@@ -23,4 +23,9 @@ ENV.update(
 require "rubygems/gem_runner"
 
 ARGV.unshift "sigstore_#{command.tr("-", "_")}"
-Gem::GemRunner.new.run ARGV.clone
+begin
+  Gem::GemRunner.new.run ARGV.clone
+rescue Gem::SystemExitException => e
+  # For rubygems < 3.3.0
+  exit e.exit_code
+end