diff --git a/lib/bundler/plugin.rb b/lib/bundler/plugin.rb
index 015be99af80..8fb41193237 100644
--- a/lib/bundler/plugin.rb
+++ b/lib/bundler/plugin.rb
@@ -62,7 +62,9 @@ def gemfile_install(gemfile = nil, &inline)
 
       save_plugins plugins, installed_specs, builder.inferred_plugins
     rescue => e
-      Bundler.ui.error "Failed to install plugin: #{e.message}\n  #{e.backtrace[0]}"
+      unless e.is_a?(GemfileError)
+        Bundler.ui.error "Failed to install plugin: #{e.message}\n  #{e.backtrace[0]}"
+      end
       raise
     end