diff --git a/builder/infra.py b/builder/infra.py
index c6914bf..7e425f3 100644
--- a/builder/infra.py
+++ b/builder/infra.py
@@ -162,6 +162,6 @@ def remove_local_wheels(
     for binary in exists:
         version = binary_package_map[binary]
         print(f"Found existing wheels for {binary}, removing local copy {version}")
-        for wheel in wheel_map[binary]:
+        for wheel in wheel_map.get(binary, ()):
             print(f"Removing local wheel {wheel}")
             wheel.unlink()
diff --git a/tests/test_infra.py b/tests/test_infra.py
index 2646958..9dc01e1 100644
--- a/tests/test_infra.py
+++ b/tests/test_infra.py
@@ -259,3 +259,18 @@ def test_remove_local_wheel_normalized_package_names(tmppath: Path) -> None:
 
     # grpcio and google-cloud-pubsub are removed
     assert {p.name for p in tmppath.glob("*.whl")} == set()
+
+
+def test_remove_local_wheel_no_build_wheels(tmppath: Path) -> None:
+    """Test remove_local_wheels does not fail with skip_exists and no build wheels."""
+    package_index = infra.extract_packages_from_index("https://example.com")
+    assert {p.name for p in tmppath.glob("*.whl")} == set()
+
+    infra.remove_local_wheels(
+        package_index,
+        skip_exists="grpcio",
+        packages=[
+            "grpcio==1.31.0",  # Exists in index
+        ],
+        wheels_dir=tmppath,
+    )