diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2a82df7..8499ddc7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## Unreleased
 ### Changed
  - Use the base interpreter path when running inside a virtual environment to avoid recompilation when switching between virtual environments. [#429](https://github.com/PyO3/setuptools-rust/pull/429)
+ - Delay import of dependencies until use to avoid import errors during a partially complete install when multiple packages are installing at once. [#437](https://github.com/PyO3/setuptools-rust/pull/437)
 
 ## 1.9.0 (2024-02-24)
 ### Changed
diff --git a/pyproject.toml b/pyproject.toml
index cae05366..148d85bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,7 +26,6 @@ classifiers = [
 dependencies = [
     "setuptools>=62.4",
     "semantic_version>=2.8.2,<3",
-    'tomli>=1.2.1; python_version<"3.11"'
 ]
 
 [project.entry-points."distutils.commands"]
diff --git a/setuptools_rust/extension.py b/setuptools_rust/extension.py
index 2d6e5aa7..e305960e 100644
--- a/setuptools_rust/extension.py
+++ b/setuptools_rust/extension.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import os
 import re
@@ -14,11 +16,13 @@
     NewType,
     Optional,
     Sequence,
+    TYPE_CHECKING,
     Union,
     cast,
 )
 
-from semantic_version import SimpleSpec
+if TYPE_CHECKING:
+    from semantic_version import SimpleSpec
 
 from ._utils import format_called_process_error
 
@@ -185,6 +189,8 @@ def get_rust_version(self) -> Optional[SimpleSpec]:  # type: ignore[no-any-unimp
         if self.rust_version is None:
             return None
         try:
+            from semantic_version import SimpleSpec
+
             return SimpleSpec(self.rust_version)
         except ValueError:
             raise SetupError(
diff --git a/setuptools_rust/rustc_info.py b/setuptools_rust/rustc_info.py
index 070be0c1..58dfd42a 100644
--- a/setuptools_rust/rustc_info.py
+++ b/setuptools_rust/rustc_info.py
@@ -1,15 +1,20 @@
+from __future__ import annotations
+
 import subprocess
 from setuptools.errors import PlatformError
 from functools import lru_cache
-from typing import Dict, List, NewType, Optional
+from typing import Dict, List, NewType, Optional, TYPE_CHECKING
 
-from semantic_version import Version
+if TYPE_CHECKING:
+    from semantic_version import Version
 
 
 def get_rust_version() -> Optional[Version]:  # type: ignore[no-any-unimported]
     try:
         # first line of rustc -Vv is something like
         # rustc 1.61.0 (fe5b13d68 2022-05-18)
+        from semantic_version import Version
+
         return Version(_rust_version().split(" ")[1])
     except (subprocess.CalledProcessError, OSError):
         return None
diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py
index d4a02049..0ea6f588 100644
--- a/setuptools_rust/setuptools_ext.py
+++ b/setuptools_rust/setuptools_ext.py
@@ -26,7 +26,10 @@
 if sys.version_info[:2] >= (3, 11):
     from tomllib import load as toml_load
 else:
-    from tomli import load as toml_load
+    try:
+        from tomli import load as toml_load
+    except ImportError:
+        from setuptools.extern.tomli import load as toml_load
 
 
 logger = logging.getLogger(__name__)