diff --git a/pyproject.toml b/pyproject.toml
index 0454cd9c..a0c458cc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,7 +52,6 @@ requires = [
     "numpy~=1.21.2; python_version<'3.11'",
     "numpy~=1.23.3; python_version>='3.11' and python_version<'3.12'",
     "numpy~=1.26.0; python_version>='3.12'",
-    "pybind11~=2.11.1",
     "setuptools~=68.1.0",
 ]
 build-backend = "setuptools.build_meta"
diff --git a/setup.py b/setup.py
index c16e5002..325b708a 100644
--- a/setup.py
+++ b/setup.py
@@ -17,19 +17,21 @@
 import fnmatch
 import platform
 import numpy as np
-from pybind11.setup_helpers import Pybind11Extension
-from setuptools import setup
+from setuptools import setup, Extension
 from setuptools.command.build_py import build_py as build_py_orig
 
 if platform.system() == "Windows":
   COMPILE_ARGS = [
       "/std:c++17",
       "/DEIGEN_MPL2_ONLY",
+      "/EHsc",
+      "/bigobj",
   ]
 else:
   COMPILE_ARGS = [
       "-std=c++17",
       "-DEIGEN_MPL2_ONLY",
+      "-fvisibility=hidden",
       # -ftrapping-math is necessary because NumPy looks at floating point
       # exception state to determine whether to emit, e.g., invalid value
       # warnings. Without this setting, on Mac ARM we see spurious "invalid
@@ -56,7 +58,7 @@ def find_package_modules(self, package, package_dir):
 
 setup(
     ext_modules=[
-        Pybind11Extension(
+        Extension(
             "ml_dtypes._ml_dtypes_ext",
             [
                 "ml_dtypes/_src/dtypes.cc",