enhance: [2.4]Make bulk_writer's requirments optional (#2086) (#2087)

See also: #2067

Signed-off-by: yangxuan <[email protected]>

examples/example_bulkwriter.py

@@ -26,13 +26,16 @@
     FieldSchema, CollectionSchema, DataType,
     Collection,
     utility,
+    BulkInsertState,
+)
+
+from pymilvus.bulk_writer import (
     LocalBulkWriter,
     RemoteBulkWriter,
     BulkFileType,
     bulk_import,
     get_import_progress,
     list_import_jobs,
-    BulkInsertState,
 )
 
 # minio

pymilvus/__init__.py

@@ -10,22 +10,6 @@
 # or implied. See the License for the specific language governing permissions and limitations under
 # the License.
 
-from .bulk_writer.bulk_import import (
-    bulk_import,
-    get_import_progress,
-    list_import_jobs,
-)
-
-# bulk writer
-from .bulk_writer.constants import (
-    BulkFileType,
-)
-from .bulk_writer.local_bulk_writer import (
-    LocalBulkWriter,
-)
-from .bulk_writer.remote_bulk_writer import (
-    RemoteBulkWriter,
-)
 from .client import __version__
 from .client.abstract import AnnSearchRequest, Hit, Hits, RRFRanker, SearchResult, WeightedRanker
 from .client.asynch import SearchFuture
@@ -143,12 +127,6 @@
     "ResourceGroupInfo",
     "Connections",
     "IndexType",
-    "BulkFileType",
-    "LocalBulkWriter",
-    "RemoteBulkWriter",
-    "bulk_import",
-    "get_import_progress",
-    "list_import_jobs",
     "AnnSearchRequest",
     "RRFRanker",
     "WeightedRanker",

pymilvus/bulk_writer/__init__.py

@@ -0,0 +1,28 @@
+from importlib.util import find_spec
+
+expected_pkgs = ["minio", "azure", "requests", "pyarrow"]
+
+missing = [pkg for pkg in expected_pkgs if find_spec(pkg) is None]
+
+if len(missing) > 0:
+    msg = f"Missing packages: {missing}. Please install bulk_writer by pip install pymilvus[bulk_writer] first"
+    raise ModuleNotFoundError(msg)
+
+
+from .bulk_import import (
+    bulk_import,
+    get_import_progress,
+    list_import_jobs,
+)
+from .constants import BulkFileType
+from .local_bulk_writer import LocalBulkWriter
+from .remote_bulk_writer import RemoteBulkWriter
+
+__all__ = [
+    "BulkFileType",
+    "LocalBulkWriter",
+    "RemoteBulkWriter",
+    "bulk_import",
+    "get_import_progress",
+    "list_import_jobs",
+]

pyproject.toml

@@ -24,10 +24,6 @@ dependencies=[
     "ujson>=2.0.0",
     "pandas>=1.2.4",
     "numpy<1.25.0;python_version<='3.8'",
-    "requests",
-    "minio>=7.0.0",
-    "pyarrow>=12.0.0",
-    "azure-storage-blob",
     "milvus_lite>=2.4.0,<2.5.0",
 ]
 
@@ -42,6 +38,13 @@ dynamic = ["version"]
 "repository" = 'https://github.com/milvus-io/pymilvus'
 
 [project.optional-dependencies]
+bulk_writer = [
+    "requests",
+    "minio>=7.0.0",
+    "pyarrow>=12.0.0",
+    "azure-storage-blob",
+]
+
 model = [
     "milvus-model>=0.1.0",
 ]
@@ -51,7 +54,7 @@ test = [
     "pytest-cov>=2.8.1",
     "pytest-timeout>=1.3.4",
     "grpcio-testing",
-    "ruff>=0.3.3",
+    "ruff>0.4.0",
     "black",
 ]
 
@@ -113,6 +116,7 @@ lint.ignore = [
     "PLR0915", # To many statements TODO
     "C901", # TODO
     "PYI041", # TODO
+    "E402",
 ]
 
 # Allow autofix for all enabled rules (when `--fix`) is provided.