Allow gluoncv.auto to work without mxnet (#1685)

* optional import estimators

* fix utils import

* fix error message

* fix tests

* fix lint

* fix

gluoncv/auto/data/dataset.py

@@ -10,7 +10,7 @@
 from PIL import Image
 import cv2
 from ..data import is_url, url_data
-from ...data.mscoco.utils import try_import_pycocotools
+from ...utils.filesystem import import_try_install
 from ...utils.bbox import bbox_xywh_to_xyxy, bbox_clip_xyxy
 try:
     import xml.etree.cElementTree as ET
@@ -31,6 +31,29 @@
 
 logger = logging.getLogger()
 
+
+
+def try_import_pycocotools():
+    """Tricks to optionally install and import pycocotools"""
+    # first we can try import pycocotools
+    try:
+        import pycocotools as _
+    except ImportError:
+        # we need to install pycootools, which is a bit tricky
+        # pycocotools sdist requires Cython, numpy(already met)
+        import_try_install('cython')
+        # pypi pycocotools is not compatible with windows
+        win_url = 'git+https://github.com/zhreshold/cocoapi.git#subdirectory=PythonAPI'
+        try:
+            if os.name == 'nt':
+                import_try_install('pycocotools', win_url)
+            else:
+                import_try_install('pycocotools')
+        except ImportError:
+            faq = 'cocoapi FAQ'
+            raise ImportError('Cannot import or install pycocotools, please refer to %s.' % faq)
+
+
 def _absolute_pathify(df, root=None, column='image'):
     """Convert relative paths to absolute"""
     if root is None:

gluoncv/auto/estimators/__init__.py

@@ -1,9 +1,33 @@
 """Estimator implementations"""
+from .utils import create_dummy_estimator
 # FIXME: for quick test purpose only
-from .image_classification import ImageClassificationEstimator
-from .ssd import SSDEstimator
-from .yolo import YOLOv3Estimator
-from .faster_rcnn import FasterRCNNEstimator
-# from .mask_rcnn import MaskRCNNEstimator
-from .center_net import CenterNetEstimator
-from .torch_image_classification import TorchImageClassificationEstimator
+try:
+    import mxnet
+    from .image_classification import ImageClassificationEstimator
+    from .ssd import SSDEstimator
+    from .yolo import YOLOv3Estimator
+    from .faster_rcnn import FasterRCNNEstimator
+    # from .mask_rcnn import MaskRCNNEstimator
+    from .center_net import CenterNetEstimator
+except ImportError:
+    # create dummy placeholder estimator classes
+    reason = 'gluoncv.auto.estimators.{} requires mxnet to be installed which is missing.'
+    ImageClassificationEstimator = create_dummy_estimator(
+        'ImageClassificationEstimator', reason)
+    SSDEstimator = create_dummy_estimator(
+        'SSDEstimator', reason)
+    YOLOv3Estimator = create_dummy_estimator(
+        'YOLOv3Estimator', reason)
+    FasterRCNNEstimator = create_dummy_estimator(
+        'FasterRCNNEstimator', reason)
+    CenterNetEstimator = create_dummy_estimator(
+        'CenterNetEstimator', reason)
+
+try:
+    import timm
+    import torch
+    from .torch_image_classification import TorchImageClassificationEstimator
+except ImportError:
+    reason = 'This estimator requires torch/timm to be installed which is missing.'
+    TorchImageClassificationEstimator = create_dummy_estimator(
+        'TorchImageClassificationEstimator', reason)

gluoncv/auto/estimators/utils.py

@@ -1,7 +1,21 @@
 """Utils for deep learning framework related functions"""
 import numpy as np
 
-__all__ = ['EarlyStopperOnPlateau', '_suggest_load_context']
+__all__ = ['EarlyStopperOnPlateau', '_suggest_load_context', 'create_dummy_estimator']
+
+def _dummy_constructor(self, *arg, **kwargs):
+    raise RuntimeError(self.reason.format(type(self).__name__))
+
+def create_dummy_estimator(name, reason):
+    assert isinstance(reason, str)
+    DummyEstimator = type(name, (object, ), {
+        # constructor
+        "__init__": _dummy_constructor,
+
+        # data members
+        "reason": reason,
+    })
+    return DummyEstimator
 
 
 class EarlyStopperOnPlateau:

gluoncv/utils/__init__.py

@@ -1,20 +1,44 @@
 """GluonCV Utility functions."""
-# pylint: disable=wildcard-import
+# pylint: disable=wildcard-import,exec-used,wrong-import-position
 from __future__ import absolute_import
 
+import types
+
+def import_dummy_module(code, name):
+    # create blank module
+    module = types.ModuleType(name)
+    # populate the module with code
+    exec(code, module.__dict__)
+    return module
+
+dummy_module = """
+def __getattr__(name):
+    raise AttributeError(f"gluoncv.utils.{__name__} module requires mxnet which is missing.")
+"""
+
+
 from . import bbox
-from . import viz
 from . import random
-from . import metrics
-from . import parallel
 from . import filesystem
+try:
+    import mxnet
+    from . import viz
+    from . import metrics
+    from . import parallel
+    from .lr_scheduler import LRSequential, LRScheduler
+    from .export_helper import export_block, export_tvm
+    from .sync_loader_helper import split_data, split_and_load
+except ImportError:
+    viz = import_dummy_module(dummy_module, 'viz')
+    metrics = import_dummy_module(dummy_module, 'metrics')
+    parallel = import_dummy_module(dummy_module, 'parallel')
+    LRSequential, LRScheduler = None, None
+    export_block, export_tvm = None, None
+    split_data, split_and_load = None, None
 
 from .download import download, check_sha1
 from .filesystem import makedirs, try_import_dali, try_import_cv2
 from .bbox import bbox_iou
 from .block import recursive_visit, set_lr_mult, freeze_bn
-from .lr_scheduler import LRSequential, LRScheduler
 from .plot_history import TrainingHistory
-from .export_helper import export_block, export_tvm
-from .sync_loader_helper import split_data, split_and_load
 from .version import *

gluoncv/utils/random.py

@@ -2,7 +2,10 @@
 from __future__ import absolute_import
 import random as pyrandom
 import numpy as np
-import mxnet as mx
+try:
+    import mxnet as mx
+except ImportError:
+    mx = None
 
 
 def seed(a=None):
@@ -23,4 +26,5 @@ def seed(a=None):
     """
     pyrandom.seed(a)
     np.random.seed(a)
-    mx.random.seed(a)
+    if mx is not None:
+        mx.random.seed(a)

gluoncv/utils/viz/bbox.py

@@ -2,7 +2,10 @@
 from __future__ import absolute_import, division
 
 import random
-import mxnet as mx
+try:
+    import mxnet as mx
+except ImportError:
+    mx = None
 from .image import plot_image
 
 def plot_bbox(img, bboxes, scores=None, labels=None, thresh=0.5,
@@ -62,11 +65,11 @@ def plot_bbox(img, bboxes, scores=None, labels=None, thresh=0.5,
     if len(bboxes) < 1:
         return ax
 
-    if isinstance(bboxes, mx.nd.NDArray):
+    if mx is not None and isinstance(bboxes, mx.nd.NDArray):
         bboxes = bboxes.asnumpy()
-    if isinstance(labels, mx.nd.NDArray):
+    if mx is not None and isinstance(labels, mx.nd.NDArray):
         labels = labels.asnumpy()
-    if isinstance(scores, mx.nd.NDArray):
+    if mx is not None and isinstance(scores, mx.nd.NDArray):
         scores = scores.asnumpy()
 
     if not absolute_coordinates:
@@ -158,13 +161,13 @@ def cv_plot_bbox(img, bboxes, scores=None, labels=None, thresh=0.5,
         raise ValueError('The length of scores and bboxes mismatch, {} vs {}'
                          .format(len(scores), len(bboxes)))
 
-    if isinstance(img, mx.nd.NDArray):
+    if mx is not None and isinstance(img, mx.nd.NDArray):
         img = img.asnumpy()
-    if isinstance(bboxes, mx.nd.NDArray):
+    if mx is not None and isinstance(bboxes, mx.nd.NDArray):
         bboxes = bboxes.asnumpy()
-    if isinstance(labels, mx.nd.NDArray):
+    if mx is not None and isinstance(labels, mx.nd.NDArray):
         labels = labels.asnumpy()
-    if isinstance(scores, mx.nd.NDArray):
+    if mx is not None and isinstance(scores, mx.nd.NDArray):
         scores = scores.asnumpy()
     if len(bboxes) < 1:
         return img

gluoncv/utils/viz/image.py

@@ -1,6 +1,9 @@
 """Visualize image."""
 import numpy as np
-import mxnet as mx
+try:
+    import mxnet as mx
+except ImportError:
+    mx = None
 
 def plot_image(img, ax=None, reverse_rgb=False):
     """Visualize image.
@@ -31,7 +34,7 @@ def plot_image(img, ax=None, reverse_rgb=False):
         # create new axes
         fig = plt.figure()
         ax = fig.add_subplot(1, 1, 1)
-    if isinstance(img, mx.nd.NDArray):
+    if mx is not None and isinstance(img, mx.nd.NDArray):
         img = img.asnumpy()
     img = img.copy()
     if reverse_rgb:
@@ -76,7 +79,7 @@ def cv_plot_image(img, scale=1, upperleft_txt=None, upperleft_txt_corner=(10, 10
     from ..filesystem import try_import_cv2
     cv2 = try_import_cv2()
 
-    if isinstance(img, mx.nd.NDArray):
+    if mx is not None and isinstance(img, mx.nd.NDArray):
         img = img.asnumpy()
 
     height, width, _ = img.shape

tests/auto/test_torch_auto_estimators.py

@@ -22,7 +22,6 @@
 from gluoncv.auto.data.dataset import ImageClassificationDataset
 import autogluon.core as ag
 from autogluon.core.scheduler.resource import get_cpu_count, get_gpu_count
-import mxnet as mx
 
 IMAGE_CLASS_DATASET, _, IMAGE_CLASS_TEST = ImageClassificationDataset.from_folders(
     'https://autogluon.s3.amazonaws.com/datasets/shopee-iet.zip')
@@ -84,7 +83,7 @@ def _save_load_test(est, filename):
 )
 def build_config(pretrained, global_pool_type, sync_bn, no_aug, mixup, cutmix, model_ema,
                 model_ema_force_cpu, save_images, pin_mem, use_multi_epochs_loader,
-                 amp, apex_amp, native_amp, prefetcher, interpolation, batch_size, hflip, vflip, 
+                 amp, apex_amp, native_amp, prefetcher, interpolation, batch_size, hflip, vflip,
                  train_interpolation, num_workers, tta):
     config = {
         'model': {