[Auto] Fix estimators (#1619)

* fix logdir after refit * final fit trial * fix * fix * fix tests
dmlc · Feb 20, 2021 · f91e161 · f91e161
1 parent 4280cb1
commit f91e161
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 12 deletions.
diff --git a/gluoncv/auto/estimators/base_estimator.py b/gluoncv/auto/estimators/base_estimator.py
@@ -155,6 +155,7 @@ def fit(self, train_data, val_data=None, train_size=0.9, random_state=None,
             return self._fit(train_data, val_data, time_limit=time_limit) if not resume \
                 else self._resume_fit(train_data, val_data, time_limit=time_limit)
 
+        os.makedirs(self._logdir, exist_ok=True)
         if val_data is None:
             assert 0 <= train_size <= 1.0
             if random_state:

diff --git a/gluoncv/auto/tasks/image_classification.py b/gluoncv/auto/tasks/image_classification.py
@@ -121,8 +121,12 @@ def _train_image_classification(args, reporter):
                 if best_checkpoint:
                     estimator = estimator_cls.load(best_checkpoint)
             if estimator is None:
-                result.update({'traceback': 'timeout'})
-        else:
+                if wall_clock_tick < tic:
+                    result.update({'traceback': 'timeout'})
+                else:
+                    # unknown error yet, try reproduce it
+                    final_fit = False
+        if not final_fit:
             # create independent log_dir for each trial
             trial_log_dir = os.path.join(log_dir, '.trial_{}'.format(task_id))
             args['log_dir'] = trial_log_dir

diff --git a/gluoncv/auto/tasks/object_detection.py b/gluoncv/auto/tasks/object_detection.py
@@ -127,8 +127,12 @@ def _train_object_detection(args, reporter):
                 if best_checkpoint:
                     estimator = estimator_cls.load(best_checkpoint)
             if estimator is None:
-                result.update({'traceback': 'timeout'})
-        else:
+                if wall_clock_tick < tic:
+                    result.update({'traceback': 'timeout'})
+                else:
+                    # unknown error
+                    final_fit = False
+        if not final_fit:
             # create independent log_dir for each trial
             trial_log_dir = os.path.join(log_dir, '.trial_{}'.format(task_id))
             args['log_dir'] = trial_log_dir

diff --git a/tests/auto/test_auto_tasks.py b/tests/auto/test_auto_tasks.py
@@ -10,7 +10,7 @@
 
 def test_image_classification():
     from gluoncv.auto.tasks import ImageClassification
-    task = ImageClassification({'num_trials': 1, 'epochs': 1})
+    task = ImageClassification({'num_trials': 1, 'epochs': 1, 'batch_size': 8})
     classifier = task.fit(IMAGE_CLASS_DATASET)
     assert task.fit_summary().get('valid_acc', 0) > 0
     test_result = classifier.predict(IMAGE_CLASS_TEST)
@@ -19,39 +19,39 @@ def test_image_classification_custom_net():
     from gluoncv.auto.tasks import ImageClassification
     from gluoncv.model_zoo import get_model
     net = get_model('resnet18_v1')
-    task = ImageClassification({'num_trials': 1, 'epochs': 1, 'custom_net': net})
+    task = ImageClassification({'num_trials': 1, 'epochs': 1, 'custom_net': net, 'batch_size': 8})
     classifier = task.fit(IMAGE_CLASS_DATASET)
     assert task.fit_summary().get('valid_acc', 0) > 0
     test_result = classifier.predict(IMAGE_CLASS_TEST)
 
 def test_object_detection_estimator():
     from gluoncv.auto.tasks import ObjectDetection
-    task = ObjectDetection({'num_trials': 1, 'epochs': 1})
+    task = ObjectDetection({'num_trials': 1, 'epochs': 1, 'batch_size': 4})
     detector = task.fit(OBJECT_DETECTION_TRAIN)
     assert task.fit_summary().get('valid_map', 0) > 0
     test_result = detector.predict(OBJECT_DETECTION_TEST)
 
 def test_object_detection_estimator_transfer():
     from gluoncv.auto.tasks import ObjectDetection
-    task = ObjectDetection({'num_trials': 1, 'epochs': 1, 'transfer': ag.Categorical('yolo3_darknet53_coco', 'ssd_512_resnet50_v1_voc'), 'estimator': 'ssd'})
+    task = ObjectDetection({'num_trials': 1, 'epochs': 1, 'transfer': ag.Categorical('yolo3_darknet53_coco', 'ssd_512_resnet50_v1_voc'), 'estimator': 'ssd', 'batch_size': 4})
     detector = task.fit(OBJECT_DETECTION_TRAIN)
     assert task.fit_summary().get('valid_map', 0) > 0
     test_result = detector.predict(OBJECT_DETECTION_TEST)
 
 def test_time_out_image_classification():
-    time_limit = 30
+    time_limit = 15
     from gluoncv.auto.tasks import ImageClassification
-    task = ImageClassification({'num_trials': 1, 'epochs': 50})
+    task = ImageClassification({'num_trials': 1, 'epochs': 10, 'batch_size': 8})
 
     tic = time.time()
     classifier = task.fit(IMAGE_CLASS_DATASET, time_limit=time_limit)
     # check time_limit with a little bit overhead
     assert (time.time() - tic) < time_limit + 180
 
 def test_time_out_detection():
-    time_limit = 30
+    time_limit = 15
     from gluoncv.auto.tasks import ObjectDetection
-    task = ObjectDetection({'num_trials': 1, 'epochs': 50, 'time_limits': time_limit})
+    task = ObjectDetection({'num_trials': 1, 'epochs': 5, 'time_limits': time_limit, 'batch_size': 4})
     tic = time.time()
     detector = task.fit(OBJECT_DETECTION_TRAIN)
     # check time_limit with a little bit overhead