[Enhancement] Refactor metric and loop

configs/_base_/datasets/basicvsr_test_config.py

@@ -163,7 +163,7 @@
 ]
 
 # config for test
-test_cfg = dict(type='MultiTestLoop')
+test_cfg = dict(type='EditTestLoop')
 test_dataloader = [
     reds_dataloader,
     vimeo_90k_bd_dataloader,

configs/_base_/datasets/comp1k.py

@@ -29,6 +29,7 @@
 
 test_dataloader = val_dataloader
 
+# TODO: matting
 val_evaluator = [
     dict(type='SAD'),
     dict(type='MattingMSE'),

configs/_base_/datasets/liif_test_config.py

@@ -73,7 +73,7 @@
 ] for scale in scale_test_list]
 
 # test config
-test_cfg = dict(type='MultiTestLoop')
+test_cfg = dict(type='EditTestLoop')
 test_dataloader = [
     *set5_dataloaders,
     *set14_dataloaders,

configs/_base_/datasets/sisr_x2_test_config.py

@@ -27,10 +27,12 @@
         data_root=set5_data_root,
         data_prefix=dict(img='LRbicx2', gt='GTmod12'),
         pipeline=test_pipeline))
-set5_evaluator = [
-    dict(type='PSNR', crop_border=2, prefix='Set5'),
-    dict(type='SSIM', crop_border=2, prefix='Set5'),
-]
+set5_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=2, prefix='Set5'),
+        dict(type='SSIM', crop_border=2, prefix='Set5'),
+    ])
 
 set14_data_root = 'data/Set14'
 set14_dataloader = dict(
@@ -44,10 +46,12 @@
         data_root=set14_data_root,
         data_prefix=dict(img='LRbicx2', gt='GTmod12'),
         pipeline=test_pipeline))
-set14_evaluator = [
-    dict(type='PSNR', crop_border=2, prefix='Set14'),
-    dict(type='SSIM', crop_border=2, prefix='Set14'),
-]
+set14_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=2, prefix='Set14'),
+        dict(type='SSIM', crop_border=2, prefix='Set14'),
+    ])
 
 # test config for DIV2K
 div2k_data_root = 'data/DIV2K'
@@ -62,18 +66,18 @@
         ann_file='meta_info_DIV2K100sub_GT.txt',
         metainfo=dict(dataset_type='div2k', task_name='sisr'),
         data_root=div2k_data_root,
-        # TODO: what this"
         data_prefix=dict(
             img='DIV2K_train_LR_bicubic/X2_sub', gt='DIV2K_train_HR_sub'),
-        # filename_tmpl=dict(img='{}_x2', gt='{}'),
         pipeline=test_pipeline))
-div2k_evaluator = [
-    dict(type='PSNR', crop_border=2, prefix='DIV2K'),
-    dict(type='SSIM', crop_border=2, prefix='DIV2K'),
-]
+div2k_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=2, prefix='DIV2K'),
+        dict(type='SSIM', crop_border=2, prefix='DIV2K'),
+    ])
 
 # test config
-test_cfg = dict(type='MultiTestLoop')
+test_cfg = dict(type='EditTestLoop')
 test_dataloader = [
     set5_dataloader,
     set14_dataloader,

configs/_base_/datasets/sisr_x3_test_config.py

@@ -27,10 +27,12 @@
         data_root=set5_data_root,
         data_prefix=dict(img='LRbicx3', gt='GTmod12'),
         pipeline=test_pipeline))
-set5_evaluator = [
-    dict(type='PSNR', crop_border=3, prefix='Set5'),
-    dict(type='SSIM', crop_border=3, prefix='Set5'),
-]
+set5_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=3, prefix='Set5'),
+        dict(type='SSIM', crop_border=3, prefix='Set5'),
+    ])
 
 set14_data_root = 'data/Set14'
 set14_dataloader = dict(
@@ -44,10 +46,12 @@
         data_root=set14_data_root,
         data_prefix=dict(img='LRbicx3', gt='GTmod12'),
         pipeline=test_pipeline))
-set14_evaluator = [
-    dict(type='PSNR', crop_border=3, prefix='Set14'),
-    dict(type='SSIM', crop_border=3, prefix='Set14'),
-]
+set14_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=3, prefix='Set14'),
+        dict(type='SSIM', crop_border=3, prefix='Set14'),
+    ])
 
 # test config for DIV2K
 div2k_data_root = 'data/DIV2K'
@@ -64,13 +68,15 @@
         data_prefix=dict(
             img='DIV2K_train_LR_bicubic/X3_sub', gt='DIV2K_train_HR_sub'),
         pipeline=test_pipeline))
-div2k_evaluator = [
-    dict(type='PSNR', crop_border=3, prefix='DIV2K'),
-    dict(type='SSIM', crop_border=3, prefix='DIV2K'),
-]
+div2k_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=3, prefix='DIV2K'),
+        dict(type='SSIM', crop_border=3, prefix='DIV2K'),
+    ])
 
 # test config
-test_cfg = dict(type='MultiTestLoop')
+test_cfg = dict(type='EditTestLoop')
 test_dataloader = [
     set5_dataloader,
     set14_dataloader,

configs/_base_/datasets/sisr_x4_test_config.py

@@ -27,10 +27,12 @@
         data_root=set5_data_root,
         data_prefix=dict(img='LRbicx4', gt='GTmod12'),
         pipeline=test_pipeline))
-set5_evaluator = [
-    dict(type='PSNR', crop_border=4, prefix='Set5'),
-    dict(type='SSIM', crop_border=4, prefix='Set5'),
-]
+set5_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=4, prefix='Set5'),
+        dict(type='SSIM', crop_border=4, prefix='Set5'),
+    ])
 
 set14_data_root = 'data/Set14'
 set14_dataloader = dict(
@@ -44,10 +46,12 @@
         data_root=set14_data_root,
         data_prefix=dict(img='LRbicx4', gt='GTmod12'),
         pipeline=test_pipeline))
-set14_evaluator = [
-    dict(type='PSNR', crop_border=4, prefix='Set14'),
-    dict(type='SSIM', crop_border=4, prefix='Set14'),
-]
+set14_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=4, prefix='Set14'),
+        dict(type='SSIM', crop_border=4, prefix='Set14'),
+    ])
 
 # test config for DIV2K
 div2k_data_root = 'data/DIV2K'
@@ -63,15 +67,16 @@
         data_root=div2k_data_root,
         data_prefix=dict(
             img='DIV2K_train_LR_bicubic/X4_sub', gt='DIV2K_train_HR_sub'),
-        # filename_tmpl=dict(img='{}_x4', gt='{}'),
         pipeline=test_pipeline))
-div2k_evaluator = [
-    dict(type='PSNR', crop_border=4, prefix='DIV2K'),
-    dict(type='SSIM', crop_border=4, prefix='DIV2K'),
-]
+div2k_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', crop_border=4, prefix='DIV2K'),
+        dict(type='SSIM', crop_border=4, prefix='DIV2K'),
+    ])
 
 # test config
-test_cfg = dict(type='MultiTestLoop')
+test_cfg = dict(type='EditTestLoop')
 test_dataloader = [
     set5_dataloader,
     set14_dataloader,

configs/_base_/datasets/tdan_test_config.py

@@ -38,14 +38,22 @@
         num_input_frames=5,
         pipeline=SPMC_pipeline))
 
-SPMC_bd_evaluator = [
-    dict(type='PSNR', crop_border=8, convert_to='Y', prefix='SPMCS-BDx4-Y'),
-    dict(type='SSIM', crop_border=8, convert_to='Y', prefix='SPMCS-BDx4-Y'),
-]
-SPMC_bi_evaluator = [
-    dict(type='PSNR', crop_border=8, convert_to='Y', prefix='SPMCS-BIx4-Y'),
-    dict(type='SSIM', crop_border=8, convert_to='Y', prefix='SPMCS-BIx4-Y'),
-]
+SPMC_bd_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(
+            type='PSNR', crop_border=8, convert_to='Y', prefix='SPMCS-BDx4-Y'),
+        dict(
+            type='SSIM', crop_border=8, convert_to='Y', prefix='SPMCS-BDx4-Y'),
+    ])
+SPMC_bi_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(
+            type='PSNR', crop_border=8, convert_to='Y', prefix='SPMCS-BIx4-Y'),
+        dict(
+            type='SSIM', crop_border=8, convert_to='Y', prefix='SPMCS-BIx4-Y'),
+    ])
 
 # config for vid4
 vid4_data_root = 'data/Vid4'
@@ -87,17 +95,21 @@
         num_input_frames=5,
         pipeline=vid4_pipeline))
 
-vid4_bd_evaluator = [
-    dict(type='PSNR', convert_to='Y', prefix='VID4-BDx4-Y'),
-    dict(type='SSIM', convert_to='Y', prefix='VID4-BDx4-Y'),
-]
-vid4_bi_evaluator = [
-    dict(type='PSNR', convert_to='Y', prefix='VID4-BIx4-Y'),
-    dict(type='SSIM', convert_to='Y', prefix='VID4-BIx4-Y'),
-]
+vid4_bd_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', convert_to='Y', prefix='VID4-BDx4-Y'),
+        dict(type='SSIM', convert_to='Y', prefix='VID4-BDx4-Y'),
+    ])
+vid4_bi_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='PSNR', convert_to='Y', prefix='VID4-BIx4-Y'),
+        dict(type='SSIM', convert_to='Y', prefix='VID4-BIx4-Y'),
+    ])
 
 # config for test
-test_cfg = dict(type='MultiTestLoop')
+test_cfg = dict(type='EditTestLoop')
 test_dataloader = [
     SPMC_bd_dataloader,
     SPMC_bi_dataloader,

configs/_base_/default_runtime.py

@@ -2,7 +2,7 @@
 save_dir = './work_dirs'
 
 default_hooks = dict(
-    timer=dict(type='IterTimerHook'),
+    timer=dict(type='EditIterTimerHook'),
     logger=dict(type='LoggerHook', interval=100),
     param_scheduler=dict(type='ParamSchedulerHook'),
     checkpoint=dict(
@@ -24,7 +24,7 @@
 )
 
 log_level = 'INFO'
-log_processor = dict(type='LogProcessor', window_size=100, by_epoch=False)
+log_processor = dict(type='EditLogProcessor', window_size=100, by_epoch=False)
 
 load_from = None
 resume = False

configs/_base_/gen_default_runtime.py

@@ -11,7 +11,7 @@
 # configure for default hooks
 default_hooks = dict(
     # record time of every iteration.
-    timer=dict(type='GenIterTimerHook'),
+    timer=dict(type='EditIterTimerHook'),
     # print log every 100 iterations.
     logger=dict(type='LoggerHook', interval=100, log_metric_by_epoch=False),
     # save checkpoint per 10000 iterations
@@ -35,7 +35,7 @@
 
 # set log level
 log_level = 'INFO'
-log_processor = dict(type='GenLogProcessor', by_epoch=False)
+log_processor = dict(type='EditLogProcessor', by_epoch=False)
 
 # load from which checkpoint
 load_from = None
@@ -57,12 +57,12 @@
 train_cfg = dict(by_epoch=False, val_begin=1, val_interval=10000)
 
 # config for val
-val_cfg = dict(type='GenValLoop')
-val_evaluator = dict(type='GenEvaluator')
+val_cfg = dict(type='EditValLoop')
+val_evaluator = dict(type='EditEvaluator')
 
 # config for test
-test_cfg = dict(type='GenTestLoop')
-test_evaluator = dict(type='GenEvaluator')
+test_cfg = dict(type='EditTestLoop')
+test_evaluator = dict(type='EditEvaluator')
 
 # config for optim_wrapper_constructor
 optim_wrapper = dict(constructor='MultiOptimWrapperConstructor')

configs/_base_/models/base_edvr.py

@@ -98,8 +98,8 @@
 
 train_cfg = dict(
     type='IterBasedTrainLoop', max_iters=600_000, val_interval=5000)
-val_cfg = dict(type='ValLoop')
-test_cfg = dict(type='TestLoop')
+val_cfg = dict(type='EditValLoop')
+test_cfg = dict(type='EditTestLoop')
 
 # optimizer
 optim_wrapper = dict(

configs/_base_/models/base_glean.py

@@ -15,8 +15,8 @@
 
 train_cfg = dict(
     type='IterBasedTrainLoop', max_iters=300_000, val_interval=5000)
-val_cfg = dict(type='ValLoop')
-test_cfg = dict(type='TestLoop')
+val_cfg = dict(type='EditValLoop')
+test_cfg = dict(type='EditTestLoop')
 
 # optimizer
 optim_wrapper = dict(

configs/_base_/models/base_liif.py

@@ -86,15 +86,17 @@
         data_prefix=dict(img='LRbicx4', gt='GTmod12'),
         pipeline=val_pipeline))
 
-val_evaluator = [
-    dict(type='MAE'),
-    dict(type='PSNR', crop_border=scale_max),
-    dict(type='SSIM', crop_border=scale_max),
-]
+val_evaluator = dict(
+    type='EditEvaluator',
+    metrics=[
+        dict(type='MAE'),
+        dict(type='PSNR', crop_border=scale_max),
+        dict(type='SSIM', crop_border=scale_max),
+    ])
 
 train_cfg = dict(
     type='IterBasedTrainLoop', max_iters=1_000_000, val_interval=3000)
-val_cfg = dict(type='ValLoop')
+val_cfg = dict(type='EditValLoop')
 
 # optimizer
 optim_wrapper = dict(

configs/_base_/models/base_tof.py

@@ -72,8 +72,8 @@
 
 train_cfg = dict(
     type='IterBasedTrainLoop', max_iters=1_000_000, val_interval=epoch_length)
-val_cfg = dict(type='ValLoop')
-test_cfg = dict(type='TestLoop')
+val_cfg = dict(type='EditValLoop')
+test_cfg = dict(type='EditTestLoop')
 
 # optimizer
 optim_wrapper = dict(

configs/aot_gan/aot-gan_smpgan_4xb4_places-512x512.py

@@ -139,8 +139,8 @@
     max_iters=500002,
     val_interval=50000,
 )
-val_cfg = dict(type='ValLoop')
-test_cfg = dict(type='TestLoop')
+val_cfg = dict(type='EditValLoop')
+test_cfg = dict(type='EditTestLoop')
 
 # optimizer
 optim_wrapper = dict(