Skip to content
This repository has been archived by the owner on Apr 17, 2023. It is now read-only.

[MPA/OTX] Apply changes in develop to otx branch #105

Merged
merged 34 commits into from
Dec 28, 2022
Merged
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
bc04c98
fixed multilabel configs (#67)
kprokofi Oct 10, 2022
2f4fb6b
Tiling Module (#40)
eugene123tw Oct 17, 2022
938d74b
Feature/val batch and seed (#69)
sungmanc Oct 18, 2022
d960005
workaround bug (#70)
eugene123tw Oct 20, 2022
738239e
Kp/devide runners (#71)
kprokofi Oct 25, 2022
12861a9
move ema model to hook (#73)
kprokofi Oct 27, 2022
66d4e3a
Appley release/OTE_0.3.0 changes (#77)
goodsong81 Nov 9, 2022
726f545
Move drop_last in cls trainer.py (#79)
JihwanEom Nov 11, 2022
da52a8f
Removed unnecessary mim version constraint for networkx package (#80)
yunchu Nov 11, 2022
a173547
Revert "Removed unnecessary mim version constraint for networkx packa…
yunchu Nov 14, 2022
3e2551b
Don't apply activations on export in classification (#83)
sovrasov Nov 14, 2022
d46a15d
Delete unused code (#84)
sovrasov Nov 15, 2022
55af1ed
Replace current saliency map generation with Recipro-CAM for cls (#81)
negvet Nov 21, 2022
a99aea0
Class-wise saliency map generation for the detection task (#97)
negvet Dec 1, 2022
540ea47
[XAI] hot-fix of error in Detection XAI support (#98)
dongkwan-kim01 Dec 7, 2022
298d3a8
[XAI] hot-fix of error in Detection XAI support (#99)
dongkwan-kim01 Dec 7, 2022
0074b94
Merge remote-tracking branch 'public/develop' into songkich/merge-dev…
goodsong81 Dec 9, 2022
6de41a6
Replace only the first occurrence in the state_dict keys (#91)
arrufat Dec 12, 2022
1cbd0d2
Merge remote-tracking branch 'public/develop' into songkich/merge-dev…
goodsong81 Dec 12, 2022
5d4616e
[OTE / XAI] Handle two stage detector in the inferrer.py (#104)
dongkwan-kim01 Dec 13, 2022
9ea4088
Merge remote-tracking branch 'public/releases/OTE_0.4.0' into songkic…
goodsong81 Dec 13, 2022
f06e4bc
[OTE / XAI][Develop] Handle two stage detector in the inferrer.py (#107)
dongkwan-kim01 Dec 13, 2022
02b5503
Merge branch 'otx' into songkich/merge-dev-otx
goodsong81 Dec 15, 2022
4d1e2c0
[Hot-fix] Fix zero-division error in one cycle lr scheduler in multil…
supersoob Dec 16, 2022
4a8fc11
Merge OTE side XAI update to OTX (#109)
dongkwan-kim01 Dec 19, 2022
6d54deb
Merge remote-tracking branch 'public/releases/OTE_0.4.0' into songkic…
goodsong81 Dec 19, 2022
279ca33
Merge remote-tracking branch 'public/otx' into songkich/merge-dev-otx
goodsong81 Dec 19, 2022
15f73b6
Merge back releases/OTE_0.4.0 to develop (#116)
goodsong81 Dec 20, 2022
8c17768
Fix extra activations when exporting nonlinear hierarchical head (#118)
sovrasov Dec 21, 2022
7a504d1
Merge remote-tracking branch 'public/develop' into songkich/merge-dev…
goodsong81 Dec 21, 2022
187e529
Merge remote-tracking branch 'public/otx' into songkich/merge-dev-otx
goodsong81 Dec 21, 2022
760894f
Merge remote-tracking branch 'public/otx' into songkich/merge-dev-otx
goodsong81 Dec 22, 2022
3216118
Fix get_train_data_cfg -> get_data_cfg
goodsong81 Dec 23, 2022
e1830ea
Merge remote-tracking branch 'public/otx' into songkich/merge-dev-otx
goodsong81 Dec 26, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'public/otx' into songkich/merge-dev-otx
goodsong81 committed Dec 21, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 187e529da3342b68b1d05fa8bae56a823df8026b
2 changes: 1 addition & 1 deletion mpa/cls/inferrer.py
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@
from mmcls.models import build_classifier
from mmcv.runner import load_checkpoint, wrap_fp16_model
from mpa.cls.stage import ClsStage
from mpa.modules.hooks.recording_forward_hooks import FeatureVectorHook, ReciproCAMHook
from mpa.modules.hooks.recording_forward_hooks import ReciproCAMHook, FeatureVectorHook
from mpa.modules.utils.task_adapt import prob_extractor
from mpa.registry import STAGES
from mpa.utils.logger import get_logger
20 changes: 20 additions & 0 deletions mpa/cls/stage.py
Original file line number Diff line number Diff line change
@@ -286,6 +286,26 @@ def configure_task(cfg, training, model_meta=None, **kwargs):
cfg.model.head.num_old_classes = len(old_classes)
return model_tasks, dst_classes

def _put_model_on_gpu(self, model, cfg):
if torch.cuda.is_available():
model = model.cuda()
if self.distributed:
# put model on gpus
find_unused_parameters = cfg.get('find_unused_parameters', False)
# Sets the `find_unused_parameters` parameter in
# torch.nn.parallel.DistributedDataParallel
model = MMDistributedDataParallel(
model,
device_ids=[torch.cuda.current_device()],
broadcast_buffers=False,
find_unused_parameters=find_unused_parameters)
else:
model = MMDataParallel(
model.cuda(), device_ids=[0])
else:
model = MMDataCPU(model)

return model

def refine_tasks(train_cfg, meta, adapt_type):
new_tasks = train_cfg['tasks']
28 changes: 14 additions & 14 deletions mpa/cls/trainer.py
Original file line number Diff line number Diff line change
@@ -103,13 +103,13 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
self._modify_cfg_for_distributed(model, cfg)

# prepare data loaders
dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset]
datasets = datasets if isinstance(datasets, (list, tuple)) else [datasets]
train_data_cfg = Stage.get_data_cfg(cfg, "train")
ote_dataset = train_data_cfg.get('ote_dataset', None)
otx_dataset = train_data_cfg.get("otx_dataset", None)
drop_last = False
dataset_len = len(ote_dataset) if ote_dataset else 0
dataset_len = len(otx_dataset) if otx_dataset else 0
# if task == h-label & dataset size is bigger than batch size
if train_data_cfg.get('hierarchical_info', None) and dataset_len > cfg.data.get('samples_per_gpu', 2):
if train_data_cfg.get("hierarchical_info", None) and dataset_len > cfg.data.get("samples_per_gpu", 2):
drop_last = True
# updated to adapt list of dataset for the 'train'
data_loaders = []
@@ -183,7 +183,7 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
cfg.optimizer_config.pop('type')
optimizer_config = opt_hook(
**cfg.optimizer_config, **fp16_cfg, distributed=self.distributed)
elif self.distributed and 'type' not in cfg.optimizer_config:
elif self.distributed and "type" not in cfg.optimizer_config:
optimizer_config = DistOptimizerHook(**cfg.optimizer_config)
else:
optimizer_config = cfg.optimizer_config
@@ -201,7 +201,7 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
for hook in cfg.get('custom_hooks', ()):
runner.register_hook_from_cfg(hook)

validate = True if cfg.data.get('val', None) else False
validate = True if cfg.data.get("val", None) else False
# register eval hooks
if validate:
val_dataset = build_dataset(cfg.data.val, dict(test_mode=True))
@@ -222,26 +222,26 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
runner.resume(cfg.resume_from)
elif cfg.get('load_from', False):
if self.distributed:
runner.load_checkpoint(cfg.load_from, map_location=f'cuda:{cfg.gpu_ids[0]}')
runner.load_checkpoint(cfg.load_from, map_location=f"cuda:{cfg.gpu_ids[0]}")
else:
runner.load_checkpoint(cfg.load_from)
runner.run(data_loaders, cfg.workflow)

logger.info(f'called train_worker() distributed={self.distributed}, validate=True')
logger.info(f"called train_worker() distributed={self.distributed}, validate=True")

# Save outputs
output_ckpt_path = osp.join(cfg.work_dir, 'best_model.pth'
if osp.exists(osp.join(cfg.work_dir, 'best_model.pth'))
else 'latest.pth')
output_ckpt_path = osp.join(cfg.work_dir, "best_model.pth"
if osp.exists(osp.join(cfg.work_dir, "best_model.pth"))
else "latest.pth")
return dict(final_ckpt=output_ckpt_path)

def _modify_cfg_for_distributed(self, model, cfg):
nn.SyncBatchNorm.convert_sync_batchnorm(model)

if cfg.dist_params.get('linear_scale_lr', False):
if cfg.dist_params.get("linear_scale_lr", False):
new_lr = len(cfg.gpu_ids) * cfg.optimizer.lr
logger.info(f'enabled linear scaling rule to the learning rate. \
changed LR from {cfg.optimizer.lr} to {new_lr}')
logger.info(f"enabled linear scaling rule to the learning rate. \
changed LR from {cfg.optimizer.lr} to {new_lr}")
cfg.optimizer.lr = new_lr

@staticmethod
46 changes: 11 additions & 35 deletions mpa/det/inferrer.py
Original file line number Diff line number Diff line change
@@ -10,15 +10,12 @@
from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor, ImageTilingDataset
from mmdet.models import build_detector
from mmdet.models.detectors import TwoStageDetector
from mmdet.parallel import MMDataCPU
from mmdet.utils.deployment import get_feature_vector
from mmdet.apis import single_gpu_test
from mmdet.utils.misc import prepare_mmdet_model_for_execution

from mpa.registry import STAGES
from mpa.utils.logger import get_logger
from mpa.det.incremental import IncrDetectionStage
from mpa.modules.hooks.recording_forward_hooks import ActivationMapHook, DetSaliencyMapHook

from mpa.modules.hooks.recording_forward_hooks import ActivationMapHook, DetSaliencyMapHook, FeatureVectorHook

logger = get_logger()

@@ -37,9 +34,9 @@ def run(self, model_cfg, model_ckpt, data_cfg, **kwargs):
"""
self._init_logger()
mode = kwargs.get('mode', 'train')
eval = kwargs.pop('eval', False)
dump_features = kwargs.pop('dump_features', False)
dump_saliency_map = kwargs.pop('dump_saliency_map', False)
eval = kwargs.pop("eval", False)
dump_features = kwargs.pop("dump_features", False)
dump_saliency_map = kwargs.pop("dump_saliency_map", False)
if mode not in self.mode:
return {}

@@ -159,34 +156,13 @@ def infer(self, cfg, eval=False, dump_features=False, dump_saliency_map=False):
saliency_hook = DetSaliencyMapHook(eval_model.module)

eval_predictions = []
feature_vectors = []

def dump_features_hook(mod, inp, out):
with torch.no_grad():
feature_vector = get_feature_vector(out)
assert feature_vector.size(0) == 1
feature_vectors.append(feature_vector.view(-1).detach().cpu().numpy())

def dummy_dump_features_hook(mod, inp, out):
feature_vectors.append(None)

feature_vector_hook = dump_features_hook if dump_features else dummy_dump_features_hook

# Use a single gpu for testing. Set in both mm_val_dataloader and eval_model
if is_module_wrapper(model):
model = model.module

# Class-wise Saliency map for Single-Stage Detector, otherwise use class-ignore saliency map.
if not dump_saliency_map:
saliency_hook = nullcontext()
elif isinstance(model, TwoStageDetector):
saliency_hook = ActivationMapHook(eval_model.module.backbone)
else:
saliency_hook = DetSaliencyMapHook(eval_model.module)

with eval_model.module.register_forward_hook(feature_vector_hook):
with FeatureVectorHook(eval_model.module) if dump_features else nullcontext() as feature_vector_hook:
with saliency_hook:
eval_predictions = single_gpu_test(eval_model, data_loader)
for data in data_loader:
with torch.no_grad():
result = eval_model(return_loss=False, rescale=True, **data)
eval_predictions.extend(result)
feature_vectors = feature_vector_hook.records if dump_features else [None] * len(self.dataset)
saliency_maps = saliency_hook.records if dump_saliency_map else [None] * len(self.dataset)

for key in [
3 changes: 2 additions & 1 deletion mpa/modules/hooks/recording_forward_hooks.py
Original file line number Diff line number Diff line change
@@ -257,7 +257,8 @@ def __init__(self, module: torch.nn.Module, fpn_idx: int = 0) -> None:
self._num_classes = module.head.num_classes

def func(self, feature_map: Union[torch.Tensor, Sequence[torch.Tensor]], fpn_idx: int = 0) -> torch.Tensor:
"""Generate the saliency maps using Recipro-CAM and then normalizing to (0, 255).
"""
Generate the class-wise saliency maps using Recipro-CAM and then normalizing to (0, 255).

Args:
feature_map (Union[torch.Tensor, List[torch.Tensor]]): feature maps from backbone or list of feature maps
151 changes: 45 additions & 106 deletions mpa/seg/stage.py
Original file line number Diff line number Diff line change
@@ -26,7 +26,6 @@ def configure(self, model_cfg, model_ckpt, data_cfg, training=True, **kwargs):
self.configure_ckpt(cfg, model_ckpt, kwargs.get('pretrained', None))
self.configure_data(cfg, data_cfg, training)
self.configure_task(cfg, training, **kwargs)
self.configure_hyperparams(cfg, training, **kwargs)

return cfg

@@ -80,100 +79,8 @@ def configure_data(self, cfg, data_cfg, training):
if training:
if cfg.data.get('val', False):
self.validate = True

# Task
if 'task_adapt' in cfg:
self.configure_task(cfg, training, **kwargs)

return cfg

def configure_model(self, cfg, training, **kwargs):
pass

def configure_task(self, cfg, training, **kwargs):
"""Adjust settings for task adaptation
"""
self.logger = get_root_logger()
self.logger.info(f'task config!!!!: training={training}')
task_adapt_type = cfg['task_adapt'].get('type', None)
task_adapt_op = cfg['task_adapt'].get('op', 'REPLACE')

# Task classes
org_model_classes, model_classes, data_classes = \
self.configure_task_classes(cfg, task_adapt_op)

# Incremental learning
if task_adapt_type == 'mpa':
self.configure_task_cls_incr(cfg, task_adapt_type, org_model_classes, model_classes)

def configure_cross_entropy_loss_with_ignore(self, model_classes):
cfg_loss_decode = ConfigDict(
type='CrossEntropyLossWithIgnore',
reduction='mean',
sampler=dict(type='MaxPoolingPixelSampler', ratio=0.25, p=1.7),
loss_weight=1.0
)
return cfg_loss_decode

def configure_am_softmax_loss_with_ignore(self, model_classes):
cfg_loss_decode = ConfigDict(
type='AMSoftmaxLossWithIgnore',
scale_cfg=ConfigDict(
type='PolyScalarScheduler',
start_scale=30,
end_scale=5,
by_epoch=True,
num_iters=250,
power=1.2
),
margin_type='cos',
margin=0.5,
gamma=0.0,
t=1.0,
target_loss='ce',
pr_product=False,
conf_penalty_weight=ConfigDict(
type='PolyScalarScheduler',
start_scale=0.2,
end_scale=0.15,
by_epoch=True,
num_iters=200,
power=1.2
),
border_reweighting=False,
sampler=ConfigDict(type='MaxPoolingPixelSampler', ratio=0.25, p=1.7),
loss_weight=1.0
)
return cfg_loss_decode

def configure_task_cls_incr(self, cfg, task_adapt_type, org_model_classes, model_classes):
new_classes = np.setdiff1d(model_classes, org_model_classes).tolist()
has_new_class = True if len(new_classes) > 0 else False
if has_new_class is False:
ValueError('Incremental learning should have at least one new class!')

# Incremental Learning
if cfg.get('ignore', False):
if 'decode_head' in cfg.model:
decode_head = cfg.model.decode_head
if isinstance(decode_head, dict):
if decode_head.type == 'FCNHead':
decode_head.type = 'CustomFCNHead'
decode_head.loss_decode = self.configure_cross_entropy_loss_with_ignore(model_classes)
elif decode_head.type == 'OCRHead':
decode_head.type = 'CustomOCRHead'
decode_head.loss_decode = self.configure_am_softmax_loss_with_ignore(model_classes)
elif isinstance(decode_head, list):
for head in decode_head:
if head.type == 'FCNHead':
head.type = 'CustomFCNHead'
head.loss_decode = [self.configure_cross_entropy_loss_with_ignore(model_classes)]
elif head.type == 'OCRHead':
head.type = 'CustomOCRHead'
head.loss_decode = [self.configure_am_softmax_loss_with_ignore(model_classes)]

# Dataset
src_data_cfg = Stage.get_data_cfg(cfg, "train")
src_data_cfg = Stage.get_train_data_cfg(cfg)
for mode in ['train', 'val', 'test']:
if src_data_cfg.type == 'MPASegDataset':
if cfg.data[mode]['type'] != 'MPASegDataset':
@@ -182,18 +89,6 @@ def configure_task_cls_incr(self, cfg, task_adapt_type, org_model_classes, model
cfg.data[mode]['type'] = 'MPASegDataset'
cfg.data[mode]['org_type'] = org_type

def configure_hyperparams(self, cfg, training, **kwargs):
if 'hyperparams' in cfg:
hyperparams = kwargs.get('hyperparams', None)
if hyperparams is not None:
bs = hyperparams.get('bs', None)
if bs is not None:
cfg.data.samples_per_gpu = bs

lr = hyperparams.get('lr', None)
if lr is not None:
cfg.optimizer.lr = lr

def configure_task(self, cfg, training, **kwargs):
"""Adjust settings for task adaptation
"""
@@ -244,3 +139,47 @@ def configure_classes(self, cfg, task_adapt_op):
head.num_classes = len(model_classes)

return org_model_classes, model_classes, data_classes

def configure_cls_incr(self, cfg, org_model_classes, model_classes):

new_classes = np.setdiff1d(model_classes, org_model_classes).tolist()

# FIXME : can be naive supervised learning (from-scratch ver.)
# Check if new classes are added
has_new_class = True if len(new_classes) > 0 else False
if has_new_class is False:
ValueError('Incremental learning should have at least one new class!')

# Change to incremental loss (ignore mode)
if cfg.get('ignore', False):
if 'decode_head' in cfg.model:
decode_head = cfg.model.decode_head
if isinstance(decode_head, dict):
if decode_head.type == 'FCNHead':
decode_head.type = 'CustomFCNHead'
decode_head.loss_decode = self.configure_cross_entropy_loss_with_ignore(model_classes)
elif isinstance(decode_head, list):
for head in decode_head:
if head.type == 'FCNHead':
head.type = 'CustomFCNHead'
head.loss_decode = [self.configure_cross_entropy_loss_with_ignore(model_classes)]

# Update TaskAdaptHook (use incremental sampler)
task_adapt_hook = ConfigDict(
type='TaskAdaptHook',
src_classes=org_model_classes,
dst_classes=model_classes,
model_type=cfg.model.type,
sampler_flag=has_new_class,
efficient_mode=cfg['task_adapt'].get('efficient_mode', False)
)
update_or_add_custom_hook(cfg, task_adapt_hook)

def configure_cross_entropy_loss_with_ignore(self, model_classes):
cfg_loss_decode = ConfigDict(
type='CrossEntropyLossWithIgnore',
reduction='mean',
sampler=dict(type='MaxPoolingPixelSampler', ratio=0.25, p=1.7),
loss_weight=1.0
)
return cfg_loss_decode
You are viewing a condensed version of this merge commit. You can view the full changes here.