From b4d546577850b156939067794f155526e262ee61 Mon Sep 17 00:00:00 2001 From: Csaba Botos Date: Tue, 9 Apr 2019 16:02:04 +0200 Subject: [PATCH 01/52] Support Binary Mask with transparent SementationMask interface (#473) * support RLE and binary mask * do not convert to numpy * be consistent with Detectron * delete wrong comment * [WIP] add tests for segmentation_mask * update tests * minor change * Refactored segmentation_mask.py * Add unit test for segmentation_mask.py * Add RLE support for BinaryMaskList * PEP8 black formatting * Minor patch * Use internal that handles 0 channels * Fix polygon slicing --- maskrcnn_benchmark/data/datasets/coco.py | 2 +- .../modeling/roi_heads/mask_head/loss.py | 10 +- .../structures/segmentation_mask.py | 478 +++++++++++++++--- tests/test_segmentation_mask.py | 74 +++ 4 files changed, 478 insertions(+), 86 deletions(-) create mode 100644 tests/test_segmentation_mask.py diff --git a/maskrcnn_benchmark/data/datasets/coco.py b/maskrcnn_benchmark/data/datasets/coco.py index f0c8c25b4..d0e42b437 100644 --- a/maskrcnn_benchmark/data/datasets/coco.py +++ b/maskrcnn_benchmark/data/datasets/coco.py @@ -80,7 +80,7 @@ def __getitem__(self, idx): target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] - masks = SegmentationMask(masks, img.size) + masks = SegmentationMask(masks, img.size, mode='poly') target.add_field("masks", masks) if anno and "keypoints" in anno[0]: diff --git a/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py b/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py index 36dcaa325..d4c5e3621 100644 --- a/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py +++ b/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py @@ -27,17 +27,15 @@ def project_masks_on_boxes(segmentation_masks, proposals, discretization_size): assert segmentation_masks.size == proposals.size, "{}, {}".format( segmentation_masks, proposals ) - # TODO put the proposals on the CPU, as the representation for the - # masks is not efficient GPU-wise (possibly several small tensors for - # representing a single instance mask) + + # FIXME: CPU computation bottleneck, this should be parallelized proposals = proposals.bbox.to(torch.device("cpu")) for segmentation_mask, proposal in zip(segmentation_masks, proposals): # crop the masks, resize them to the desired resolution and - # then convert them to the tensor representation, - # instead of the list representation that was used + # then convert them to the tensor representation. cropped_mask = segmentation_mask.crop(proposal) scaled_mask = cropped_mask.resize((M, M)) - mask = scaled_mask.convert(mode="mask") + mask = scaled_mask.get_mask_tensor() masks.append(mask) if len(masks) == 0: return torch.empty(0, dtype=torch.float32, device=device) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index ba1290b91..bf91d5568 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -1,5 +1,8 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +import cv2 + import torch +import numpy as np +from maskrcnn_benchmark.layers.misc import interpolate import pycocotools.mask as mask_utils @@ -8,63 +11,207 @@ FLIP_TOP_BOTTOM = 1 -class Mask(object): +""" ABSTRACT +Segmentations come in either: +1) Binary masks +2) Polygons + +Binary masks can be represented in a contiguous array +and operations can be carried out more efficiently, +therefore BinaryMaskList handles them together. + +Polygons are handled separately for each instance, +by PolygonInstance and instances are handled by +PolygonList. + +SegmentationList is supposed to represent both, +therefore it wraps the functions of BinaryMaskList +and PolygonList to make it transparent. +""" + + +class BinaryMaskList(object): """ - This class is unfinished and not meant for use yet - It is supposed to contain the mask for an object as - a 2d tensor + This class handles binary masks for all objects in the image """ - def __init__(self, masks, size, mode): + def __init__(self, masks, size): + """ + Arguments: + masks: Either torch.tensor of [num_instances, H, W] + or list of torch.tensors of [H, W] with num_instances elems, + or RLE (Run Length Encoding) - interpreted as list of dicts, + or BinaryMaskList. + size: absolute image size, width first + + After initialization, a hard copy will be made, to leave the + initializing source data intact. + """ + + if isinstance(masks, torch.Tensor): + # The raw data representation is passed as argument + masks = masks.clone() + elif isinstance(masks, (list, tuple)): + if isinstance(masks[0], torch.Tensor): + masks = torch.stack(masks, dim=2).clone() + elif isinstance(masks[0], dict) and "count" in masks[0]: + # RLE interpretation + + masks = mask_utils + else: + RuntimeError( + "Type of `masks[0]` could not be interpreted: %s" % type(masks) + ) + elif isinstance(masks, BinaryMaskList): + # just hard copy the BinaryMaskList instance's underlying data + masks = masks.masks.clone() + else: + RuntimeError( + "Type of `masks` argument could not be interpreted:%s" % tpye(masks) + ) + + if len(masks.shape) == 2: + # if only a single instance mask is passed + masks = masks[None] + + assert len(masks.shape) == 3 + assert masks.shape[1] == size[1], "%s != %s" % (masks.shape[1], size[1]) + assert masks.shape[2] == size[0], "%s != %s" % (masks.shape[2], size[0]) + self.masks = masks - self.size = size - self.mode = mode + self.size = tuple(size) def transpose(self, method): - if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): - raise NotImplementedError( - "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" + dim = 1 if method == FLIP_TOP_BOTTOM else 2 + flipped_masks = self.masks.flip(dim) + return BinaryMaskList(flipped_masks, self.size) + + def crop(self, box): + assert isinstance(box, (list, tuple, torch.Tensor)), str(type(box)) + # box is assumed to be xyxy + current_width, current_height = self.size + xmin, ymin, xmax, ymax = [round(float(b)) for b in box] + + assert xmin <= xmax and ymin <= ymax, str(box) + xmin = min(max(xmin, 0), current_width - 1) + ymin = min(max(ymin, 0), current_height - 1) + + xmax = min(max(xmax, 0), current_width) + ymax = min(max(ymax, 0), current_height) + + xmax = max(xmax, xmin + 1) + ymax = max(ymax, ymin + 1) + + width, height = xmax - xmin, ymax - ymin + cropped_masks = self.masks[:, ymin:ymax, xmin:xmax] + cropped_size = width, height + return BinaryMaskList(cropped_masks, cropped_size) + + def resize(self, size): + try: + iter(size) + except TypeError: + assert isinstance(size, (int, float)) + size = size, size + width, height = map(int, size) + + assert width > 0 + assert height > 0 + + # Height comes first here! + resized_masks = torch.nn.functional.interpolate( + input=self.masks[None].float(), + size=(height, width), + mode="bilinear", + align_corners=False, + )[0].type_as(self.masks) + resized_size = width, height + return BinaryMaskList(resized_masks, resized_size) + + def convert_to_polygon(self): + contours = self._findContours() + return PolygonList(contours, self.size) + + def to(self, *args, **kwargs): + return self + + def _findContours(self): + contours = [] + masks = self.masks.detach().numpy() + for mask in masks: + mask = cv2.UMat(mask) + contour, hierarchy = cv2.findContours( + mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_L1 ) - width, height = self.size - if method == FLIP_LEFT_RIGHT: - dim = width - idx = 2 - elif method == FLIP_TOP_BOTTOM: - dim = height - idx = 1 + reshaped_contour = [] + for entity in contour: + assert len(entity.shape) == 3 + assert entity.shape[1] == 1, "Hierarchical contours are not allowed" + reshaped_contour.append(entity.reshape(-1).tolist()) + contours.append(reshaped_contour) + return contours - flip_idx = list(range(dim)[::-1]) - flipped_masks = self.masks.index_select(dim, flip_idx) - return Mask(flipped_masks, self.size, self.mode) + def __len__(self): + return len(self.masks) - def crop(self, box): - w, h = box[2] - box[0], box[3] - box[1] + def __getitem__(self, index): + # Probably it can cause some overhead + # but preserves consistency + masks = self.masks[index].clone() + return BinaryMaskList(masks, self.size) - cropped_masks = self.masks[:, box[1] : box[3], box[0] : box[2]] - return Mask(cropped_masks, size=(w, h), mode=self.mode) + def __iter__(self): + return iter(self.masks) - def resize(self, size, *args, **kwargs): - pass + def __repr__(self): + s = self.__class__.__name__ + "(" + s += "num_instances={}, ".format(len(self.masks)) + s += "image_width={}, ".format(self.size[0]) + s += "image_height={})".format(self.size[1]) + return s -class Polygons(object): +class PolygonInstance(object): """ This class holds a set of polygons that represents a single instance of an object mask. The object can be represented as a set of polygons """ - def __init__(self, polygons, size, mode): - # assert isinstance(polygons, list), '{}'.format(polygons) - if isinstance(polygons, list): - polygons = [torch.as_tensor(p, dtype=torch.float32) for p in polygons] - elif isinstance(polygons, Polygons): - polygons = polygons.polygons + def __init__(self, polygons, size): + """ + Arguments: + a list of lists of numbers. + The first level refers to all the polygons that compose the + object, and the second level to the polygon coordinates. + """ + if isinstance(polygons, (list, tuple)): + valid_polygons = [] + for p in polygons: + p = torch.as_tensor(p, dtype=torch.float32) + if len(p) >= 6: # 3 * 2 coordinates + valid_polygons.append(p) + polygons = valid_polygons + + elif isinstance(polygons, PolygonInstance): + polygons = polygons.polygons.copy() + + else: + RuntimeError( + "Type of argument `polygons` is not allowed:%s" % (type(polygons)) + ) + + """ This crashes the training way too many times... + for p in polygons: + assert p[::2].min() >= 0 + assert p[::2].max() < size[0] + assert p[1::2].min() >= 0 + assert p[1::2].max() , size[1] + """ self.polygons = polygons - self.size = size - self.mode = mode + self.size = tuple(size) def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): @@ -87,30 +234,49 @@ def transpose(self, method): p[idx::2] = dim - poly[idx::2] - TO_REMOVE flipped_polygons.append(p) - return Polygons(flipped_polygons, size=self.size, mode=self.mode) + return PolygonInstance(flipped_polygons, size=self.size) def crop(self, box): - w, h = box[2] - box[0], box[3] - box[1] + assert isinstance(box, (list, tuple, torch.Tensor)), str(type(box)) + + # box is assumed to be xyxy + current_width, current_height = self.size + xmin, ymin, xmax, ymax = map(float, box) + + assert xmin <= xmax and ymin <= ymax, str(box) + xmin = min(max(xmin, 0), current_width - 1) + ymin = min(max(ymin, 0), current_height - 1) + + xmax = min(max(xmax, 0), current_width) + ymax = min(max(ymax, 0), current_height) - # TODO chck if necessary - w = max(w, 1) - h = max(h, 1) + xmax = max(xmax, xmin + 1) + ymax = max(ymax, ymin + 1) + + w, h = xmax - xmin, ymax - ymin cropped_polygons = [] for poly in self.polygons: p = poly.clone() - p[0::2] = p[0::2] - box[0] # .clamp(min=0, max=w) - p[1::2] = p[1::2] - box[1] # .clamp(min=0, max=h) + p[0::2] = p[0::2] - xmin # .clamp(min=0, max=w) + p[1::2] = p[1::2] - ymin # .clamp(min=0, max=h) cropped_polygons.append(p) - return Polygons(cropped_polygons, size=(w, h), mode=self.mode) + return PolygonInstance(cropped_polygons, size=(w, h)) + + def resize(self, size): + try: + iter(size) + except TypeError: + assert isinstance(size, (int, float)) + size = size, size - def resize(self, size, *args, **kwargs): ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(size, self.size)) + if ratios[0] == ratios[1]: ratio = ratios[0] scaled_polys = [p * ratio for p in self.polygons] - return Polygons(scaled_polys, size, mode=self.mode) + return PolygonInstance(scaled_polys, size) ratio_w, ratio_h = ratios scaled_polygons = [] @@ -120,47 +286,82 @@ def resize(self, size, *args, **kwargs): p[1::2] *= ratio_h scaled_polygons.append(p) - return Polygons(scaled_polygons, size=size, mode=self.mode) + return PolygonInstance(scaled_polygons, size=size) - def convert(self, mode): + def convert_to_binarymask(self): width, height = self.size - if mode == "mask": - rles = mask_utils.frPyObjects( - [p.numpy() for p in self.polygons], height, width - ) - rle = mask_utils.merge(rles) - mask = mask_utils.decode(rle) - mask = torch.from_numpy(mask) - # TODO add squeeze? - return mask + # formatting for COCO PythonAPI + polygons = [p.numpy() for p in self.polygons] + rles = mask_utils.frPyObjects(polygons, height, width) + rle = mask_utils.merge(rles) + mask = mask_utils.decode(rle) + mask = torch.from_numpy(mask) + return mask + + def __len__(self): + return len(self.polygons) def __repr__(self): s = self.__class__.__name__ + "(" - s += "num_polygons={}, ".format(len(self.polygons)) + s += "num_groups={}, ".format(len(self.polygons)) s += "image_width={}, ".format(self.size[0]) s += "image_height={}, ".format(self.size[1]) - s += "mode={})".format(self.mode) return s -class SegmentationMask(object): +class PolygonList(object): """ - This class stores the segmentations for all objects in the image + This class handles PolygonInstances for all objects in the image """ - def __init__(self, polygons, size, mode=None): + def __init__(self, polygons, size): """ Arguments: - polygons: a list of list of lists of numbers. The first + polygons: + a list of list of lists of numbers. The first level of the list correspond to individual instances, the second level to all the polygons that compose the object, and the third level to the polygon coordinates. + + OR + + a list of PolygonInstances. + + OR + + a PolygonList + + size: absolute image size + """ - assert isinstance(polygons, list) + if isinstance(polygons, (list, tuple)): + if len(polygons) == 0: + polygons = [[[]]] + if isinstance(polygons[0], (list, tuple)): + assert isinstance(polygons[0][0], (list, tuple)), str( + type(polygons[0][0]) + ) + else: + assert isinstance(polygons[0], PolygonInstance), str(type(polygons[0])) + + elif isinstance(polygons, PolygonList): + size = polygons.size + polygons = polygons.polygons - self.polygons = [Polygons(p, size, mode) for p in polygons] - self.size = size - self.mode = mode + else: + RuntimeError( + "Type of argument `polygons` is not allowed:%s" % (type(polygons)) + ) + + assert isinstance(size, (list, tuple)), str(type(size)) + + self.polygons = [] + for p in polygons: + p = PolygonInstance(p, size) + if len(p) > 0: + self.polygons.append(p) + + self.size = tuple(size) def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): @@ -168,30 +369,49 @@ def transpose(self, method): "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" ) - flipped = [] + flipped_polygons = [] for polygon in self.polygons: - flipped.append(polygon.transpose(method)) - return SegmentationMask(flipped, size=self.size, mode=self.mode) + flipped_polygons.append(polygon.transpose(method)) + + return PolygonList(flipped_polygons, size=self.size) def crop(self, box): w, h = box[2] - box[0], box[3] - box[1] - cropped = [] + cropped_polygons = [] for polygon in self.polygons: - cropped.append(polygon.crop(box)) - return SegmentationMask(cropped, size=(w, h), mode=self.mode) + cropped_polygons.append(polygon.crop(box)) - def resize(self, size, *args, **kwargs): - scaled = [] + cropped_size = w, h + return PolygonList(cropped_polygons, cropped_size) + + def resize(self, size): + resized_polygons = [] for polygon in self.polygons: - scaled.append(polygon.resize(size, *args, **kwargs)) - return SegmentationMask(scaled, size=size, mode=self.mode) + resized_polygons.append(polygon.resize(size)) + + resized_size = size + return PolygonList(resized_polygons, resized_size) def to(self, *args, **kwargs): return self + def convert_to_binarymask(self): + if len(self) > 0: + masks = torch.stack([p.convert_to_binarymask() for p in self.polygons]) + else: + size = self.size + masks = torch.empty([0, size[1], size[0]], dtype=torch.uint8) + + return BinaryMaskList(masks, size=self.size) + + def __len__(self): + return len(self.polygons) + def __getitem__(self, item): - if isinstance(item, (int, slice)): + if isinstance(item, int): selected_polygons = [self.polygons[item]] + elif isinstance(item, slice): + selected_polygons = self.polygons[item] else: # advanced indexing on a single dimension selected_polygons = [] @@ -201,7 +421,7 @@ def __getitem__(self, item): item = item.tolist() for i in item: selected_polygons.append(self.polygons[i]) - return SegmentationMask(selected_polygons, size=self.size, mode=self.mode) + return PolygonList(selected_polygons, size=self.size) def __iter__(self): return iter(self.polygons) @@ -212,3 +432,103 @@ def __repr__(self): s += "image_width={}, ".format(self.size[0]) s += "image_height={})".format(self.size[1]) return s + + +class SegmentationMask(object): + + """ + This class stores the segmentations for all objects in the image. + It wraps BinaryMaskList and PolygonList conveniently. + """ + + def __init__(self, instances, size, mode="poly"): + """ + Arguments: + instances: two types + (1) polygon + (2) binary mask + size: (width, height) + mode: 'poly', 'mask'. if mode is 'mask', convert mask of any format to binary mask + """ + + assert isinstance(size, (list, tuple)) + assert len(size) == 2 + if isinstance(size[0], torch.Tensor): + assert isinstance(size[1], torch.Tensor) + size = size[0].item(), size[1].item() + + assert isinstance(size[0], (int, float)) + assert isinstance(size[1], (int, float)) + + if mode == "poly": + self.instances = PolygonList(instances, size) + elif mode == "mask": + self.instances = BinaryMaskList(instances, size) + else: + raise NotImplementedError("Unknown mode: %s" % str(mode)) + + self.mode = mode + self.size = tuple(size) + + def transpose(self, method): + flipped_instances = self.instances.transpose(method) + return SegmentationMask(flipped_instances, self.size, self.mode) + + def crop(self, box): + cropped_instances = self.instances.crop(box) + cropped_size = cropped_instances.size + return SegmentationMask(cropped_instances, cropped_size, self.mode) + + def resize(self, size, *args, **kwargs): + resized_instances = self.instances.resize(size) + resized_size = size + return SegmentationMask(resized_instances, resized_size, self.mode) + + def to(self, *args, **kwargs): + return self + + def convert(self, mode): + if mode == self.mode: + return self + + if mode == "poly": + converted_instances = self.instances.convert_to_polygon() + elif mode == "mask": + converted_instances = self.instances.convert_to_binarymask() + else: + raise NotImplementedError("Unknown mode: %s" % str(mode)) + + return SegmentationMask(converted_instances, self.size, mode) + + def get_mask_tensor(self): + instances = self.instances + if self.mode == "poly": + instances = instances.convert_to_binarymask() + # If there is only 1 instance + return instances.masks.squeeze(0) + + def __len__(self): + return len(self.instances) + + def __getitem__(self, item): + selected_instances = self.instances.__getitem__(item) + return SegmentationMask(selected_instances, self.size, self.mode) + + def __iter__(self): + self.iter_idx = 0 + return self + + def __next__(self): + if self.iter_idx < self.__len__(): + next_segmentation = self.__getitem__(self.iter_idx) + self.iter_idx += 1 + return next_segmentation + raise StopIteration + + def __repr__(self): + s = self.__class__.__name__ + "(" + s += "num_instances={}, ".format(len(self.instances)) + s += "image_width={}, ".format(self.size[0]) + s += "image_height={}, ".format(self.size[1]) + s += "mode={})".format(self.mode) + return s diff --git a/tests/test_segmentation_mask.py b/tests/test_segmentation_mask.py new file mode 100644 index 000000000..d01ed9452 --- /dev/null +++ b/tests/test_segmentation_mask.py @@ -0,0 +1,74 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +import unittest +import torch +from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask + + +class TestSegmentationMask(unittest.TestCase): + def __init__(self, method_name='runTest'): + super(TestSegmentationMask, self).__init__(method_name) + poly = [[[423.0, 306.5, 406.5, 277.0, 400.0, 271.5, 389.5, 277.0, + 387.5, 292.0, 384.5, 295.0, 374.5, 220.0, 378.5, 210.0, + 391.0, 200.5, 404.0, 199.5, 414.0, 203.5, 425.5, 221.0, + 438.5, 297.0, 423.0, 306.5], + [100, 100, 200, 100, 200, 200, 100, 200], + ]] + width = 640 + height = 480 + size = width, height + + self.P = SegmentationMask(poly, size, 'poly') + self.M = SegmentationMask(poly, size, 'poly').convert('mask') + + + def L1(self, A, B): + diff = A.get_mask_tensor() - B.get_mask_tensor() + diff = torch.sum(torch.abs(diff.float())).item() + return diff + + + def test_convert(self): + M_hat = self.M.convert('poly').convert('mask') + P_hat = self.P.convert('mask').convert('poly') + + diff_mask = self.L1(self.M, M_hat) + diff_poly = self.L1(self.P, P_hat) + self.assertTrue(diff_mask == diff_poly) + self.assertTrue(diff_mask <= 8169.) + self.assertTrue(diff_poly <= 8169.) + + + def test_crop(self): + box = [400, 250, 500, 300] # xyxy + diff = self.L1(self.M.crop(box), self.P.crop(box)) + self.assertTrue(diff <= 1.) + + + def test_resize(self): + new_size = 50, 25 + M_hat = self.M.resize(new_size) + P_hat = self.P.resize(new_size) + diff = self.L1(M_hat, P_hat) + + self.assertTrue(self.M.size == self.P.size) + self.assertTrue(M_hat.size == P_hat.size) + self.assertTrue(self.M.size != M_hat.size) + self.assertTrue(diff <= 255.) + + + def test_transpose(self): + FLIP_LEFT_RIGHT = 0 + FLIP_TOP_BOTTOM = 1 + diff_hor = self.L1(self.M.transpose(FLIP_LEFT_RIGHT), + self.P.transpose(FLIP_LEFT_RIGHT)) + + diff_ver = self.L1(self.M.transpose(FLIP_TOP_BOTTOM), + self.P.transpose(FLIP_TOP_BOTTOM)) + + self.assertTrue(diff_hor <= 53250.) + self.assertTrue(diff_ver <= 42494.) + + +if __name__ == "__main__": + + unittest.main() From 59d445dfb42d58d18e10e75423d7795d11022024 Mon Sep 17 00:00:00 2001 From: Congcong Li Date: Wed, 10 Apr 2019 19:25:47 +0800 Subject: [PATCH 02/52] fix #258 (#648) --- maskrcnn_benchmark/data/datasets/evaluation/voc/voc_eval.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/maskrcnn_benchmark/data/datasets/evaluation/voc/voc_eval.py b/maskrcnn_benchmark/data/datasets/evaluation/voc/voc_eval.py index f8b0c1084..693710902 100644 --- a/maskrcnn_benchmark/data/datasets/evaluation/voc/voc_eval.py +++ b/maskrcnn_benchmark/data/datasets/evaluation/voc/voc_eval.py @@ -16,8 +16,6 @@ def do_voc_evaluation(dataset, predictions, output_folder, logger): gt_boxlists = [] for image_id, prediction in enumerate(predictions): img_info = dataset.get_img_info(image_id) - if len(prediction) == 0: - continue image_width = img_info["width"] image_height = img_info["height"] prediction = prediction.resize((image_width, image_height)) From fdb9c687e8b21f7eb0d5f46e4dfaf80e3f698b65 Mon Sep 17 00:00:00 2001 From: qianyizhang Date: Thu, 11 Apr 2019 16:34:57 +0800 Subject: [PATCH 03/52] fix python2 compatibility (#662) * fix py2 * fix py2 --- maskrcnn_benchmark/structures/segmentation_mask.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index bf91d5568..391829318 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -1,5 +1,5 @@ import cv2 - +import copy import torch import numpy as np from maskrcnn_benchmark.layers.misc import interpolate @@ -195,7 +195,7 @@ def __init__(self, polygons, size): polygons = valid_polygons elif isinstance(polygons, PolygonInstance): - polygons = polygons.polygons.copy() + polygons = copy.copy(polygons.polygons) else: RuntimeError( @@ -523,7 +523,9 @@ def __next__(self): next_segmentation = self.__getitem__(self.iter_idx) self.iter_idx += 1 return next_segmentation - raise StopIteration + raise StopIteration() + + next = __next__ # Python 2 compatibility def __repr__(self): s = self.__class__.__name__ + "(" From 42fa3a1d1509d44a534d50b9dd139f1cd540072e Mon Sep 17 00:00:00 2001 From: Tian Zhi Date: Sat, 13 Apr 2019 23:52:18 +0930 Subject: [PATCH 04/52] Add FCOS into projects using maskrcnn-benchmark. (#670) --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 780722ed8..c31451c31 100644 --- a/README.md +++ b/README.md @@ -243,8 +243,9 @@ note = {Accessed: [Insert date here]} - [RetinaMask: Learning to predict masks improves state-of-the-art single-shot detection for free](https://arxiv.org/abs/1901.03353). Cheng-Yang Fu, Mykhailo Shvets, and Alexander C. Berg. Tech report, arXiv,1901.03353. - - +- [FCOS: Fully Convolutional One-Stage Object Detection](https://arxiv.org/abs/1904.01355). + Zhi Tian, Chunhua Shen, Hao Chen and Tong He. + Tech report, arXiv,1904.01355. [[code](https://github.com/tianzhi0549/FCOS)] ## License From c5c4d52eeb924c800027a51862c984558d57fa4c Mon Sep 17 00:00:00 2001 From: Csaba Botos Date: Sat, 13 Apr 2019 17:04:40 +0200 Subject: [PATCH 05/52] Add RLE support (#657) https://gist.github.com/botcs/a59f3f59e22e5df93e3e5e4f86718af3 --- maskrcnn_benchmark/structures/segmentation_mask.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 391829318..0ab11ad24 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -54,10 +54,13 @@ def __init__(self, masks, size): elif isinstance(masks, (list, tuple)): if isinstance(masks[0], torch.Tensor): masks = torch.stack(masks, dim=2).clone() - elif isinstance(masks[0], dict) and "count" in masks[0]: + elif isinstance(masks[0], dict) and "counts" in masks[0]: # RLE interpretation - - masks = mask_utils + assert all( + [(size[0], size[1]) == tuple(inst["size"]) for inst in masks] + ) + masks = mask_utils.decode(masks) + masks = torch.tensor(masks).permute(2, 0, 1) else: RuntimeError( "Type of `masks[0]` could not be interpreted: %s" % type(masks) From 1714b7c20d919f8c536a577e15ba3a03ce733cde Mon Sep 17 00:00:00 2001 From: Menglin Jia <36873096+KMnP@users.noreply.github.com> Date: Tue, 16 Apr 2019 05:03:26 -0400 Subject: [PATCH 06/52] revised for RLE assert (#679) --- maskrcnn_benchmark/structures/segmentation_mask.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 0ab11ad24..060d512b3 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -57,10 +57,10 @@ def __init__(self, masks, size): elif isinstance(masks[0], dict) and "counts" in masks[0]: # RLE interpretation assert all( - [(size[0], size[1]) == tuple(inst["size"]) for inst in masks] - ) - masks = mask_utils.decode(masks) - masks = torch.tensor(masks).permute(2, 0, 1) + [(size[1], size[0]) == tuple(inst["size"]) for inst in masks] + ) # in RLE, height come first in "size" + masks = mask_utils.decode(masks) # [h, w, n] + masks = torch.tensor(masks).permute(2, 0, 1) # [n, h, w] else: RuntimeError( "Type of `masks[0]` could not be interpreted: %s" % type(masks) @@ -70,7 +70,7 @@ def __init__(self, masks, size): masks = masks.masks.clone() else: RuntimeError( - "Type of `masks` argument could not be interpreted:%s" % tpye(masks) + "Type of `masks` argument could not be interpreted:%s" % type(masks) ) if len(masks.shape) == 2: @@ -527,7 +527,7 @@ def __next__(self): self.iter_idx += 1 return next_segmentation raise StopIteration() - + next = __next__ # Python 2 compatibility def __repr__(self): From 1d6e9add946ac6c3a7e72ed0ec7020d4667dac94 Mon Sep 17 00:00:00 2001 From: zimenglan Date: Fri, 19 Apr 2019 23:25:03 +0800 Subject: [PATCH 07/52] add dcn from mmdetection (#693) * make pixel indexes 0-based for bounding box in pascal voc dataset * replacing all instances of torch.distributed.deprecated with torch.distributed * replacing all instances of torch.distributed.deprecated with torch.distributed * add GroupNorm * add GroupNorm -- sort out yaml files * use torch.nn.GroupNorm instead, replace 'use_gn' with 'conv_block' and use 'BaseStem'&'Bottleneck' to simply codes * modification on 'group_norm' and 'conv_with_kaiming_uniform' function * modification on yaml files in configs/gn_baselines/ and reduce the amount of indentation and code duplication * use 'kaiming_uniform' to initialize resnet, disable gn after fc layer, and add dilation into ResNetHead * agnostic-regression for bbox * please set 'STRIDE_IN_1X1' to be 'False' when backbone use GN * add README.md for GN * add dcn from mmdetection --- .gitignore | 1 + configs/dcn/README.md | 31 + .../e2e_faster_rcnn_dconv_R_50_FPN_1x.yaml | 44 + .../e2e_faster_rcnn_mdconv_R_50_FPN_1x.yaml | 44 + .../dcn/e2e_mask_rcnn_dconv_R_50_FPN_1x.yaml | 54 ++ .../dcn/e2e_mask_rcnn_mdconv_R_50_FPN_1x.yaml | 53 ++ maskrcnn_benchmark/config/defaults.py | 4 + .../csrc/cuda/deform_conv_cuda.cu | 691 ++++++++++++++ .../csrc/cuda/deform_conv_kernel_cuda.cu | 874 ++++++++++++++++++ .../csrc/cuda/deform_pool_cuda.cu | 87 ++ .../csrc/cuda/deform_pool_kernel_cuda.cu | 365 ++++++++ maskrcnn_benchmark/csrc/cuda/vision.h | 53 ++ maskrcnn_benchmark/csrc/deform_conv.h | 191 ++++ maskrcnn_benchmark/csrc/deform_pool.h | 70 ++ maskrcnn_benchmark/csrc/vision.cpp | 12 +- maskrcnn_benchmark/layers/__init__.py | 34 +- maskrcnn_benchmark/layers/dcn/__init__.py | 3 + .../layers/dcn/deform_conv_func.py | 259 ++++++ .../layers/dcn/deform_conv_module.py | 177 ++++ .../layers/dcn/deform_pool_func.py | 93 ++ .../layers/dcn/deform_pool_module.py | 150 +++ maskrcnn_benchmark/layers/misc.py | 84 ++ .../modeling/backbone/resnet.py | 73 +- maskrcnn_benchmark/utils/c2_model_loading.py | 31 + 24 files changed, 3453 insertions(+), 25 deletions(-) create mode 100644 configs/dcn/README.md create mode 100644 configs/dcn/e2e_faster_rcnn_dconv_R_50_FPN_1x.yaml create mode 100644 configs/dcn/e2e_faster_rcnn_mdconv_R_50_FPN_1x.yaml create mode 100644 configs/dcn/e2e_mask_rcnn_dconv_R_50_FPN_1x.yaml create mode 100644 configs/dcn/e2e_mask_rcnn_mdconv_R_50_FPN_1x.yaml create mode 100644 maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.cu create mode 100644 maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu create mode 100644 maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.cu create mode 100644 maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu create mode 100644 maskrcnn_benchmark/csrc/deform_conv.h create mode 100644 maskrcnn_benchmark/csrc/deform_pool.h create mode 100644 maskrcnn_benchmark/layers/dcn/__init__.py create mode 100644 maskrcnn_benchmark/layers/dcn/deform_conv_func.py create mode 100644 maskrcnn_benchmark/layers/dcn/deform_conv_module.py create mode 100644 maskrcnn_benchmark/layers/dcn/deform_pool_func.py create mode 100644 maskrcnn_benchmark/layers/dcn/deform_pool_module.py diff --git a/.gitignore b/.gitignore index 223e87a19..4124b2bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ dist/ # project dirs /datasets /models +/output diff --git a/configs/dcn/README.md b/configs/dcn/README.md new file mode 100644 index 000000000..2c3a0784e --- /dev/null +++ b/configs/dcn/README.md @@ -0,0 +1,31 @@ +### Reference +1 [Deformable ConvNets v2: More Deformable, Better Results](https://arxiv.org/pdf/1811.11168.pdf) +2 third-party: [mmdetection](https://github.com/open-mmlab/mmdetection/tree/master/configs/dcn) + +### Performance +| case | bbox AP | mask AP | +|----------------------------:|--------:|:-------:| +| R-50-FPN-dcn (implement) | 39.8 | - | +| R-50-FPN-dcn (mmdetection) | 40.0 | - | +| R-50-FPN-mdcn (implement) | 40.0 | - | +| R-50-FPN-mdcn (mmdetection) | 40.3 | - | +| R-50-FPN-dcn (implement) | 40.8 | 36.8 | +| R-50-FPN-dcn (mmdetection) | 41.1 | 37.2 | +| R-50-FPN-dcn (implement) | 40.7 | 36.7 | +| R-50-FPN-dcn (mmdetection) | 41.4 | 37.4 | + + +### Note +see [dcn-v2](https://github.com/open-mmlab/mmdetection/blob/master/MODEL_ZOO.md#deformable-convolution-v2) in `mmdetection` for more details. + + +### Usage +add these three lines +``` +MODEL: + RESNETS: + # corresponding to C2,C3,C4,C5 + STAGE_WITH_DCN: (False, True, True, True) + WITH_MODULATED_DCN: True + DEFORMABLE_GROUPS: 1 +``` \ No newline at end of file diff --git a/configs/dcn/e2e_faster_rcnn_dconv_R_50_FPN_1x.yaml b/configs/dcn/e2e_faster_rcnn_dconv_R_50_FPN_1x.yaml new file mode 100644 index 000000000..25c037592 --- /dev/null +++ b/configs/dcn/e2e_faster_rcnn_dconv_R_50_FPN_1x.yaml @@ -0,0 +1,44 @@ +INPUT: + MIN_SIZE_TRAIN: (800,) + MAX_SIZE_TRAIN: 1333 + MIN_SIZE_TEST: 800 + MAX_SIZE_TEST: 1333 +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" + BACKBONE: + CONV_BODY: "R-50-FPN" + RESNETS: + BACKBONE_OUT_CHANNELS: 256 + STAGE_WITH_DCN: (False, True, True, True) + WITH_MODULATED_DCN: False + DEFORMABLE_GROUPS: 1 + RPN: + USE_FPN: True + ANCHOR_STRIDE: (4, 8, 16, 32, 64) + PRE_NMS_TOP_N_TRAIN: 2000 + PRE_NMS_TOP_N_TEST: 1000 + POST_NMS_TOP_N_TEST: 1000 + FPN_POST_NMS_TOP_N_TEST: 1000 + ROI_HEADS: + USE_FPN: True + ROI_BOX_HEAD: + POOLER_RESOLUTION: 7 + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + POOLER_SAMPLING_RATIO: 2 + FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" + PREDICTOR: "FPNPredictor" +DATASETS: + TRAIN: ("coco_2014_train", "coco_2014_valminusminival") + TEST: ("coco_2014_minival",) +DATALOADER: + SIZE_DIVISIBILITY: 32 +SOLVER: + # Assume 8 gpus + BASE_LR: 0.02 + WEIGHT_DECAY: 0.0001 + STEPS: (60000, 80000) + MAX_ITER: 90000 + IMS_PER_BATCH: 16 +TEST: + IMS_PER_BATCH: 8 diff --git a/configs/dcn/e2e_faster_rcnn_mdconv_R_50_FPN_1x.yaml b/configs/dcn/e2e_faster_rcnn_mdconv_R_50_FPN_1x.yaml new file mode 100644 index 000000000..6bc04212c --- /dev/null +++ b/configs/dcn/e2e_faster_rcnn_mdconv_R_50_FPN_1x.yaml @@ -0,0 +1,44 @@ +INPUT: + MIN_SIZE_TRAIN: (800,) + MAX_SIZE_TRAIN: 1333 + MIN_SIZE_TEST: 800 + MAX_SIZE_TEST: 1333 +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" + BACKBONE: + CONV_BODY: "R-50-FPN" + RESNETS: + BACKBONE_OUT_CHANNELS: 256 + STAGE_WITH_DCN: (False, True, True, True) + WITH_MODULATED_DCN: True + DEFORMABLE_GROUPS: 1 + RPN: + USE_FPN: True + ANCHOR_STRIDE: (4, 8, 16, 32, 64) + PRE_NMS_TOP_N_TRAIN: 2000 + PRE_NMS_TOP_N_TEST: 1000 + POST_NMS_TOP_N_TEST: 1000 + FPN_POST_NMS_TOP_N_TEST: 1000 + ROI_HEADS: + USE_FPN: True + ROI_BOX_HEAD: + POOLER_RESOLUTION: 7 + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + POOLER_SAMPLING_RATIO: 2 + FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" + PREDICTOR: "FPNPredictor" +DATASETS: + TRAIN: ("coco_2014_train", "coco_2014_valminusminival") + TEST: ("coco_2014_minival",) +DATALOADER: + SIZE_DIVISIBILITY: 32 +SOLVER: + # Assume 8 gpus + BASE_LR: 0.02 + WEIGHT_DECAY: 0.0001 + STEPS: (60000, 80000) + MAX_ITER: 90000 + IMS_PER_BATCH: 16 +TEST: + IMS_PER_BATCH: 8 diff --git a/configs/dcn/e2e_mask_rcnn_dconv_R_50_FPN_1x.yaml b/configs/dcn/e2e_mask_rcnn_dconv_R_50_FPN_1x.yaml new file mode 100644 index 000000000..5cffe5c57 --- /dev/null +++ b/configs/dcn/e2e_mask_rcnn_dconv_R_50_FPN_1x.yaml @@ -0,0 +1,54 @@ +INPUT: + MIN_SIZE_TRAIN: (800,) + MAX_SIZE_TRAIN: 1333 + MIN_SIZE_TEST: 800 + MAX_SIZE_TEST: 1333 +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" + BACKBONE: + CONV_BODY: "R-50-FPN" + RESNETS: + BACKBONE_OUT_CHANNELS: 256 + STAGE_WITH_DCN: (False, True, True, True) + WITH_MODULATED_DCN: False + DEFORMABLE_GROUPS: 1 + RPN: + USE_FPN: True + ANCHOR_STRIDE: (4, 8, 16, 32, 64) + PRE_NMS_TOP_N_TRAIN: 2000 + PRE_NMS_TOP_N_TEST: 1000 + POST_NMS_TOP_N_TEST: 1000 + FPN_POST_NMS_TOP_N_TEST: 1000 + ROI_HEADS: + USE_FPN: True + ROI_BOX_HEAD: + POOLER_RESOLUTION: 7 + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + POOLER_SAMPLING_RATIO: 2 + FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" + PREDICTOR: "FPNPredictor" + ROI_MASK_HEAD: + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" + PREDICTOR: "MaskRCNNC4Predictor" + POOLER_RESOLUTION: 14 + POOLER_SAMPLING_RATIO: 2 + RESOLUTION: 28 + SHARE_BOX_FEATURE_EXTRACTOR: False + MASK_ON: True +DATASETS: + TRAIN: ("coco_2014_train", "coco_2014_valminusminival") + TEST: ("coco_2014_minival",) +DATALOADER: + SIZE_DIVISIBILITY: 32 +SOLVER: + # Assume 8 gpus + BASE_LR: 0.02 + WEIGHT_DECAY: 0.0001 + STEPS: (60000, 80000) + MAX_ITER: 90000 + IMS_PER_BATCH: 16 +TEST: + IMS_PER_BATCH: 8 + diff --git a/configs/dcn/e2e_mask_rcnn_mdconv_R_50_FPN_1x.yaml b/configs/dcn/e2e_mask_rcnn_mdconv_R_50_FPN_1x.yaml new file mode 100644 index 000000000..9921adfa5 --- /dev/null +++ b/configs/dcn/e2e_mask_rcnn_mdconv_R_50_FPN_1x.yaml @@ -0,0 +1,53 @@ +INPUT: + MIN_SIZE_TRAIN: (800,) + MAX_SIZE_TRAIN: 1333 + MIN_SIZE_TEST: 800 + MAX_SIZE_TEST: 1333 +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" + BACKBONE: + CONV_BODY: "R-50-FPN" + RESNETS: + BACKBONE_OUT_CHANNELS: 256 + STAGE_WITH_DCN: (False, True, True, True) + WITH_MODULATED_DCN: True + DEFORMABLE_GROUPS: 1 + RPN: + USE_FPN: True + ANCHOR_STRIDE: (4, 8, 16, 32, 64) + PRE_NMS_TOP_N_TRAIN: 2000 + PRE_NMS_TOP_N_TEST: 1000 + POST_NMS_TOP_N_TEST: 1000 + FPN_POST_NMS_TOP_N_TEST: 1000 + ROI_HEADS: + USE_FPN: True + ROI_BOX_HEAD: + POOLER_RESOLUTION: 7 + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + POOLER_SAMPLING_RATIO: 2 + FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" + PREDICTOR: "FPNPredictor" + ROI_MASK_HEAD: + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" + PREDICTOR: "MaskRCNNC4Predictor" + POOLER_RESOLUTION: 14 + POOLER_SAMPLING_RATIO: 2 + RESOLUTION: 28 + SHARE_BOX_FEATURE_EXTRACTOR: False + MASK_ON: True +DATASETS: + TRAIN: ("coco_2014_train", "coco_2014_valminusminival") + TEST: ("coco_2014_minival",) +DATALOADER: + SIZE_DIVISIBILITY: 32 +SOLVER: + # Assume 8 gpus + BASE_LR: 0.02 + WEIGHT_DECAY: 0.0001 + STEPS: (60000, 80000) + MAX_ITER: 90000 + IMS_PER_BATCH: 16 +TEST: + IMS_PER_BATCH: 8 diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index fc750fd4f..23a599ef7 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -274,6 +274,10 @@ _C.MODEL.RESNETS.RES2_OUT_CHANNELS = 256 _C.MODEL.RESNETS.STEM_OUT_CHANNELS = 64 +_C.MODEL.RESNETS.STAGE_WITH_DCN = (False, False, False, False) +_C.MODEL.RESNETS.WITH_MODULATED_DCN = False +_C.MODEL.RESNETS.DEFORMABLE_GROUPS = 1 + # ---------------------------------------------------------------------------- # # RetinaNet Options (Follow the Detectron version) diff --git a/maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.cu b/maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.cu new file mode 100644 index 000000000..74f7d3399 --- /dev/null +++ b/maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.cu @@ -0,0 +1,691 @@ +// modify from +// https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/deform_conv_cuda.c + +#include +#include + +#include +#include + +#include +#include +#include + + +void deformable_im2col(const at::Tensor data_im, const at::Tensor data_offset, + const int channels, const int height, const int width, + const int ksize_h, const int ksize_w, const int pad_h, + const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int parallel_imgs, const int deformable_group, + at::Tensor data_col); + +void deformable_col2im(const at::Tensor data_col, const at::Tensor data_offset, + const int channels, const int height, const int width, + const int ksize_h, const int ksize_w, const int pad_h, + const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int parallel_imgs, const int deformable_group, + at::Tensor grad_im); + +void deformable_col2im_coord( + const at::Tensor data_col, const at::Tensor data_im, + const at::Tensor data_offset, const int channels, const int height, + const int width, const int ksize_h, const int ksize_w, const int pad_h, + const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, const int parallel_imgs, + const int deformable_group, at::Tensor grad_offset); + +void modulated_deformable_im2col_cuda( + const at::Tensor data_im, const at::Tensor data_offset, + const at::Tensor data_mask, const int batch_size, const int channels, + const int height_im, const int width_im, const int height_col, + const int width_col, const int kernel_h, const int kenerl_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, const int deformable_group, + at::Tensor data_col); + +void modulated_deformable_col2im_cuda( + const at::Tensor data_col, const at::Tensor data_offset, + const at::Tensor data_mask, const int batch_size, const int channels, + const int height_im, const int width_im, const int height_col, + const int width_col, const int kernel_h, const int kenerl_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, const int deformable_group, + at::Tensor grad_im); + +void modulated_deformable_col2im_coord_cuda( + const at::Tensor data_col, const at::Tensor data_im, + const at::Tensor data_offset, const at::Tensor data_mask, + const int batch_size, const int channels, const int height_im, + const int width_im, const int height_col, const int width_col, + const int kernel_h, const int kenerl_w, const int pad_h, const int pad_w, + const int stride_h, const int stride_w, const int dilation_h, + const int dilation_w, const int deformable_group, at::Tensor grad_offset, + at::Tensor grad_mask); + +void shape_check(at::Tensor input, at::Tensor offset, at::Tensor *gradOutput, + at::Tensor weight, int kH, int kW, int dH, int dW, int padH, + int padW, int dilationH, int dilationW, int group, + int deformable_group) +{ + AT_CHECK(weight.ndimension() == 4, + "4D weight tensor (nOutputPlane,nInputPlane,kH,kW) expected, " + "but got: %s", + weight.ndimension()); + + AT_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + AT_CHECK(kW > 0 && kH > 0, + "kernel size should be greater than zero, but got kH: %d kW: %d", kH, + kW); + + AT_CHECK((weight.size(2) == kH && weight.size(3) == kW), + "kernel size should be consistent with weight, ", + "but got kH: %d kW: %d weight.size(2): %d, weight.size(3): %d", kH, + kW, weight.size(2), weight.size(3)); + + AT_CHECK(dW > 0 && dH > 0, + "stride should be greater than zero, but got dH: %d dW: %d", dH, dW); + + AT_CHECK( + dilationW > 0 && dilationH > 0, + "dilation should be greater than 0, but got dilationH: %d dilationW: %d", + dilationH, dilationW); + + int ndim = input.ndimension(); + int dimf = 0; + int dimh = 1; + int dimw = 2; + + if (ndim == 4) { + dimf++; + dimh++; + dimw++; + } + + AT_CHECK(ndim == 3 || ndim == 4, "3D or 4D input tensor expected but got: %s", + ndim); + + long nInputPlane = weight.size(1) * group; + long inputHeight = input.size(dimh); + long inputWidth = input.size(dimw); + long nOutputPlane = weight.size(0); + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + + AT_CHECK(nInputPlane % deformable_group == 0, + "input channels must divide deformable group size"); + + if (outputWidth < 1 || outputHeight < 1) + AT_ERROR( + "Given input size: (%ld x %ld x %ld). " + "Calculated output size: (%ld x %ld x %ld). Output size is too small", + nInputPlane, inputHeight, inputWidth, nOutputPlane, outputHeight, + outputWidth); + + AT_CHECK(input.size(1) == nInputPlane, + "invalid number of input planes, expected: %d, but got: %d", + nInputPlane, input.size(1)); + + AT_CHECK((inputHeight >= kH && inputWidth >= kW), + "input image is smaller than kernel"); + + AT_CHECK((offset.size(2) == outputHeight && offset.size(3) == outputWidth), + "invalid spatial size of offset, expected height: %d width: %d, but " + "got height: %d width: %d", + outputHeight, outputWidth, offset.size(2), offset.size(3)); + + AT_CHECK((offset.size(1) == deformable_group * 2 * kH * kW), + "invalid number of channels of offset"); + + if (gradOutput != NULL) { + AT_CHECK(gradOutput->size(dimf) == nOutputPlane, + "invalid number of gradOutput planes, expected: %d, but got: %d", + nOutputPlane, gradOutput->size(dimf)); + + AT_CHECK((gradOutput->size(dimh) == outputHeight && + gradOutput->size(dimw) == outputWidth), + "invalid size of gradOutput, expected height: %d width: %d , but " + "got height: %d width: %d", + outputHeight, outputWidth, gradOutput->size(dimh), + gradOutput->size(dimw)); + } +} + +int deform_conv_forward_cuda(at::Tensor input, at::Tensor weight, + at::Tensor offset, at::Tensor output, + at::Tensor columns, at::Tensor ones, int kW, + int kH, int dW, int dH, int padW, int padH, + int dilationW, int dilationH, int group, + int deformable_group, int im2col_step) +{ + // todo: resize columns to include im2col: done + // todo: add im2col_step as input + // todo: add new output buffer and transpose it to output (or directly + // transpose output) todo: possibly change data indexing because of + // parallel_imgs + + shape_check(input, offset, NULL, weight, kH, kW, dH, dW, padH, padW, + dilationH, dilationW, group, deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + weight = weight.contiguous(); + + int batch = 1; + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input.unsqueeze_(0); + offset.unsqueeze_(0); + } + + // todo: assert batchsize dividable by im2col_step + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = weight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + AT_CHECK((offset.size(0) == batchSize), "invalid batch size of offset"); + + output = output.view({batchSize / im2col_step, im2col_step, nOutputPlane, + outputHeight, outputWidth}); + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < outputHeight * outputWidth) { + ones = at::ones({outputHeight, outputWidth}, input.options()); + } + + input = input.view({batchSize / im2col_step, im2col_step, nInputPlane, + inputHeight, inputWidth}); + offset = + offset.view({batchSize / im2col_step, im2col_step, + deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + at::Tensor output_buffer = + at::zeros({batchSize / im2col_step, nOutputPlane, + im2col_step * outputHeight, outputWidth}, + output.options()); + + output_buffer = output_buffer.view( + {output_buffer.size(0), group, output_buffer.size(1) / group, + output_buffer.size(2), output_buffer.size(3)}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + deformable_im2col(input[elt], offset[elt], nInputPlane, inputHeight, + inputWidth, kH, kW, padH, padW, dH, dW, dilationH, + dilationW, im2col_step, deformable_group, columns); + + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view({group, weight.size(0) / group, weight.size(1), + weight.size(2), weight.size(3)}); + + for (int g = 0; g < group; g++) { + output_buffer[elt][g] = output_buffer[elt][g] + .flatten(1) + .addmm_(weight[g].flatten(1), columns[g]) + .view_as(output_buffer[elt][g]); + } + } + + output_buffer = output_buffer.view( + {output_buffer.size(0), output_buffer.size(1) * output_buffer.size(2), + output_buffer.size(3), output_buffer.size(4)}); + + output_buffer = output_buffer.view({batchSize / im2col_step, nOutputPlane, + im2col_step, outputHeight, outputWidth}); + output_buffer.transpose_(1, 2); + output.copy_(output_buffer); + output = output.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + output = output.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + offset = offset.view({offset.size(1), offset.size(2), offset.size(3)}); + } + + return 1; +} + +int deform_conv_backward_input_cuda(at::Tensor input, at::Tensor offset, + at::Tensor gradOutput, at::Tensor gradInput, + at::Tensor gradOffset, at::Tensor weight, + at::Tensor columns, int kW, int kH, int dW, + int dH, int padW, int padH, int dilationW, + int dilationH, int group, + int deformable_group, int im2col_step) +{ + shape_check(input, offset, &gradOutput, weight, kH, kW, dH, dW, padH, padW, + dilationH, dilationW, group, deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + gradOutput = gradOutput.contiguous(); + weight = weight.contiguous(); + + int batch = 1; + + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input = input.view({1, input.size(0), input.size(1), input.size(2)}); + offset = offset.view({1, offset.size(0), offset.size(1), offset.size(2)}); + gradOutput = gradOutput.view( + {1, gradOutput.size(0), gradOutput.size(1), gradOutput.size(2)}); + } + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = weight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + AT_CHECK((offset.size(0) == batchSize), 3, "invalid batch size of offset"); + gradInput = gradInput.view({batchSize, nInputPlane, inputHeight, inputWidth}); + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + // change order of grad output + gradOutput = gradOutput.view({batchSize / im2col_step, im2col_step, + nOutputPlane, outputHeight, outputWidth}); + gradOutput.transpose_(1, 2); + + gradInput = gradInput.view({batchSize / im2col_step, im2col_step, nInputPlane, + inputHeight, inputWidth}); + input = input.view({batchSize / im2col_step, im2col_step, nInputPlane, + inputHeight, inputWidth}); + gradOffset = gradOffset.view({batchSize / im2col_step, im2col_step, + deformable_group * 2 * kH * kW, outputHeight, + outputWidth}); + offset = + offset.view({batchSize / im2col_step, im2col_step, + deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + // divide into groups + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view({group, weight.size(0) / group, weight.size(1), + weight.size(2), weight.size(3)}); + gradOutput = gradOutput.view( + {gradOutput.size(0), group, gradOutput.size(1) / group, + gradOutput.size(2), gradOutput.size(3), gradOutput.size(4)}); + + for (int g = 0; g < group; g++) { + columns[g] = columns[g].addmm_(weight[g].flatten(1).transpose(0, 1), + gradOutput[elt][g].flatten(1), 0.0f, 1.0f); + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + gradOutput = gradOutput.view( + {gradOutput.size(0), gradOutput.size(1) * gradOutput.size(2), + gradOutput.size(3), gradOutput.size(4), gradOutput.size(5)}); + + deformable_col2im_coord(columns, input[elt], offset[elt], nInputPlane, + inputHeight, inputWidth, kH, kW, padH, padW, dH, dW, + dilationH, dilationW, im2col_step, deformable_group, + gradOffset[elt]); + + deformable_col2im(columns, offset[elt], nInputPlane, inputHeight, + inputWidth, kH, kW, padH, padW, dH, dW, dilationH, + dilationW, im2col_step, deformable_group, gradInput[elt]); + } + + gradOutput.transpose_(1, 2); + gradOutput = + gradOutput.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + gradInput = gradInput.view({batchSize, nInputPlane, inputHeight, inputWidth}); + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + gradOffset = gradOffset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + gradOutput = gradOutput.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + gradInput = gradInput.view({nInputPlane, inputHeight, inputWidth}); + offset = offset.view({offset.size(1), offset.size(2), offset.size(3)}); + gradOffset = + gradOffset.view({offset.size(1), offset.size(2), offset.size(3)}); + } + + return 1; +} + +int deform_conv_backward_parameters_cuda( + at::Tensor input, at::Tensor offset, at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, at::Tensor ones, int kW, int kH, int dW, int dH, + int padW, int padH, int dilationW, int dilationH, int group, + int deformable_group, float scale, int im2col_step) +{ + // todo: transpose and reshape outGrad + // todo: reshape columns + // todo: add im2col_step as input + + shape_check(input, offset, &gradOutput, gradWeight, kH, kW, dH, dW, padH, + padW, dilationH, dilationW, group, deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + gradOutput = gradOutput.contiguous(); + + int batch = 1; + + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input = input.view( + at::IntList({1, input.size(0), input.size(1), input.size(2)})); + gradOutput = gradOutput.view( + {1, gradOutput.size(0), gradOutput.size(1), gradOutput.size(2)}); + } + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = gradWeight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + AT_CHECK((offset.size(0) == batchSize), "invalid batch size of offset"); + + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + gradOutput = gradOutput.view({batchSize / im2col_step, im2col_step, + nOutputPlane, outputHeight, outputWidth}); + gradOutput.transpose_(1, 2); + + at::Tensor gradOutputBuffer = at::zeros_like(gradOutput); + gradOutputBuffer = + gradOutputBuffer.view({batchSize / im2col_step, nOutputPlane, im2col_step, + outputHeight, outputWidth}); + gradOutputBuffer.copy_(gradOutput); + gradOutputBuffer = + gradOutputBuffer.view({batchSize / im2col_step, nOutputPlane, + im2col_step * outputHeight, outputWidth}); + + gradOutput.transpose_(1, 2); + gradOutput = + gradOutput.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + input = input.view({batchSize / im2col_step, im2col_step, nInputPlane, + inputHeight, inputWidth}); + offset = + offset.view({batchSize / im2col_step, im2col_step, + deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + deformable_im2col(input[elt], offset[elt], nInputPlane, inputHeight, + inputWidth, kH, kW, padH, padW, dH, dW, dilationH, + dilationW, im2col_step, deformable_group, columns); + + // divide into group + gradOutputBuffer = gradOutputBuffer.view( + {gradOutputBuffer.size(0), group, gradOutputBuffer.size(1) / group, + gradOutputBuffer.size(2), gradOutputBuffer.size(3)}); + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + gradWeight = + gradWeight.view({group, gradWeight.size(0) / group, gradWeight.size(1), + gradWeight.size(2), gradWeight.size(3)}); + + for (int g = 0; g < group; g++) { + gradWeight[g] = gradWeight[g] + .flatten(1) + .addmm_(gradOutputBuffer[elt][g].flatten(1), + columns[g].transpose(1, 0), 1.0, scale) + .view_as(gradWeight[g]); + } + gradOutputBuffer = gradOutputBuffer.view( + {gradOutputBuffer.size(0), + gradOutputBuffer.size(1) * gradOutputBuffer.size(2), + gradOutputBuffer.size(3), gradOutputBuffer.size(4)}); + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + gradWeight = gradWeight.view({gradWeight.size(0) * gradWeight.size(1), + gradWeight.size(2), gradWeight.size(3), + gradWeight.size(4)}); + } + + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + gradOutput = gradOutput.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + } + + return 1; +} + +void modulated_deform_conv_cuda_forward( + at::Tensor input, at::Tensor weight, at::Tensor bias, at::Tensor ones, + at::Tensor offset, at::Tensor mask, at::Tensor output, at::Tensor columns, + int kernel_h, int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int deformable_group, + const bool with_bias) +{ + AT_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + AT_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + if (kernel_h_ != kernel_h || kernel_w_ != kernel_w) + AT_ERROR("Input shape and kernel shape wont match: (%d x %d vs %d x %d).", + kernel_h_, kernel_w, kernel_h_, kernel_w_); + if (channels != channels_kernel * group) + AT_ERROR("Input shape and kernel channels wont match: (%d vs %d).", + channels, channels_kernel * group); + + const int height_out = + (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = + (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < height_out * width_out) { + // Resize plane and fill with ones... + ones = at::ones({height_out, width_out}, input.options()); + } + + // resize output + output = output.view({batch, channels_out, height_out, width_out}).zero_(); + // resize temporary columns + columns = + at::zeros({channels * kernel_h * kernel_w, 1 * height_out * width_out}, + input.options()); + + output = output.view({output.size(0), group, output.size(1) / group, + output.size(2), output.size(3)}); + + for (int b = 0; b < batch; b++) { + modulated_deformable_im2col_cuda( + input[b], offset[b], mask[b], 1, channels, height, width, height_out, + width_out, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, columns); + + // divide into group + weight = weight.view({group, weight.size(0) / group, weight.size(1), + weight.size(2), weight.size(3)}); + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + + for (int g = 0; g < group; g++) { + output[b][g] = output[b][g] + .flatten(1) + .addmm_(weight[g].flatten(1), columns[g]) + .view_as(output[b][g]); + } + + weight = weight.view({weight.size(0) * weight.size(1), weight.size(2), + weight.size(3), weight.size(4)}); + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + } + + output = output.view({output.size(0), output.size(1) * output.size(2), + output.size(3), output.size(4)}); + + if (with_bias) { + output += bias.view({1, bias.size(0), 1, 1}); + } +} + +void modulated_deform_conv_cuda_backward( + at::Tensor input, at::Tensor weight, at::Tensor bias, at::Tensor ones, + at::Tensor offset, at::Tensor mask, at::Tensor columns, + at::Tensor grad_input, at::Tensor grad_weight, at::Tensor grad_bias, + at::Tensor grad_offset, at::Tensor grad_mask, at::Tensor grad_output, + int kernel_h, int kernel_w, int stride_h, int stride_w, int pad_h, + int pad_w, int dilation_h, int dilation_w, int group, int deformable_group, + const bool with_bias) +{ + AT_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + AT_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + if (kernel_h_ != kernel_h || kernel_w_ != kernel_w) + AT_ERROR("Input shape and kernel shape wont match: (%d x %d vs %d x %d).", + kernel_h_, kernel_w, kernel_h_, kernel_w_); + if (channels != channels_kernel * group) + AT_ERROR("Input shape and kernel channels wont match: (%d vs %d).", + channels, channels_kernel * group); + + const int height_out = + (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = + (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < height_out * width_out) { + // Resize plane and fill with ones... + ones = at::ones({height_out, width_out}, input.options()); + } + + grad_input = grad_input.view({batch, channels, height, width}); + columns = at::zeros({channels * kernel_h * kernel_w, height_out * width_out}, + input.options()); + + grad_output = + grad_output.view({grad_output.size(0), group, grad_output.size(1) / group, + grad_output.size(2), grad_output.size(3)}); + + for (int b = 0; b < batch; b++) { + // divide int group + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view({group, weight.size(0) / group, weight.size(1), + weight.size(2), weight.size(3)}); + + for (int g = 0; g < group; g++) { + columns[g].addmm_(weight[g].flatten(1).transpose(0, 1), + grad_output[b][g].flatten(1), 0.0f, 1.0f); + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + weight = weight.view({weight.size(0) * weight.size(1), weight.size(2), + weight.size(3), weight.size(4)}); + + // gradient w.r.t. input coordinate data + modulated_deformable_col2im_coord_cuda( + columns, input[b], offset[b], mask[b], 1, channels, height, width, + height_out, width_out, kernel_h, kernel_w, pad_h, pad_w, stride_h, + stride_w, dilation_h, dilation_w, deformable_group, grad_offset[b], + grad_mask[b]); + // gradient w.r.t. input data + modulated_deformable_col2im_cuda( + columns, offset[b], mask[b], 1, channels, height, width, height_out, + width_out, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, grad_input[b]); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and + // group + modulated_deformable_im2col_cuda( + input[b], offset[b], mask[b], 1, channels, height, width, height_out, + width_out, kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, deformable_group, columns); + + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + grad_weight = grad_weight.view({group, grad_weight.size(0) / group, + grad_weight.size(1), grad_weight.size(2), + grad_weight.size(3)}); + if (with_bias) + grad_bias = grad_bias.view({group, grad_bias.size(0) / group}); + + for (int g = 0; g < group; g++) { + grad_weight[g] = + grad_weight[g] + .flatten(1) + .addmm_(grad_output[b][g].flatten(1), columns[g].transpose(0, 1)) + .view_as(grad_weight[g]); + if (with_bias) { + grad_bias[g] = + grad_bias[g] + .view({-1, 1}) + .addmm_(grad_output[b][g].flatten(1), ones.view({-1, 1})) + .view(-1); + } + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + grad_weight = grad_weight.view({grad_weight.size(0) * grad_weight.size(1), + grad_weight.size(2), grad_weight.size(3), + grad_weight.size(4)}); + if (with_bias) + grad_bias = grad_bias.view({grad_bias.size(0) * grad_bias.size(1)}); + } + grad_output = grad_output.view({grad_output.size(0) * grad_output.size(1), + grad_output.size(2), grad_output.size(3), + grad_output.size(4)}); +} diff --git a/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu b/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu new file mode 100644 index 000000000..b4f8813b4 --- /dev/null +++ b/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu @@ -0,0 +1,874 @@ +/*! + ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** + * + * COPYRIGHT + * + * All contributions by the University of California: + * Copyright (c) 2014-2017 The Regents of the University of California (Regents) + * All rights reserved. + * + * All other contributions: + * Copyright (c) 2014-2017, the respective contributors + * All rights reserved. + * + * Caffe uses a shared copyright model: each contributor holds copyright over + * their contributions to Caffe. The project versioning records all such + * contribution and copyright details. If a contributor wants to further mark + * their specific copyright on a particular contribution, they should indicate + * their copyright solely in the commit message of the change when it is + * committed. + * + * LICENSE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * CONTRIBUTION AGREEMENT + * + * By contributing to the BVLC/caffe repository through pull-request, comment, + * or otherwise, the contributor releases their content to the + * license and copyright terms herein. + * + ***************** END Caffe Copyright Notice and Disclaimer ******************** + * + * Copyright (c) 2018 Microsoft + * Licensed under The MIT License [see LICENSE for details] + * \file modulated_deformable_im2col.cuh + * \brief Function definitions of converting an image to + * column matrix based on kernel, padding, dilation, and offset. + * These functions are mainly used in deformable convolution operators. + * \ref: https://arxiv.org/abs/1703.06211 + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai, Xizhou Zhu, Han Hu, Dazhi Cheng + */ + +// modify from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/deform_conv_cuda_kernel.cu + + +#include +#include +#include +#include +#include + +using namespace at; + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +const int kMaxGridNum = 65535; +inline int GET_BLOCKS(const int N) +{ + return std::min(kMaxGridNum, (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS); +} + +/* +const int CUDA_NUM_THREADS = 1024; + +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +}*/ + +template +__device__ scalar_t deformable_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void deformable_im2col_gpu_kernel(const int n, const scalar_t *data_im, const scalar_t *data_offset, + const int height, const int width, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, const int channel_per_deformable_group, + const int batch_size, const int num_channels, const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const scalar_t map_h = i * dilation_h + offset_h; + //const scalar_t map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = deformable_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = deformable_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + +void deformable_im2col( + const at::Tensor data_im, const at::Tensor data_offset, const int channels, + const int height, const int width, const int ksize_h, const int ksize_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, const int parallel_imgs, + const int deformable_group, at::Tensor data_col) +{ + // num_axes should be smaller than block size + // todo: check parallel_imgs is correctly passed in + int height_col = (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = channels * height_col * width_col * parallel_imgs; + int channel_per_deformable_group = channels / deformable_group; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_im.type(), "deformable_im2col_gpu", ([&] { + const scalar_t *data_im_ = data_im.data(); + const scalar_t *data_offset_ = data_offset.data(); + scalar_t *data_col_ = data_col.data(); + + deformable_im2col_gpu_kernel<<>>( + num_kernels, data_im_, data_offset_, height, width, ksize_h, ksize_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, + channel_per_deformable_group, parallel_imgs, channels, deformable_group, + height_col, width_col, data_col_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_im2col: %s\n", cudaGetErrorString(err)); + } +} + +template +__global__ void deformable_col2im_gpu_kernel( + const int n, const scalar_t *data_col, const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * + 2 * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +void deformable_col2im( + const at::Tensor data_col, const at::Tensor data_offset, const int channels, + const int height, const int width, const int ksize_h, + const int ksize_w, const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int parallel_imgs, const int deformable_group, + at::Tensor grad_im) +{ + + // todo: make sure parallel_imgs is passed in correctly + int height_col = (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = channels * ksize_h * ksize_w * height_col * width_col * parallel_imgs; + int channel_per_deformable_group = channels / deformable_group; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.type(), "deformable_col2im_gpu", ([&] { + const scalar_t *data_col_ = data_col.data(); + const scalar_t *data_offset_ = data_offset.data(); + scalar_t *grad_im_ = grad_im.data(); + + deformable_col2im_gpu_kernel<<>>( + num_kernels, data_col_, data_offset_, channels, height, width, ksize_h, + ksize_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + parallel_imgs, deformable_group, height_col, width_col, grad_im_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in deformable_col2im: %s\n", cudaGetErrorString(err)); + } +} + +template +__global__ void deformable_col2im_coord_gpu_kernel(const int n, const scalar_t *data_col, + const scalar_t *data_im, const scalar_t *data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, const int deformable_group, + const int height_col, const int width_col, scalar_t *grad_offset) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * + batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * + channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * + kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + const scalar_t weight = get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + + grad_offset[index] = val; + } +} + +void deformable_col2im_coord( + const at::Tensor data_col, const at::Tensor data_im, const at::Tensor data_offset, + const int channels, const int height, const int width, const int ksize_h, + const int ksize_w, const int pad_h, const int pad_w, const int stride_h, + const int stride_w, const int dilation_h, const int dilation_w, + const int parallel_imgs, const int deformable_group, at::Tensor grad_offset) +{ + + int height_col = (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = height_col * width_col * 2 * ksize_h * ksize_w * deformable_group * parallel_imgs; + int channel_per_deformable_group = channels * ksize_h * ksize_w / deformable_group; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.type(), "deformable_col2im_coord_gpu", ([&] { + const scalar_t *data_col_ = data_col.data(); + const scalar_t *data_im_ = data_im.data(); + const scalar_t *data_offset_ = data_offset.data(); + scalar_t *grad_offset_ = grad_offset.data(); + + deformable_col2im_coord_gpu_kernel<<>>( + num_kernels, data_col_, data_im_, data_offset_, channels, height, width, + ksize_h, ksize_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + parallel_imgs, 2 * ksize_h * ksize_w * deformable_group, deformable_group, + height_col, width_col, grad_offset_); + })); +} + +template +__device__ scalar_t dmcn_im2col_bilinear(const scalar_t *bottom_data, const int data_width, + const int height, const int width, scalar_t h, scalar_t w) +{ + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t dmcn_get_gradient_weight(scalar_t argmax_h, scalar_t argmax_w, + const int h, const int w, const int height, const int width) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t dmcn_get_coordinate_weight(scalar_t argmax_h, scalar_t argmax_w, + const int height, const int width, const scalar_t *im_data, + const int data_width, const int bp_dir) +{ + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || argmax_w >= width) + { + //empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + else if (bp_dir == 1) + { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void modulated_deformable_im2col_gpu_kernel(const int n, + const scalar_t *data_im, const scalar_t *data_offset, const scalar_t *data_mask, + const int height, const int width, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int num_channels, const int deformable_group, + const int height_col, const int width_col, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t *data_col_ptr = data_col + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + //const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * height + h_in) * width + w_in; + const scalar_t *data_im_ptr = data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t *data_offset_ptr = data_offset + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + + const scalar_t *data_mask_ptr = data_mask + (b_col * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) + { + for (int j = 0; j < kernel_w; ++j) + { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + //if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) + { + //const float map_h = i * dilation_h + offset_h; + //const float map_w = j * dilation_w + offset_w; + //const int cur_height = height - h_in; + //const int cur_width = width - w_in; + //val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + val = dmcn_im2col_bilinear(data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * height_col * width_col; + //data_col_ptr += height_col * width_col; + } + } + } +} + +template +__global__ void modulated_deformable_col2im_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_offset, const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_im) +{ + CUDA_KERNEL_LOOP(index, n) + { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const int data_mask_hw_ptr = ((i * kernel_w + j) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index] * mask; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) + { + for (int dx = -2; dx <= 2; dx++) + { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) + { + int cur_bottom_grad_pos = ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = dmcn_get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void modulated_deformable_col2im_coord_gpu_kernel(const int n, + const scalar_t *data_col, const scalar_t *data_im, + const scalar_t *data_offset, const scalar_t *data_mask, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, const int offset_channels, const int deformable_group, + const int height_col, const int width_col, + scalar_t *grad_offset, scalar_t *grad_mask) +{ + CUDA_KERNEL_LOOP(index, n) + { + scalar_t val = 0, mval = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t *data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * batch_size * width_col * height_col; + const scalar_t *data_im_ptr = data_im + (b * deformable_group + deformable_group_index) * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t *data_offset_ptr = data_offset + (b * deformable_group + deformable_group_index) * 2 * kernel_h * kernel_w * height_col * width_col; + const scalar_t *data_mask_ptr = data_mask + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) + { + const int col_pos = (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const int data_mask_hw_ptr = (((i * kernel_w + j) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) + { + inv_h = inv_w = -2; + } + else + { + mval += data_col_ptr[col_pos] * dmcn_im2col_bilinear(data_im_ptr + cnt * height * width, width, height, width, inv_h, inv_w); + } + const scalar_t weight = dmcn_get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos] * mask; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + if (offset_c % 2 == 0) + // KERNEL_ASSIGN(grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w], mask_req, mval); + grad_mask[(((b * deformable_group + deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * height_col + h) * width_col + w] = mval; + } +} + +void modulated_deformable_im2col_cuda( + const at::Tensor data_im, const at::Tensor data_offset, const at::Tensor data_mask, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kenerl_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, at::Tensor data_col) +{ + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_im.type(), "modulated_deformable_im2col_gpu", ([&] { + const scalar_t *data_im_ = data_im.data(); + const scalar_t *data_offset_ = data_offset.data(); + const scalar_t *data_mask_ = data_mask.data(); + scalar_t *data_col_ = data_col.data(); + + modulated_deformable_im2col_gpu_kernel<<>>( + num_kernels, data_im_, data_offset_, data_mask_, height_im, width_im, kernel_h, kenerl_w, + pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, channel_per_deformable_group, + batch_size, channels, deformable_group, height_col, width_col, data_col_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } +} + +void modulated_deformable_col2im_cuda( + const at::Tensor data_col, const at::Tensor data_offset, const at::Tensor data_mask, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, at::Tensor grad_im) +{ + + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * kernel_h * kernel_w * batch_size * height_col * width_col; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.type(), "modulated_deformable_col2im_gpu", ([&] { + const scalar_t *data_col_ = data_col.data(); + const scalar_t *data_offset_ = data_offset.data(); + const scalar_t *data_mask_ = data_mask.data(); + scalar_t *grad_im_ = grad_im.data(); + + modulated_deformable_col2im_gpu_kernel<<>>( + num_kernels, data_col_, data_offset_, data_mask_, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_h, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, deformable_group, height_col, width_col, grad_im_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } +} + +void modulated_deformable_col2im_coord_cuda( + const at::Tensor data_col, const at::Tensor data_im, const at::Tensor data_offset, const at::Tensor data_mask, + const int batch_size, const int channels, const int height_im, const int width_im, + const int height_col, const int width_col, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int deformable_group, + at::Tensor grad_offset, at::Tensor grad_mask) +{ + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * kernel_w * deformable_group; + const int channel_per_deformable_group = channels * kernel_h * kernel_w / deformable_group; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.type(), "modulated_deformable_col2im_coord_gpu", ([&] { + const scalar_t *data_col_ = data_col.data(); + const scalar_t *data_im_ = data_im.data(); + const scalar_t *data_offset_ = data_offset.data(); + const scalar_t *data_mask_ = data_mask.data(); + scalar_t *grad_offset_ = grad_offset.data(); + scalar_t *grad_mask_ = grad_mask.data(); + + modulated_deformable_col2im_coord_gpu_kernel<<>>( + num_kernels, data_col_, data_im_, data_offset_, data_mask_, channels, height_im, width_im, + kernel_h, kernel_w, pad_h, pad_w, stride_h, stride_w, + dilation_h, dilation_w, channel_per_deformable_group, + batch_size, 2 * kernel_h * kernel_w * deformable_group, deformable_group, height_col, width_col, + grad_offset_, grad_mask_); + })); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in modulated_deformable_col2im_coord_cuda: %s\n", cudaGetErrorString(err)); + } +} diff --git a/maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.cu b/maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.cu new file mode 100644 index 000000000..71f305af9 --- /dev/null +++ b/maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.cu @@ -0,0 +1,87 @@ +// modify from +// https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/modulated_dcn_cuda.c + +// based on +// author: Charles Shang +// https://github.com/torch/cunn/blob/master/lib/THCUNN/generic/SpatialConvolutionMM.cu + +#include +#include + +#include +#include + +#include +#include +#include + + +void DeformablePSROIPoolForward( + const at::Tensor data, const at::Tensor bbox, const at::Tensor trans, + at::Tensor out, at::Tensor top_count, const int batch, const int channels, + const int height, const int width, const int num_bbox, + const int channels_trans, const int no_trans, const float spatial_scale, + const int output_dim, const int group_size, const int pooled_size, + const int part_size, const int sample_per_part, const float trans_std); + +void DeformablePSROIPoolBackwardAcc( + const at::Tensor out_grad, const at::Tensor data, const at::Tensor bbox, + const at::Tensor trans, const at::Tensor top_count, at::Tensor in_grad, + at::Tensor trans_grad, const int batch, const int channels, + const int height, const int width, const int num_bbox, + const int channels_trans, const int no_trans, const float spatial_scale, + const int output_dim, const int group_size, const int pooled_size, + const int part_size, const int sample_per_part, const float trans_std); + +void deform_psroi_pooling_cuda_forward( + at::Tensor input, at::Tensor bbox, at::Tensor trans, at::Tensor out, + at::Tensor top_count, const int no_trans, const float spatial_scale, + const int output_dim, const int group_size, const int pooled_size, + const int part_size, const int sample_per_part, const float trans_std) +{ + AT_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + const int channels_trans = no_trans ? 2 : trans.size(1); + + const int num_bbox = bbox.size(0); + if (num_bbox != out.size(0)) + AT_ERROR("Output shape and bbox number wont match: (%d vs %d).", + out.size(0), num_bbox); + + DeformablePSROIPoolForward( + input, bbox, trans, out, top_count, batch, channels, height, width, + num_bbox, channels_trans, no_trans, spatial_scale, output_dim, group_size, + pooled_size, part_size, sample_per_part, trans_std); +} + +void deform_psroi_pooling_cuda_backward( + at::Tensor out_grad, at::Tensor input, at::Tensor bbox, at::Tensor trans, + at::Tensor top_count, at::Tensor input_grad, at::Tensor trans_grad, + const int no_trans, const float spatial_scale, const int output_dim, + const int group_size, const int pooled_size, const int part_size, + const int sample_per_part, const float trans_std) +{ + AT_CHECK(out_grad.is_contiguous(), "out_grad tensor has to be contiguous"); + AT_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + const int channels_trans = no_trans ? 2 : trans.size(1); + + const int num_bbox = bbox.size(0); + if (num_bbox != out_grad.size(0)) + AT_ERROR("Output shape and bbox number wont match: (%d vs %d).", + out_grad.size(0), num_bbox); + + DeformablePSROIPoolBackwardAcc( + out_grad, input, bbox, trans, top_count, input_grad, trans_grad, batch, + channels, height, width, num_bbox, channels_trans, no_trans, + spatial_scale, output_dim, group_size, pooled_size, part_size, + sample_per_part, trans_std); +} diff --git a/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu b/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu new file mode 100644 index 000000000..127899ec6 --- /dev/null +++ b/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu @@ -0,0 +1,365 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The MIT License [see LICENSE for details] + * \file deformable_psroi_pooling.cu + * \brief + * \author Yi Li, Guodong Zhang, Jifeng Dai +*/ +/***************** Adapted by Charles Shang *********************/ +// modify from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/cuda/deform_psroi_pooling_cuda.cu + + +#include +#include +#include +#include +#include + +using namespace at; + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N) +{ + return (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS; +} + +template +__device__ scalar_t bilinear_interp( + const scalar_t *data, + const scalar_t x, + const scalar_t y, + const int width, + const int height) +{ + int x1 = floor(x); + int x2 = ceil(x); + int y1 = floor(y); + int y2 = ceil(y); + scalar_t dist_x = (scalar_t)(x - x1); + scalar_t dist_y = (scalar_t)(y - y1); + scalar_t value11 = data[y1 * width + x1]; + scalar_t value12 = data[y2 * width + x1]; + scalar_t value21 = data[y1 * width + x2]; + scalar_t value22 = data[y2 * width + x2]; + scalar_t value = (1 - dist_x) * (1 - dist_y) * value11 + (1 - dist_x) * dist_y * value12 + dist_x * (1 - dist_y) * value21 + dist_x * dist_y * value22; + return value; +} + +template +__global__ void DeformablePSROIPoolForwardKernel( + const int count, + const scalar_t *bottom_data, + const scalar_t spatial_scale, + const int channels, + const int height, const int width, + const int pooled_height, const int pooled_width, + const scalar_t *bottom_rois, const scalar_t *bottom_trans, + const int no_trans, + const scalar_t trans_std, + const int sample_per_part, + const int output_dim, + const int group_size, + const int part_size, + const int num_classes, + const int channels_each_class, + scalar_t *top_data, + scalar_t *top_count) +{ + CUDA_KERNEL_LOOP(index, count) + { + // The output is in order (n, ctop, ph, pw) + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int ctop = (index / pooled_width / pooled_height) % output_dim; + int n = index / pooled_width / pooled_height / output_dim; + + // [start, end) interval for spatial sampling + const scalar_t *offset_bottom_rois = bottom_rois + n * 5; + int roi_batch_ind = offset_bottom_rois[0]; + scalar_t roi_start_w = (scalar_t)(round(offset_bottom_rois[1])) * spatial_scale - 0.5; + scalar_t roi_start_h = (scalar_t)(round(offset_bottom_rois[2])) * spatial_scale - 0.5; + scalar_t roi_end_w = (scalar_t)(round(offset_bottom_rois[3]) + 1.) * spatial_scale - 0.5; + scalar_t roi_end_h = (scalar_t)(round(offset_bottom_rois[4]) + 1.) * spatial_scale - 0.5; + + // Force too small ROIs to be 1x1 + scalar_t roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0 + scalar_t roi_height = max(roi_end_h - roi_start_h, 0.1); + + // Compute w and h at bottom + scalar_t bin_size_h = roi_height / (scalar_t)(pooled_height); + scalar_t bin_size_w = roi_width / (scalar_t)(pooled_width); + + scalar_t sub_bin_size_h = bin_size_h / (scalar_t)(sample_per_part); + scalar_t sub_bin_size_w = bin_size_w / (scalar_t)(sample_per_part); + + int part_h = floor((scalar_t)(ph) / pooled_height * part_size); + int part_w = floor((scalar_t)(pw) / pooled_width * part_size); + int class_id = ctop / channels_each_class; + scalar_t trans_x = no_trans ? (scalar_t)(0) : bottom_trans[(((n * num_classes + class_id) * 2) * part_size + part_h) * part_size + part_w] * (scalar_t)trans_std; + scalar_t trans_y = no_trans ? (scalar_t)(0) : bottom_trans[(((n * num_classes + class_id) * 2 + 1) * part_size + part_h) * part_size + part_w] * (scalar_t)trans_std; + + scalar_t wstart = (scalar_t)(pw)*bin_size_w + roi_start_w; + wstart += trans_x * roi_width; + scalar_t hstart = (scalar_t)(ph)*bin_size_h + roi_start_h; + hstart += trans_y * roi_height; + + scalar_t sum = 0; + int count = 0; + int gw = floor((scalar_t)(pw)*group_size / pooled_width); + int gh = floor((scalar_t)(ph)*group_size / pooled_height); + gw = min(max(gw, 0), group_size - 1); + gh = min(max(gh, 0), group_size - 1); + + const scalar_t *offset_bottom_data = bottom_data + (roi_batch_ind * channels) * height * width; + for (int ih = 0; ih < sample_per_part; ih++) + { + for (int iw = 0; iw < sample_per_part; iw++) + { + scalar_t w = wstart + iw * sub_bin_size_w; + scalar_t h = hstart + ih * sub_bin_size_h; + // bilinear interpolation + if (w < -0.5 || w > width - 0.5 || h < -0.5 || h > height - 0.5) + { + continue; + } + w = min(max(w, 0.), width - 1.); + h = min(max(h, 0.), height - 1.); + int c = (ctop * group_size + gh) * group_size + gw; + scalar_t val = bilinear_interp(offset_bottom_data + c * height * width, w, h, width, height); + sum += val; + count++; + } + } + top_data[index] = count == 0 ? (scalar_t)(0) : sum / count; + top_count[index] = count; + } +} + +template +__global__ void DeformablePSROIPoolBackwardAccKernel( + const int count, + const scalar_t *top_diff, + const scalar_t *top_count, + const int num_rois, + const scalar_t spatial_scale, + const int channels, + const int height, const int width, + const int pooled_height, const int pooled_width, + const int output_dim, + scalar_t *bottom_data_diff, scalar_t *bottom_trans_diff, + const scalar_t *bottom_data, + const scalar_t *bottom_rois, + const scalar_t *bottom_trans, + const int no_trans, + const scalar_t trans_std, + const int sample_per_part, + const int group_size, + const int part_size, + const int num_classes, + const int channels_each_class) +{ + CUDA_KERNEL_LOOP(index, count) + { + // The output is in order (n, ctop, ph, pw) + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int ctop = (index / pooled_width / pooled_height) % output_dim; + int n = index / pooled_width / pooled_height / output_dim; + + // [start, end) interval for spatial sampling + const scalar_t *offset_bottom_rois = bottom_rois + n * 5; + int roi_batch_ind = offset_bottom_rois[0]; + scalar_t roi_start_w = (scalar_t)(round(offset_bottom_rois[1])) * spatial_scale - 0.5; + scalar_t roi_start_h = (scalar_t)(round(offset_bottom_rois[2])) * spatial_scale - 0.5; + scalar_t roi_end_w = (scalar_t)(round(offset_bottom_rois[3]) + 1.) * spatial_scale - 0.5; + scalar_t roi_end_h = (scalar_t)(round(offset_bottom_rois[4]) + 1.) * spatial_scale - 0.5; + + // Force too small ROIs to be 1x1 + scalar_t roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0 + scalar_t roi_height = max(roi_end_h - roi_start_h, 0.1); + + // Compute w and h at bottom + scalar_t bin_size_h = roi_height / (scalar_t)(pooled_height); + scalar_t bin_size_w = roi_width / (scalar_t)(pooled_width); + + scalar_t sub_bin_size_h = bin_size_h / (scalar_t)(sample_per_part); + scalar_t sub_bin_size_w = bin_size_w / (scalar_t)(sample_per_part); + + int part_h = floor((scalar_t)(ph) / pooled_height * part_size); + int part_w = floor((scalar_t)(pw) / pooled_width * part_size); + int class_id = ctop / channels_each_class; + scalar_t trans_x = no_trans ? (scalar_t)(0) : bottom_trans[(((n * num_classes + class_id) * 2) * part_size + part_h) * part_size + part_w] * (scalar_t)trans_std; + scalar_t trans_y = no_trans ? (scalar_t)(0) : bottom_trans[(((n * num_classes + class_id) * 2 + 1) * part_size + part_h) * part_size + part_w] * (scalar_t)trans_std; + + scalar_t wstart = (scalar_t)(pw)*bin_size_w + roi_start_w; + wstart += trans_x * roi_width; + scalar_t hstart = (scalar_t)(ph)*bin_size_h + roi_start_h; + hstart += trans_y * roi_height; + + if (top_count[index] <= 0) + { + continue; + } + scalar_t diff_val = top_diff[index] / top_count[index]; + const scalar_t *offset_bottom_data = bottom_data + roi_batch_ind * channels * height * width; + scalar_t *offset_bottom_data_diff = bottom_data_diff + roi_batch_ind * channels * height * width; + int gw = floor((scalar_t)(pw)*group_size / pooled_width); + int gh = floor((scalar_t)(ph)*group_size / pooled_height); + gw = min(max(gw, 0), group_size - 1); + gh = min(max(gh, 0), group_size - 1); + + for (int ih = 0; ih < sample_per_part; ih++) + { + for (int iw = 0; iw < sample_per_part; iw++) + { + scalar_t w = wstart + iw * sub_bin_size_w; + scalar_t h = hstart + ih * sub_bin_size_h; + // bilinear interpolation + if (w < -0.5 || w > width - 0.5 || h < -0.5 || h > height - 0.5) + { + continue; + } + w = min(max(w, 0.), width - 1.); + h = min(max(h, 0.), height - 1.); + int c = (ctop * group_size + gh) * group_size + gw; + // backward on feature + int x0 = floor(w); + int x1 = ceil(w); + int y0 = floor(h); + int y1 = ceil(h); + scalar_t dist_x = w - x0, dist_y = h - y0; + scalar_t q00 = (1 - dist_x) * (1 - dist_y); + scalar_t q01 = (1 - dist_x) * dist_y; + scalar_t q10 = dist_x * (1 - dist_y); + scalar_t q11 = dist_x * dist_y; + int bottom_index_base = c * height * width; + atomicAdd(offset_bottom_data_diff + bottom_index_base + y0 * width + x0, q00 * diff_val); + atomicAdd(offset_bottom_data_diff + bottom_index_base + y1 * width + x0, q01 * diff_val); + atomicAdd(offset_bottom_data_diff + bottom_index_base + y0 * width + x1, q10 * diff_val); + atomicAdd(offset_bottom_data_diff + bottom_index_base + y1 * width + x1, q11 * diff_val); + + if (no_trans) + { + continue; + } + scalar_t U00 = offset_bottom_data[bottom_index_base + y0 * width + x0]; + scalar_t U01 = offset_bottom_data[bottom_index_base + y1 * width + x0]; + scalar_t U10 = offset_bottom_data[bottom_index_base + y0 * width + x1]; + scalar_t U11 = offset_bottom_data[bottom_index_base + y1 * width + x1]; + scalar_t diff_x = (U11 * dist_y + U10 * (1 - dist_y) - U01 * dist_y - U00 * (1 - dist_y)) * trans_std * diff_val; + diff_x *= roi_width; + scalar_t diff_y = (U11 * dist_x + U01 * (1 - dist_x) - U10 * dist_x - U00 * (1 - dist_x)) * trans_std * diff_val; + diff_y *= roi_height; + + atomicAdd(bottom_trans_diff + (((n * num_classes + class_id) * 2) * part_size + part_h) * part_size + part_w, diff_x); + atomicAdd(bottom_trans_diff + (((n * num_classes + class_id) * 2 + 1) * part_size + part_h) * part_size + part_w, diff_y); + } + } + } +} + +void DeformablePSROIPoolForward(const at::Tensor data, + const at::Tensor bbox, + const at::Tensor trans, + at::Tensor out, + at::Tensor top_count, + const int batch, + const int channels, + const int height, + const int width, + const int num_bbox, + const int channels_trans, + const int no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) +{ + const int pooled_height = pooled_size; + const int pooled_width = pooled_size; + const int count = num_bbox * output_dim * pooled_height * pooled_width; + const int num_classes = no_trans ? 1 : channels_trans / 2; + const int channels_each_class = no_trans ? output_dim : output_dim / num_classes; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data.type(), "deformable_psroi_pool_forward", ([&] { + const scalar_t *bottom_data = data.data(); + const scalar_t *bottom_rois = bbox.data(); + const scalar_t *bottom_trans = no_trans ? NULL : trans.data(); + scalar_t *top_data = out.data(); + scalar_t *top_count_data = top_count.data(); + + DeformablePSROIPoolForwardKernel<<>>( + count, bottom_data, (scalar_t)spatial_scale, channels, height, width, pooled_height, pooled_width, + bottom_rois, bottom_trans, no_trans, (scalar_t)trans_std, sample_per_part, output_dim, + group_size, part_size, num_classes, channels_each_class, top_data, top_count_data); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in DeformablePSROIPoolForward: %s\n", cudaGetErrorString(err)); + } +} + +void DeformablePSROIPoolBackwardAcc(const at::Tensor out_grad, + const at::Tensor data, + const at::Tensor bbox, + const at::Tensor trans, + const at::Tensor top_count, + at::Tensor in_grad, + at::Tensor trans_grad, + const int batch, + const int channels, + const int height, + const int width, + const int num_bbox, + const int channels_trans, + const int no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) +{ + // LOG(INFO) << "DeformablePSROIPoolBackward"; + const int num_rois = num_bbox; + const int pooled_height = pooled_size; + const int pooled_width = pooled_size; + const int count = num_bbox * output_dim * pooled_height * pooled_width; + const int num_classes = no_trans ? 1 : channels_trans / 2; + const int channels_each_class = no_trans ? output_dim : output_dim / num_classes; + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + out_grad.type(), "deformable_psroi_pool_backward_acc", ([&] { + const scalar_t *top_diff = out_grad.data(); + const scalar_t *bottom_data = data.data(); + const scalar_t *bottom_rois = bbox.data(); + const scalar_t *bottom_trans = no_trans ? NULL : trans.data(); + scalar_t *bottom_data_diff = in_grad.data(); + scalar_t *bottom_trans_diff = no_trans ? NULL : trans_grad.data(); + const scalar_t *top_count_data = top_count.data(); + + DeformablePSROIPoolBackwardAccKernel<<>>( + count, top_diff, top_count_data, num_rois, (scalar_t)spatial_scale, channels, height, width, + pooled_height, pooled_width, output_dim, bottom_data_diff, bottom_trans_diff, + bottom_data, bottom_rois, bottom_trans, no_trans, (scalar_t)trans_std, sample_per_part, + group_size, part_size, num_classes, channels_each_class); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in DeformablePSROIPoolForward: %s\n", cudaGetErrorString(err)); + } +} \ No newline at end of file diff --git a/maskrcnn_benchmark/csrc/cuda/vision.h b/maskrcnn_benchmark/csrc/cuda/vision.h index 6d9f8871f..32d3c6956 100644 --- a/maskrcnn_benchmark/csrc/cuda/vision.h +++ b/maskrcnn_benchmark/csrc/cuda/vision.h @@ -58,6 +58,59 @@ at::Tensor ROIPool_backward_cuda(const at::Tensor& grad, at::Tensor nms_cuda(const at::Tensor boxes, float nms_overlap_thresh); +int deform_conv_forward_cuda(at::Tensor input, at::Tensor weight, + at::Tensor offset, at::Tensor output, + at::Tensor columns, at::Tensor ones, int kW, + int kH, int dW, int dH, int padW, int padH, + int dilationW, int dilationH, int group, + int deformable_group, int im2col_step); + +int deform_conv_backward_input_cuda(at::Tensor input, at::Tensor offset, + at::Tensor gradOutput, at::Tensor gradInput, + at::Tensor gradOffset, at::Tensor weight, + at::Tensor columns, int kW, int kH, int dW, + int dH, int padW, int padH, int dilationW, + int dilationH, int group, + int deformable_group, int im2col_step); + +int deform_conv_backward_parameters_cuda( + at::Tensor input, at::Tensor offset, at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, at::Tensor ones, int kW, int kH, int dW, int dH, + int padW, int padH, int dilationW, int dilationH, int group, + int deformable_group, float scale, int im2col_step); + +void modulated_deform_conv_cuda_forward( + at::Tensor input, at::Tensor weight, at::Tensor bias, at::Tensor ones, + at::Tensor offset, at::Tensor mask, at::Tensor output, at::Tensor columns, + int kernel_h, int kernel_w, const int stride_h, const int stride_w, + const int pad_h, const int pad_w, const int dilation_h, + const int dilation_w, const int group, const int deformable_group, + const bool with_bias); + +void modulated_deform_conv_cuda_backward( + at::Tensor input, at::Tensor weight, at::Tensor bias, at::Tensor ones, + at::Tensor offset, at::Tensor mask, at::Tensor columns, + at::Tensor grad_input, at::Tensor grad_weight, at::Tensor grad_bias, + at::Tensor grad_offset, at::Tensor grad_mask, at::Tensor grad_output, + int kernel_h, int kernel_w, int stride_h, int stride_w, int pad_h, + int pad_w, int dilation_h, int dilation_w, int group, int deformable_group, + const bool with_bias); + +void deform_psroi_pooling_cuda_forward( + at::Tensor input, at::Tensor bbox, at::Tensor trans, at::Tensor out, + at::Tensor top_count, const int no_trans, const float spatial_scale, + const int output_dim, const int group_size, const int pooled_size, + const int part_size, const int sample_per_part, const float trans_std); + +void deform_psroi_pooling_cuda_backward( + at::Tensor out_grad, at::Tensor input, at::Tensor bbox, at::Tensor trans, + at::Tensor top_count, at::Tensor input_grad, at::Tensor trans_grad, + const int no_trans, const float spatial_scale, const int output_dim, + const int group_size, const int pooled_size, const int part_size, + const int sample_per_part, const float trans_std); + + at::Tensor compute_flow_cuda(const at::Tensor& boxes, const int height, const int width); diff --git a/maskrcnn_benchmark/csrc/deform_conv.h b/maskrcnn_benchmark/csrc/deform_conv.h new file mode 100644 index 000000000..a5930e390 --- /dev/null +++ b/maskrcnn_benchmark/csrc/deform_conv.h @@ -0,0 +1,191 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +#pragma once +#include "cpu/vision.h" + +#ifdef WITH_CUDA +#include "cuda/vision.h" +#endif + + +// Interface for Python +int deform_conv_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return deform_conv_forward_cuda( + input, weight, offset, output, columns, ones, + kW, kH, dW, dH, padW, padH, dilationW, dilationH, + group, deformable_group, im2col_step + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + + +int deform_conv_backward_input( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return deform_conv_backward_input_cuda( + input, offset, gradOutput, gradInput, gradOffset, weight, columns, + kW, kH, dW, dH, padW, padH, dilationW, dilationH, + group, deformable_group, im2col_step + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + + +int deform_conv_backward_parameters( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return deform_conv_backward_parameters_cuda( + input, offset, gradOutput, gradWeight, columns, ones, + kW, kH, dW, dH, padW, padH, dilationW, dilationH, + group, deformable_group, scale, im2col_step + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + + +void modulated_deform_conv_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return modulated_deform_conv_cuda_forward( + input, weight, bias, ones, offset, mask, output, columns, + kernel_h, kernel_w, stride_h, stride_w, + pad_h, pad_w, dilation_h, dilation_w, + group, deformable_group, with_bias + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + + +void modulated_deform_conv_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return modulated_deform_conv_cuda_backward( + input, weight, bias, ones, offset, mask, columns, + grad_input, grad_weight, grad_bias, grad_offset, grad_mask, grad_output, + kernel_h, kernel_w, stride_h, stride_w, pad_h, pad_w, dilation_h, dilation_w, + group, deformable_group, with_bias + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} \ No newline at end of file diff --git a/maskrcnn_benchmark/csrc/deform_pool.h b/maskrcnn_benchmark/csrc/deform_pool.h new file mode 100644 index 000000000..234223809 --- /dev/null +++ b/maskrcnn_benchmark/csrc/deform_pool.h @@ -0,0 +1,70 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +#pragma once +#include "cpu/vision.h" + +#ifdef WITH_CUDA +#include "cuda/vision.h" +#endif + + +// Interface for Python +void deform_psroi_pooling_forward( + at::Tensor input, + at::Tensor bbox, + at::Tensor trans, + at::Tensor out, + at::Tensor top_count, + const int no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return deform_psroi_pooling_cuda_forward( + input, bbox, trans, out, top_count, + no_trans, spatial_scale, output_dim, group_size, + pooled_size, part_size, sample_per_part, trans_std + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + + +void deform_psroi_pooling_backward( + at::Tensor out_grad, + at::Tensor input, + at::Tensor bbox, + at::Tensor trans, + at::Tensor top_count, + at::Tensor input_grad, + at::Tensor trans_grad, + const int no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) +{ + if (input.type().is_cuda()) { +#ifdef WITH_CUDA + return deform_psroi_pooling_cuda_backward( + out_grad, input, bbox, trans, top_count, input_grad, trans_grad, + no_trans, spatial_scale, output_dim, group_size, pooled_size, + part_size, sample_per_part, trans_std + ); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} diff --git a/maskrcnn_benchmark/csrc/vision.cpp b/maskrcnn_benchmark/csrc/vision.cpp index 8234f43b1..30971995d 100644 --- a/maskrcnn_benchmark/csrc/vision.cpp +++ b/maskrcnn_benchmark/csrc/vision.cpp @@ -3,6 +3,8 @@ #include "ROIAlign.h" #include "ROIPool.h" #include "SigmoidFocalLoss.h" +#include "deform_conv.h" +#include "deform_pool.h" PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("nms", &nms, "non-maximum suppression"); @@ -12,4 +14,12 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("roi_pool_backward", &ROIPool_backward, "ROIPool_backward"); m.def("sigmoid_focalloss_forward", &SigmoidFocalLoss_forward, "SigmoidFocalLoss_forward"); m.def("sigmoid_focalloss_backward", &SigmoidFocalLoss_backward, "SigmoidFocalLoss_backward"); -} + // dcn-v2 + m.def("deform_conv_forward", &deform_conv_forward, "deform_conv_forward"); + m.def("deform_conv_backward_input", &deform_conv_backward_input, "deform_conv_backward_input"); + m.def("deform_conv_backward_parameters", &deform_conv_backward_parameters, "deform_conv_backward_parameters"); + m.def("modulated_deform_conv_forward", &modulated_deform_conv_forward, "modulated_deform_conv_forward"); + m.def("modulated_deform_conv_backward", &modulated_deform_conv_backward, "modulated_deform_conv_backward"); + m.def("deform_psroi_pooling_forward", &deform_psroi_pooling_forward, "deform_psroi_pooling_forward"); + m.def("deform_psroi_pooling_backward", &deform_psroi_pooling_backward, "deform_psroi_pooling_backward"); +} \ No newline at end of file diff --git a/maskrcnn_benchmark/layers/__init__.py b/maskrcnn_benchmark/layers/__init__.py index bab50abae..0e1ccc513 100644 --- a/maskrcnn_benchmark/layers/__init__.py +++ b/maskrcnn_benchmark/layers/__init__.py @@ -3,6 +3,7 @@ from .batch_norm import FrozenBatchNorm2d from .misc import Conv2d +from .misc import DFConv2d from .misc import ConvTranspose2d from .misc import BatchNorm2d from .misc import interpolate @@ -13,9 +14,34 @@ from .roi_pool import roi_pool from .smooth_l1_loss import smooth_l1_loss from .sigmoid_focal_loss import SigmoidFocalLoss +from .dcn.deform_conv_func import deform_conv, modulated_deform_conv +from .dcn.deform_conv_module import DeformConv, ModulatedDeformConv, ModulatedDeformConvPack +from .dcn.deform_pool_func import deform_roi_pooling +from .dcn.deform_pool_module import DeformRoIPooling, DeformRoIPoolingPack, ModulatedDeformRoIPoolingPack -__all__ = ["nms", "roi_align", "ROIAlign", "roi_pool", "ROIPool", - "smooth_l1_loss", "Conv2d", "ConvTranspose2d", "interpolate", - "BatchNorm2d", "FrozenBatchNorm2d", "SigmoidFocalLoss" - ] + +__all__ = [ + "nms", + "roi_align", + "ROIAlign", + "roi_pool", + "ROIPool", + "smooth_l1_loss", + "Conv2d", + "DFConv2d", + "ConvTranspose2d", + "interpolate", + "BatchNorm2d", + "FrozenBatchNorm2d", + "SigmoidFocalLoss", + 'deform_conv', + 'modulated_deform_conv', + 'DeformConv', + 'ModulatedDeformConv', + 'ModulatedDeformConvPack', + 'deform_roi_pooling', + 'DeformRoIPooling', + 'DeformRoIPoolingPack', + 'ModulatedDeformRoIPoolingPack', +] diff --git a/maskrcnn_benchmark/layers/dcn/__init__.py b/maskrcnn_benchmark/layers/dcn/__init__.py new file mode 100644 index 000000000..22fe18ff3 --- /dev/null +++ b/maskrcnn_benchmark/layers/dcn/__init__.py @@ -0,0 +1,3 @@ +# +# Copied From [mmdetection](https://github.com/open-mmlab/mmdetection/tree/master/mmdet/ops/dcn) +# \ No newline at end of file diff --git a/maskrcnn_benchmark/layers/dcn/deform_conv_func.py b/maskrcnn_benchmark/layers/dcn/deform_conv_func.py new file mode 100644 index 000000000..ddc92bb0c --- /dev/null +++ b/maskrcnn_benchmark/layers/dcn/deform_conv_func.py @@ -0,0 +1,259 @@ +import torch +from torch.autograd import Function +from torch.nn.modules.utils import _pair + +from maskrcnn_benchmark import _C + + +class DeformConvFunction(Function): + + @staticmethod + def forward( + ctx, + input, + offset, + weight, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + im2col_step=64 + ): + if input is not None and input.dim() != 4: + raise ValueError( + "Expected 4D tensor as input, got {}D tensor instead.".format( + input.dim())) + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + + ctx.save_for_backward(input, offset, weight) + + output = input.new_empty( + DeformConvFunction._output_size(input, weight, ctx.padding, + ctx.dilation, ctx.stride)) + + ctx.bufs_ = [input.new_empty(0), input.new_empty(0)] # columns, ones + + if not input.is_cuda: + raise NotImplementedError + else: + cur_im2col_step = min(ctx.im2col_step, input.shape[0]) + assert (input.shape[0] % + cur_im2col_step) == 0, 'im2col step must divide batchsize' + _C.deform_conv_forward( + input, + weight, + offset, + output, + ctx.bufs_[0], + ctx.bufs_[1], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + cur_im2col_step + ) + return output + + @staticmethod + def backward(ctx, grad_output): + input, offset, weight = ctx.saved_tensors + + grad_input = grad_offset = grad_weight = None + + if not grad_output.is_cuda: + raise NotImplementedError + else: + cur_im2col_step = min(ctx.im2col_step, input.shape[0]) + assert (input.shape[0] % + cur_im2col_step) == 0, 'im2col step must divide batchsize' + + if ctx.needs_input_grad[0] or ctx.needs_input_grad[1]: + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + _C.deform_conv_backward_input( + input, + offset, + grad_output, + grad_input, + grad_offset, + weight, + ctx.bufs_[0], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + cur_im2col_step + ) + + if ctx.needs_input_grad[2]: + grad_weight = torch.zeros_like(weight) + _C.deform_conv_backward_parameters( + input, + offset, + grad_output, + grad_weight, + ctx.bufs_[0], + ctx.bufs_[1], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + 1, + cur_im2col_step + ) + + return (grad_input, grad_offset, grad_weight, None, None, None, None, None) + + @staticmethod + def _output_size(input, weight, padding, dilation, stride): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = padding[d] + kernel = dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1, ) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + "convolution input is too small (output would be {})".format( + 'x'.join(map(str, output_size)))) + return output_size + + +class ModulatedDeformConvFunction(Function): + + @staticmethod + def forward( + ctx, + input, + offset, + mask, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1 + ): + ctx.stride = stride + ctx.padding = padding + ctx.dilation = dilation + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.with_bias = bias is not None + if not ctx.with_bias: + bias = input.new_empty(1) # fake tensor + if not input.is_cuda: + raise NotImplementedError + if weight.requires_grad or mask.requires_grad or offset.requires_grad \ + or input.requires_grad: + ctx.save_for_backward(input, offset, mask, weight, bias) + output = input.new_empty( + ModulatedDeformConvFunction._infer_shape(ctx, input, weight)) + ctx._bufs = [input.new_empty(0), input.new_empty(0)] + _C.modulated_deform_conv_forward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + output, + ctx._bufs[1], + weight.shape[2], + weight.shape[3], + ctx.stride, + ctx.stride, + ctx.padding, + ctx.padding, + ctx.dilation, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, + ctx.with_bias + ) + return output + + @staticmethod + def backward(ctx, grad_output): + if not grad_output.is_cuda: + raise NotImplementedError + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + grad_mask = torch.zeros_like(mask) + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(bias) + _C.modulated_deform_conv_backward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + ctx._bufs[1], + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + weight.shape[2], + weight.shape[3], + ctx.stride, + ctx.stride, + ctx.padding, + ctx.padding, + ctx.dilation, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, + ctx.with_bias + ) + if not ctx.with_bias: + grad_bias = None + + return (grad_input, grad_offset, grad_mask, grad_weight, grad_bias, + None, None, None, None, None) + + @staticmethod + def _infer_shape(ctx, input, weight): + n = input.size(0) + channels_out = weight.size(0) + height, width = input.shape[2:4] + kernel_h, kernel_w = weight.shape[2:4] + height_out = (height + 2 * ctx.padding - + (ctx.dilation * (kernel_h - 1) + 1)) // ctx.stride + 1 + width_out = (width + 2 * ctx.padding - + (ctx.dilation * (kernel_w - 1) + 1)) // ctx.stride + 1 + return n, channels_out, height_out, width_out + + +deform_conv = DeformConvFunction.apply +modulated_deform_conv = ModulatedDeformConvFunction.apply diff --git a/maskrcnn_benchmark/layers/dcn/deform_conv_module.py b/maskrcnn_benchmark/layers/dcn/deform_conv_module.py new file mode 100644 index 000000000..e6b58c840 --- /dev/null +++ b/maskrcnn_benchmark/layers/dcn/deform_conv_module.py @@ -0,0 +1,177 @@ +import math + +import torch +import torch.nn as nn +from torch.nn.modules.utils import _pair + +from .deform_conv_func import deform_conv, modulated_deform_conv + + +class DeformConv(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=False + ): + assert not bias + super(DeformConv, self).__init__() + self.with_bias = bias + + assert in_channels % groups == 0, \ + 'in_channels {} cannot be divisible by groups {}'.format( + in_channels, groups) + assert out_channels % groups == 0, \ + 'out_channels {} cannot be divisible by groups {}'.format( + out_channels, groups) + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // self.groups, + *self.kernel_size)) + + self.reset_parameters() + + def reset_parameters(self): + n = self.in_channels + for k in self.kernel_size: + n *= k + stdv = 1. / math.sqrt(n) + self.weight.data.uniform_(-stdv, stdv) + + def forward(self, input, offset): + return deform_conv(input, offset, self.weight, self.stride, + self.padding, self.dilation, self.groups, + self.deformable_groups) + + def __repr__(self): + return "".join([ + "{}(".format(self.__class__.__name__), + "in_channels={}, ".format(self.in_channels), + "out_channels={}, ".format(self.out_channels), + "kernel_size={}, ".format(self.kernel_size), + "stride={}, ".format(self.stride), + "dilation={}, ".format(self.dilation), + "padding={}, ".format(self.padding), + "groups={}, ".format(self.groups), + "deformable_groups={}, ".format(self.deformable_groups), + "bias={})".format(self.with_bias), + ]) + + +class ModulatedDeformConv(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=True + ): + super(ModulatedDeformConv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + self.deformable_groups = deformable_groups + self.with_bias = bias + + self.weight = nn.Parameter(torch.Tensor( + out_channels, + in_channels // groups, + *self.kernel_size + )) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.register_parameter('bias', None) + self.reset_parameters() + + def reset_parameters(self): + n = self.in_channels + for k in self.kernel_size: + n *= k + stdv = 1. / math.sqrt(n) + self.weight.data.uniform_(-stdv, stdv) + if self.bias is not None: + self.bias.data.zero_() + + def forward(self, input, offset, mask): + return modulated_deform_conv( + input, offset, mask, self.weight, self.bias, self.stride, + self.padding, self.dilation, self.groups, self.deformable_groups) + + def __repr__(self): + return "".join([ + "{}(".format(self.__class__.__name__), + "in_channels={}, ".format(self.in_channels), + "out_channels={}, ".format(self.out_channels), + "kernel_size={}, ".format(self.kernel_size), + "stride={}, ".format(self.stride), + "dilation={}, ".format(self.dilation), + "padding={}, ".format(self.padding), + "groups={}, ".format(self.groups), + "deformable_groups={}, ".format(self.deformable_groups), + "bias={})".format(self.with_bias), + ]) + +class ModulatedDeformConvPack(ModulatedDeformConv): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=True): + super(ModulatedDeformConvPack, self).__init__( + in_channels, out_channels, kernel_size, stride, padding, dilation, + groups, deformable_groups, bias) + + self.conv_offset_mask = nn.Conv2d( + self.in_channels // self.groups, + self.deformable_groups * 3 * self.kernel_size[0] * + self.kernel_size[1], + kernel_size=self.kernel_size, + stride=_pair(self.stride), + padding=_pair(self.padding), + bias=True) + self.init_offset() + + def init_offset(self): + self.conv_offset_mask.weight.data.zero_() + self.conv_offset_mask.bias.data.zero_() + + def forward(self, input): + out = self.conv_offset_mask(input) + o1, o2, mask = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask = torch.sigmoid(mask) + return modulated_deform_conv( + input, offset, mask, self.weight, self.bias, self.stride, + self.padding, self.dilation, self.groups, self.deformable_groups) diff --git a/maskrcnn_benchmark/layers/dcn/deform_pool_func.py b/maskrcnn_benchmark/layers/dcn/deform_pool_func.py new file mode 100644 index 000000000..f18fdd4cb --- /dev/null +++ b/maskrcnn_benchmark/layers/dcn/deform_pool_func.py @@ -0,0 +1,93 @@ +import torch +from torch.autograd import Function + +from maskrcnn_benchmark import _C + + +class DeformRoIPoolingFunction(Function): + + @staticmethod + def forward( + ctx, + data, + rois, + offset, + spatial_scale, + out_size, + out_channels, + no_trans, + group_size=1, + part_size=None, + sample_per_part=4, + trans_std=.0 + ): + ctx.spatial_scale = spatial_scale + ctx.out_size = out_size + ctx.out_channels = out_channels + ctx.no_trans = no_trans + ctx.group_size = group_size + ctx.part_size = out_size if part_size is None else part_size + ctx.sample_per_part = sample_per_part + ctx.trans_std = trans_std + + assert 0.0 <= ctx.trans_std <= 1.0 + if not data.is_cuda: + raise NotImplementedError + + n = rois.shape[0] + output = data.new_empty(n, out_channels, out_size, out_size) + output_count = data.new_empty(n, out_channels, out_size, out_size) + _C.deform_psroi_pooling_forward( + data, + rois, + offset, + output, + output_count, + ctx.no_trans, + ctx.spatial_scale, + ctx.out_channels, + ctx.group_size, + ctx.out_size, + ctx.part_size, + ctx.sample_per_part, + ctx.trans_std + ) + + if data.requires_grad or rois.requires_grad or offset.requires_grad: + ctx.save_for_backward(data, rois, offset) + ctx.output_count = output_count + + return output + + @staticmethod + def backward(ctx, grad_output): + if not grad_output.is_cuda: + raise NotImplementedError + + data, rois, offset = ctx.saved_tensors + output_count = ctx.output_count + grad_input = torch.zeros_like(data) + grad_rois = None + grad_offset = torch.zeros_like(offset) + + _C.deform_psroi_pooling_backward( + grad_output, + data, + rois, + offset, + output_count, + grad_input, + grad_offset, + ctx.no_trans, + ctx.spatial_scale, + ctx.out_channels, + ctx.group_size, + ctx.out_size, + ctx.part_size, + ctx.sample_per_part, + ctx.trans_std + ) + return (grad_input, grad_rois, grad_offset, None, None, None, None, None, None, None, None) + + +deform_roi_pooling = DeformRoIPoolingFunction.apply diff --git a/maskrcnn_benchmark/layers/dcn/deform_pool_module.py b/maskrcnn_benchmark/layers/dcn/deform_pool_module.py new file mode 100644 index 000000000..bab6c2604 --- /dev/null +++ b/maskrcnn_benchmark/layers/dcn/deform_pool_module.py @@ -0,0 +1,150 @@ +from torch import nn + +from .deform_pool_func import deform_roi_pooling + + +class DeformRoIPooling(nn.Module): + + def __init__(self, + spatial_scale, + out_size, + out_channels, + no_trans, + group_size=1, + part_size=None, + sample_per_part=4, + trans_std=.0): + super(DeformRoIPooling, self).__init__() + self.spatial_scale = spatial_scale + self.out_size = out_size + self.out_channels = out_channels + self.no_trans = no_trans + self.group_size = group_size + self.part_size = out_size if part_size is None else part_size + self.sample_per_part = sample_per_part + self.trans_std = trans_std + + def forward(self, data, rois, offset): + if self.no_trans: + offset = data.new_empty(0) + return deform_roi_pooling( + data, rois, offset, self.spatial_scale, self.out_size, + self.out_channels, self.no_trans, self.group_size, self.part_size, + self.sample_per_part, self.trans_std) + + +class DeformRoIPoolingPack(DeformRoIPooling): + + def __init__(self, + spatial_scale, + out_size, + out_channels, + no_trans, + group_size=1, + part_size=None, + sample_per_part=4, + trans_std=.0, + deform_fc_channels=1024): + super(DeformRoIPoolingPack, + self).__init__(spatial_scale, out_size, out_channels, no_trans, + group_size, part_size, sample_per_part, trans_std) + + self.deform_fc_channels = deform_fc_channels + + if not no_trans: + self.offset_fc = nn.Sequential( + nn.Linear(self.out_size * self.out_size * self.out_channels, + self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.out_size * self.out_size * 2)) + self.offset_fc[-1].weight.data.zero_() + self.offset_fc[-1].bias.data.zero_() + + def forward(self, data, rois): + assert data.size(1) == self.out_channels + if self.no_trans: + offset = data.new_empty(0) + return deform_roi_pooling( + data, rois, offset, self.spatial_scale, self.out_size, + self.out_channels, self.no_trans, self.group_size, + self.part_size, self.sample_per_part, self.trans_std) + else: + n = rois.shape[0] + offset = data.new_empty(0) + x = deform_roi_pooling(data, rois, offset, self.spatial_scale, + self.out_size, self.out_channels, True, + self.group_size, self.part_size, + self.sample_per_part, self.trans_std) + offset = self.offset_fc(x.view(n, -1)) + offset = offset.view(n, 2, self.out_size, self.out_size) + return deform_roi_pooling( + data, rois, offset, self.spatial_scale, self.out_size, + self.out_channels, self.no_trans, self.group_size, + self.part_size, self.sample_per_part, self.trans_std) + + +class ModulatedDeformRoIPoolingPack(DeformRoIPooling): + + def __init__(self, + spatial_scale, + out_size, + out_channels, + no_trans, + group_size=1, + part_size=None, + sample_per_part=4, + trans_std=.0, + deform_fc_channels=1024): + super(ModulatedDeformRoIPoolingPack, self).__init__( + spatial_scale, out_size, out_channels, no_trans, group_size, + part_size, sample_per_part, trans_std) + + self.deform_fc_channels = deform_fc_channels + + if not no_trans: + self.offset_fc = nn.Sequential( + nn.Linear(self.out_size * self.out_size * self.out_channels, + self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.out_size * self.out_size * 2)) + self.offset_fc[-1].weight.data.zero_() + self.offset_fc[-1].bias.data.zero_() + self.mask_fc = nn.Sequential( + nn.Linear(self.out_size * self.out_size * self.out_channels, + self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.out_size * self.out_size * 1), + nn.Sigmoid()) + self.mask_fc[2].weight.data.zero_() + self.mask_fc[2].bias.data.zero_() + + def forward(self, data, rois): + assert data.size(1) == self.out_channels + if self.no_trans: + offset = data.new_empty(0) + return deform_roi_pooling( + data, rois, offset, self.spatial_scale, self.out_size, + self.out_channels, self.no_trans, self.group_size, + self.part_size, self.sample_per_part, self.trans_std) + else: + n = rois.shape[0] + offset = data.new_empty(0) + x = deform_roi_pooling(data, rois, offset, self.spatial_scale, + self.out_size, self.out_channels, True, + self.group_size, self.part_size, + self.sample_per_part, self.trans_std) + offset = self.offset_fc(x.view(n, -1)) + offset = offset.view(n, 2, self.out_size, self.out_size) + mask = self.mask_fc(x.view(n, -1)) + mask = mask.view(n, 1, self.out_size, self.out_size) + return deform_roi_pooling( + data, rois, offset, self.spatial_scale, self.out_size, + self.out_channels, self.no_trans, self.group_size, + self.part_size, self.sample_per_part, self.trans_std) * mask diff --git a/maskrcnn_benchmark/layers/misc.py b/maskrcnn_benchmark/layers/misc.py index a8cf1c680..b3bca0db5 100644 --- a/maskrcnn_benchmark/layers/misc.py +++ b/maskrcnn_benchmark/layers/misc.py @@ -11,6 +11,7 @@ import math import torch +from torch import nn from torch.nn.modules.utils import _ntuple @@ -108,3 +109,86 @@ def _output_size(dim): output_shape = tuple(_output_size(2)) output_shape = input.shape[:-2] + output_shape return _NewEmptyTensorOp.apply(input, output_shape) + + +class DFConv2d(nn.Module): + """Deformable convolutional layer""" + def __init__( + self, + in_channels, + out_channels, + with_modulated_dcn=True, + kernel_size=3, + stride=1, + groups=1, + dilation=1, + deformable_groups=1, + bias=False + ): + super(DFConv2d, self).__init__() + if isinstance(kernel_size, (list, tuple)): + assert len(kernel_size) == 2 + offset_base_channels = kernel_size[0] * kernel_size[1] + else: + offset_base_channels = kernel_size * kernel_size + if with_modulated_dcn: + from maskrcnn_benchmark.layers import ModulatedDeformConv + offset_channels = offset_base_channels * 3 #default: 27 + conv_block = ModulatedDeformConv + else: + from maskrcnn_benchmark.layers import DeformConv + offset_channels = offset_base_channels * 2 #default: 18 + conv_block = DeformConv + self.offset = Conv2d( + in_channels, + deformable_groups * offset_channels, + kernel_size=kernel_size, + stride= stride, + padding= dilation, + groups=1, + dilation=dilation + ) + for l in [self.offset,]: + nn.init.kaiming_uniform_(l.weight, a=1) + torch.nn.init.constant_(l.bias, 0.) + self.conv = conv_block( + in_channels, + out_channels, + kernel_size=kernel_size, + stride= stride, + padding=dilation, + dilation=dilation, + groups=groups, + deformable_groups=deformable_groups, + bias=bias + ) + self.with_modulated_dcn = with_modulated_dcn + self.kernel_size = kernel_size + self.stride = stride + self.padding = dilation + self.dilation = dilation + + def forward(self, x): + if x.numel() > 0: + if not self.with_modulated_dcn: + offset = self.offset(x) + x = self.conv(x, offset) + else: + offset_mask = self.offset(x) + offset = offset_mask[:, :18, :, :] + mask = offset_mask[:, -9:, :, :].sigmoid() + x = self.conv(x, offset, mask) + return x + # get output shape + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // d + 1 + for i, p, di, k, d in zip( + x.shape[-2:], + self.padding, + self.dilation, + self.kernel_size, + self.stride + ) + ] + output_shape = [x.shape[0], self.conv.weight.shape[0]] + output_shape + return _NewEmptyTensorOp.apply(x, output_shape) diff --git a/maskrcnn_benchmark/modeling/backbone/resnet.py b/maskrcnn_benchmark/modeling/backbone/resnet.py index aaa438026..fc02dc1e8 100644 --- a/maskrcnn_benchmark/modeling/backbone/resnet.py +++ b/maskrcnn_benchmark/modeling/backbone/resnet.py @@ -24,6 +24,7 @@ from maskrcnn_benchmark.layers import FrozenBatchNorm2d from maskrcnn_benchmark.layers import Conv2d +from maskrcnn_benchmark.layers import DFConv2d from maskrcnn_benchmark.modeling.make_layers import group_norm from maskrcnn_benchmark.utils.registry import Registry @@ -106,6 +107,7 @@ def __init__(self, cfg): stage2_relative_factor = 2 ** (stage_spec.index - 1) bottleneck_channels = stage2_bottleneck_channels * stage2_relative_factor out_channels = stage2_out_channels * stage2_relative_factor + stage_with_dcn = cfg.MODEL.RESNETS.STAGE_WITH_DCN[stage_spec.index -1] module = _make_stage( transformation_module, in_channels, @@ -115,6 +117,11 @@ def __init__(self, cfg): num_groups, cfg.MODEL.RESNETS.STRIDE_IN_1X1, first_stride=int(stage_spec.index > 1) + 1, + dcn_config={ + "stage_with_dcn": stage_with_dcn, + "with_modulated_dcn": cfg.MODEL.RESNETS.WITH_MODULATED_DCN, + "deformable_groups": cfg.MODEL.RESNETS.DEFORMABLE_GROUPS, + } ) in_channels = out_channels self.add_module(name, module) @@ -155,7 +162,8 @@ def __init__( stride_in_1x1=True, stride_init=None, res2_out_channels=256, - dilation=1 + dilation=1, + dcn_config={} ): super(ResNetHead, self).__init__() @@ -182,7 +190,8 @@ def __init__( num_groups, stride_in_1x1, first_stride=stride, - dilation=dilation + dilation=dilation, + dcn_config=dcn_config ) stride = None self.add_module(name, module) @@ -204,7 +213,8 @@ def _make_stage( num_groups, stride_in_1x1, first_stride, - dilation=1 + dilation=1, + dcn_config={} ): blocks = [] stride = first_stride @@ -217,7 +227,8 @@ def _make_stage( num_groups, stride_in_1x1, stride, - dilation=dilation + dilation=dilation, + dcn_config=dcn_config ) ) stride = 1 @@ -235,7 +246,8 @@ def __init__( stride_in_1x1, stride, dilation, - norm_func + norm_func, + dcn_config ): super(Bottleneck, self).__init__() @@ -271,17 +283,34 @@ def __init__( ) self.bn1 = norm_func(bottleneck_channels) # TODO: specify init for the above + with_dcn = dcn_config.get("stage_with_dcn", False) + if with_dcn: + deformable_groups = dcn_config.get("deformable_groups", 1) + with_modulated_dcn = dcn_config.get("with_modulated_dcn", False) + self.conv2 = DFConv2d( + bottleneck_channels, + bottleneck_channels, + with_modulated_dcn=with_modulated_dcn, + kernel_size=3, + stride=stride_3x3, + groups=num_groups, + dilation=dilation, + deformable_groups=deformable_groups, + bias=False + ) + else: + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=dilation, + bias=False, + groups=num_groups, + dilation=dilation + ) + nn.init.kaiming_uniform_(self.conv2.weight, a=1) - self.conv2 = Conv2d( - bottleneck_channels, - bottleneck_channels, - kernel_size=3, - stride=stride_3x3, - padding=dilation, - bias=False, - groups=num_groups, - dilation=dilation - ) self.bn2 = norm_func(bottleneck_channels) self.conv3 = Conv2d( @@ -289,7 +318,7 @@ def __init__( ) self.bn3 = norm_func(out_channels) - for l in [self.conv1, self.conv2, self.conv3,]: + for l in [self.conv1, self.conv3,]: nn.init.kaiming_uniform_(l.weight, a=1) def forward(self, x): @@ -346,7 +375,8 @@ def __init__( num_groups=1, stride_in_1x1=True, stride=1, - dilation=1 + dilation=1, + dcn_config={} ): super(BottleneckWithFixedBatchNorm, self).__init__( in_channels=in_channels, @@ -356,7 +386,8 @@ def __init__( stride_in_1x1=stride_in_1x1, stride=stride, dilation=dilation, - norm_func=FrozenBatchNorm2d + norm_func=FrozenBatchNorm2d, + dcn_config=dcn_config ) @@ -376,7 +407,8 @@ def __init__( num_groups=1, stride_in_1x1=True, stride=1, - dilation=1 + dilation=1, + dcn_config={} ): super(BottleneckWithGN, self).__init__( in_channels=in_channels, @@ -386,7 +418,8 @@ def __init__( stride_in_1x1=stride_in_1x1, stride=stride, dilation=dilation, - norm_func=group_norm + norm_func=group_norm, + dcn_config=dcn_config ) diff --git a/maskrcnn_benchmark/utils/c2_model_loading.py b/maskrcnn_benchmark/utils/c2_model_loading.py index b1b9996e8..cbf4c050a 100644 --- a/maskrcnn_benchmark/utils/c2_model_loading.py +++ b/maskrcnn_benchmark/utils/c2_model_loading.py @@ -143,6 +143,33 @@ def _load_c2_pickled_weights(file_path): return weights +def _rename_conv_weights_for_deformable_conv_layers(state_dict, cfg): + import re + logger = logging.getLogger(__name__) + logger.info("Remapping conv weights for deformable conv weights") + layer_keys = sorted(state_dict.keys()) + for ix, stage_with_dcn in enumerate(cfg.MODEL.RESNETS.STAGE_WITH_DCN, 1): + if not stage_with_dcn: + continue + for old_key in layer_keys: + pattern = ".*layer{}.*conv2.*".format(ix) + r = re.match(pattern, old_key) + if r is None: + continue + for param in ["weight", "bias"]: + if old_key.find(param) is -1: + continue + new_key = old_key.replace( + "conv2.{}".format(param), "conv2.conv.{}".format(param) + ) + logger.info("pattern: {}, old_key: {}, new_key: {}".format( + pattern, old_key, new_key + )) + state_dict[new_key] = state_dict[old_key] + del state_dict[old_key] + return state_dict + + _C2_STAGE_NAMES = { "R-50": ["1.2", "2.3", "3.5", "4.2"], "R-101": ["1.2", "2.3", "3.22", "4.2"], @@ -168,6 +195,10 @@ def load_resnet_c2_format(cfg, f): arch = arch.replace("-RETINANET", "") stages = _C2_STAGE_NAMES[arch] state_dict = _rename_weights_for_resnet(state_dict, stages) + # *********************************** + # for deformable convolutional layer + state_dict = _rename_conv_weights_for_deformable_conv_layers(state_dict, cfg) + # *********************************** return dict(model=state_dict) From 862347d5383e7982fc3bdf0b5d82c359db3e7dcc Mon Sep 17 00:00:00 2001 From: CoinCheung <867153576@qq.com> Date: Fri, 19 Apr 2019 23:29:30 +0800 Subject: [PATCH 08/52] add color jitter augmentation (#680) * add color jitter augmentation * fix spelling --- maskrcnn_benchmark/config/defaults.py | 6 ++++++ maskrcnn_benchmark/data/transforms/build.py | 7 +++++++ .../data/transforms/transforms.py | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index 23a599ef7..2f5a88009 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -54,6 +54,12 @@ # Convert image to BGR format (for Caffe2 models), in range 0-255 _C.INPUT.TO_BGR255 = True +# Image ColorJitter +_C.INPUT.BRIGHTNESS = 0.0 +_C.INPUT.CONTRAST = 0.0 +_C.INPUT.SATURATION = 0.0 +_C.INPUT.HUE = 0.0 + # ----------------------------------------------------------------------------- # Dataset diff --git a/maskrcnn_benchmark/data/transforms/build.py b/maskrcnn_benchmark/data/transforms/build.py index 8645d4df4..825c3ee42 100644 --- a/maskrcnn_benchmark/data/transforms/build.py +++ b/maskrcnn_benchmark/data/transforms/build.py @@ -16,9 +16,16 @@ def build_transforms(cfg, is_train=True): normalize_transform = T.Normalize( mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr255=to_bgr255 ) + color_jitter = T.ColorJitter( + brightness=cfg.INPUT.BRIGHTNESS, + contrast=cfg.INPUT.CONTRAST, + saturation=cfg.INPUT.SATURATION, + hue=cfg.INPUT.HUE, + ) transform = T.Compose( [ + color_jitter, T.Resize(min_size, max_size), T.RandomHorizontalFlip(flip_prob), T.ToTensor(), diff --git a/maskrcnn_benchmark/data/transforms/transforms.py b/maskrcnn_benchmark/data/transforms/transforms.py index 7e3ebbd6c..1c322f8ba 100644 --- a/maskrcnn_benchmark/data/transforms/transforms.py +++ b/maskrcnn_benchmark/data/transforms/transforms.py @@ -72,6 +72,24 @@ def __call__(self, image, target): return image, target +class ColorJitter(object): + def __init__(self, + brightness=None, + contrast=None, + saturation=None, + hue=None, + ): + self.color_jitter = torchvision.transforms.ColorJitter( + brightness=brightness, + contrast=contrast, + saturation=saturation, + hue=hue,) + + def __call__(self, image, target): + image = self.color_jitter(image) + return image, target + + class ToTensor(object): def __call__(self, image, target): return F.to_tensor(image), target From bf0437923c8ed16623fc9dc0ba3f688d2d9e8cee Mon Sep 17 00:00:00 2001 From: ChenJoya Date: Fri, 19 Apr 2019 23:30:31 +0800 Subject: [PATCH 09/52] proposals from RPN per image during training (#676) * proposals from RPN per image during training * README * Update README for setting FPN_POST_NMS_TOP_N_TRAIN * Update README.md * removing extra space change --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c31451c31..5c68de5f0 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ you'll also need to change the learning rate, the number of iterations and the l Here is an example for Mask R-CNN R-50 FPN with the 1x schedule: ```bash -python tools/train_net.py --config-file "configs/e2e_mask_rcnn_R_50_FPN_1x.yaml" SOLVER.IMS_PER_BATCH 2 SOLVER.BASE_LR 0.0025 SOLVER.MAX_ITER 720000 SOLVER.STEPS "(480000, 640000)" TEST.IMS_PER_BATCH 1 +python tools/train_net.py --config-file "configs/e2e_mask_rcnn_R_50_FPN_1x.yaml" SOLVER.IMS_PER_BATCH 2 SOLVER.BASE_LR 0.0025 SOLVER.MAX_ITER 720000 SOLVER.STEPS "(480000, 640000)" TEST.IMS_PER_BATCH 1 MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN 2000 ``` This follows the [scheduling rules from Detectron.](https://github.com/facebookresearch/Detectron/blob/master/configs/getting_started/tutorial_1gpu_e2e_faster_rcnn_R-50-FPN.yaml#L14-L30) Note that we have multiplied the number of iterations by 8x (as well as the learning rate schedules), @@ -138,6 +138,7 @@ and we have divided the learning rate by 8x. We also changed the batch size during testing, but that is generally not necessary because testing requires much less memory than training. +Furthermore, we set ```MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN 2000``` as the proposals are selected for per the batch rather than per image. The value is calculated by **1000 x images-per-gpu**. Here we have 2 images per GPU, therefore we set the number as 1000 x 2 = 2000. If we have 8 images per GPU, the value should be set as 8000. See [#672](https://github.com/facebookresearch/maskrcnn-benchmark/issues/672) for more details. ### Multi-GPU training We use internally `torch.distributed.launch` in order to launch @@ -147,8 +148,9 @@ process will only use a single GPU. ```bash export NGPUS=8 -python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_benchmark/tools/train_net.py --config-file "path/to/config/file.yaml" +python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_benchmark/tools/train_net.py --config-file "path/to/config/file.yaml" MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN images_per_gpu x 1000 ``` +Note we should set ```MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN``` follow the rule in Single-GPU training. ## Abstractions For more information on some of the main abstractions in our implementation, see [ABSTRACTIONS.md](ABSTRACTIONS.md). From 08fcf12f4b8ef16e5903f859a62307950b886f89 Mon Sep 17 00:00:00 2001 From: Simon Layton Date: Fri, 19 Apr 2019 11:32:59 -0400 Subject: [PATCH 10/52] Initial mixed-precision training (#196) * Initial multi-precision training Adds fp16 support via apex.amp Also switches communication to apex.DistributedDataParallel * Add Apex install to dockerfile * Fixes from @fmassa review Added support to tools/test_net.py SOLVER.MIXED_PRECISION -> DTYPE \in {float32, float16} apex.amp not installed now raises ImportError * Remove extraneous apex DDP import * Move to new amp API --- INSTALL.md | 6 ++++++ docker/Dockerfile | 5 +++++ maskrcnn_benchmark/config/defaults.py | 10 ++++++++++ maskrcnn_benchmark/engine/trainer.py | 6 +++++- maskrcnn_benchmark/layers/batch_norm.py | 7 +++++++ maskrcnn_benchmark/layers/nms.py | 6 +++++- maskrcnn_benchmark/layers/roi_align.py | 3 ++- maskrcnn_benchmark/layers/roi_pool.py | 2 ++ maskrcnn_benchmark/modeling/poolers.py | 2 +- .../modeling/roi_heads/mask_head/inference.py | 5 +++++ tools/test_net.py | 10 ++++++++++ tools/train_net.py | 12 ++++++++++++ 12 files changed, 70 insertions(+), 4 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 4db4b5bb6..c56cc9f2c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -38,6 +38,12 @@ git clone https://github.com/cocodataset/cocoapi.git cd cocoapi/PythonAPI python setup.py build_ext install +# install apex +cd ~github +git clone https://github.com/NVIDIA/apex.git +cd apex +python setup.py install --cuda_ext --cpp_ext + # install PyTorch Detection cd $INSTALL_DIR git clone https://github.com/facebookresearch/maskrcnn-benchmark.git diff --git a/docker/Dockerfile b/docker/Dockerfile index 11a2d370a..4242117db 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,6 +46,11 @@ RUN git clone https://github.com/cocodataset/cocoapi.git \ && cd cocoapi/PythonAPI \ && python setup.py build_ext install +# install apex +RUN git clone https://github.com/NVIDIA/apex.git \ + && cd apex \ + && python setup.py install --cuda_ext --cpp_ext + # install PyTorch Detection ARG FORCE_CUDA="1" ENV FORCE_CUDA=${FORCE_CUDA} diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index 2f5a88009..4e2fe1acc 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -431,3 +431,13 @@ _C.OUTPUT_DIR = "." _C.PATHS_CATALOG = os.path.join(os.path.dirname(__file__), "paths_catalog.py") + +# ---------------------------------------------------------------------------- # +# Precision options +# ---------------------------------------------------------------------------- # + +# Precision of input, allowable: (float32, float16) +_C.DTYPE = "float32" + +# Enable verbosity in apex.amp +_C.AMP_VERBOSE = False diff --git a/maskrcnn_benchmark/engine/trainer.py b/maskrcnn_benchmark/engine/trainer.py index 38a9e524b..281d91339 100644 --- a/maskrcnn_benchmark/engine/trainer.py +++ b/maskrcnn_benchmark/engine/trainer.py @@ -9,6 +9,7 @@ from maskrcnn_benchmark.utils.comm import get_world_size from maskrcnn_benchmark.utils.metric_logger import MetricLogger +from apex import amp def reduce_loss_dict(loss_dict): """ @@ -73,7 +74,10 @@ def do_train( meters.update(loss=losses_reduced, **loss_dict_reduced) optimizer.zero_grad() - losses.backward() + # Note: If mixed precision is not used, this ends up doing nothing + # Otherwise apply loss scaling for mixed-precision recipe + with amp.scale_loss(losses, optimizer) as scaled_losses: + scaled_losses.backward() optimizer.step() batch_time = time.time() - end diff --git a/maskrcnn_benchmark/layers/batch_norm.py b/maskrcnn_benchmark/layers/batch_norm.py index 903607ac3..3762e49e8 100644 --- a/maskrcnn_benchmark/layers/batch_norm.py +++ b/maskrcnn_benchmark/layers/batch_norm.py @@ -17,6 +17,13 @@ def __init__(self, n): self.register_buffer("running_var", torch.ones(n)) def forward(self, x): + # Cast all fixed parameters to half() if necessary + if x.dtype == torch.float16: + self.weight = self.weight.half() + self.bias = self.bias.half() + self.running_mean = self.running_mean.half() + self.running_var = self.running_var.half() + scale = self.weight * self.running_var.rsqrt() bias = self.bias - self.running_mean * scale scale = scale.reshape(1, -1, 1, 1) diff --git a/maskrcnn_benchmark/layers/nms.py b/maskrcnn_benchmark/layers/nms.py index 1e80b5550..39bff82b3 100644 --- a/maskrcnn_benchmark/layers/nms.py +++ b/maskrcnn_benchmark/layers/nms.py @@ -2,6 +2,10 @@ # from ._utils import _C from maskrcnn_benchmark import _C -nms = _C.nms +from apex import amp + +# Only valid with fp32 inputs - give AMP the hint +nms = amp.float_function(_C.nms) + # nms.__doc__ = """ # This function performs Non-maximum suppresion""" diff --git a/maskrcnn_benchmark/layers/roi_align.py b/maskrcnn_benchmark/layers/roi_align.py index 170c8f186..ec797ed25 100644 --- a/maskrcnn_benchmark/layers/roi_align.py +++ b/maskrcnn_benchmark/layers/roi_align.py @@ -7,6 +7,7 @@ from maskrcnn_benchmark import _C +from apex import amp class _ROIAlign(Function): @staticmethod @@ -46,7 +47,6 @@ def backward(ctx, grad_output): roi_align = _ROIAlign.apply - class ROIAlign(nn.Module): def __init__(self, output_size, spatial_scale, sampling_ratio): super(ROIAlign, self).__init__() @@ -54,6 +54,7 @@ def __init__(self, output_size, spatial_scale, sampling_ratio): self.spatial_scale = spatial_scale self.sampling_ratio = sampling_ratio + @amp.float_function def forward(self, input, rois): return roi_align( input, rois, self.output_size, self.spatial_scale, self.sampling_ratio diff --git a/maskrcnn_benchmark/layers/roi_pool.py b/maskrcnn_benchmark/layers/roi_pool.py index c0e42756e..586339076 100644 --- a/maskrcnn_benchmark/layers/roi_pool.py +++ b/maskrcnn_benchmark/layers/roi_pool.py @@ -7,6 +7,7 @@ from maskrcnn_benchmark import _C +from apex import amp class _ROIPool(Function): @staticmethod @@ -52,6 +53,7 @@ def __init__(self, output_size, spatial_scale): self.output_size = output_size self.spatial_scale = spatial_scale + @amp.float_function def forward(self, input, rois): return roi_pool(input, rois, self.output_size, self.spatial_scale) diff --git a/maskrcnn_benchmark/modeling/poolers.py b/maskrcnn_benchmark/modeling/poolers.py index 9b3524d20..519440e79 100644 --- a/maskrcnn_benchmark/modeling/poolers.py +++ b/maskrcnn_benchmark/modeling/poolers.py @@ -116,7 +116,7 @@ def forward(self, x, boxes): for level, (per_level_feature, pooler) in enumerate(zip(x, self.poolers)): idx_in_level = torch.nonzero(levels == level).squeeze(1) rois_per_level = rois[idx_in_level] - result[idx_in_level] = pooler(per_level_feature, rois_per_level) + result[idx_in_level] = pooler(per_level_feature, rois_per_level).to(dtype) return result diff --git a/maskrcnn_benchmark/modeling/roi_heads/mask_head/inference.py b/maskrcnn_benchmark/modeling/roi_heads/mask_head/inference.py index cd033e06c..bd831c085 100644 --- a/maskrcnn_benchmark/modeling/roi_heads/mask_head/inference.py +++ b/maskrcnn_benchmark/modeling/roi_heads/mask_head/inference.py @@ -111,11 +111,16 @@ def expand_masks(mask, padding): pad2 = 2 * padding scale = float(M + pad2) / M padded_mask = mask.new_zeros((N, 1, M + pad2, M + pad2)) + padded_mask[:, :, padding:-padding, padding:-padding] = mask return padded_mask, scale def paste_mask_in_image(mask, box, im_h, im_w, thresh=0.5, padding=1): + # Need to work on the CPU, where fp16 isn't supported - cast to float to avoid this + mask = mask.float() + box = box.float() + padded_mask, scale = expand_masks(mask[None], padding=padding) mask = padded_mask[0, 0] box = expand_boxes(box[None], scale)[0] diff --git a/tools/test_net.py b/tools/test_net.py index d0acd2833..c666a4655 100644 --- a/tools/test_net.py +++ b/tools/test_net.py @@ -17,6 +17,12 @@ from maskrcnn_benchmark.utils.logger import setup_logger from maskrcnn_benchmark.utils.miscellaneous import mkdir +# Check if we can enable mixed-precision via apex.amp +try: + from apex import amp +except ImportError: + raise ImportError('Use APEX for mixed precision via apex.amp') + def main(): parser = argparse.ArgumentParser(description="PyTorch Object Detection Inference") @@ -61,6 +67,10 @@ def main(): model = build_detection_model(cfg) model.to(cfg.MODEL.DEVICE) + # Initialize mixed-precision if necessary + use_mixed_precision = cfg.DTYPE == 'float16' + amp_handle = amp.init(enabled=use_mixed_precision, verbose=cfg.AMP_VERBOSE) + output_dir = cfg.OUTPUT_DIR checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir) _ = checkpointer.load(cfg.MODEL.WEIGHT) diff --git a/tools/train_net.py b/tools/train_net.py index e4f95f015..9f4761b3f 100644 --- a/tools/train_net.py +++ b/tools/train_net.py @@ -25,6 +25,13 @@ from maskrcnn_benchmark.utils.logger import setup_logger from maskrcnn_benchmark.utils.miscellaneous import mkdir +# See if we can use apex.DistributedDataParallel instead of the torch default, +# and enable mixed-precision via apex.amp +try: + from apex import amp +except ImportError: + raise ImportError('Use APEX for multi-precision via apex.amp') + def train(cfg, local_rank, distributed): model = build_detection_model(cfg) @@ -34,6 +41,11 @@ def train(cfg, local_rank, distributed): optimizer = make_optimizer(cfg, model) scheduler = make_lr_scheduler(cfg, optimizer) + # Initialize mixed-precision training + use_mixed_precision = cfg.DTYPE == "float16" + amp_opt_level = 'O1' if use_mixed_precision else 'O0' + model, optimizer = amp.initialize(model, optimizer, opt_level=amp_opt_level) + if distributed: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[local_rank], output_device=local_rank, From a44d65dcdb9c9098a25dd6b23ca3c36f1b887aba Mon Sep 17 00:00:00 2001 From: qianyizhang Date: Sat, 20 Apr 2019 19:01:21 +0800 Subject: [PATCH 11/52] fix imports for latest pytorch-nightly (#698) --- maskrcnn_benchmark/utils/model_zoo.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/utils/model_zoo.py b/maskrcnn_benchmark/utils/model_zoo.py index 7a0ebb349..bd644d7cc 100644 --- a/maskrcnn_benchmark/utils/model_zoo.py +++ b/maskrcnn_benchmark/utils/model_zoo.py @@ -2,9 +2,14 @@ import os import sys -from torch.utils.model_zoo import _download_url_to_file -from torch.utils.model_zoo import urlparse -from torch.utils.model_zoo import HASH_REGEX +try: + from torch.utils.model_zoo import _download_url_to_file + from torch.utils.model_zoo import urlparse + from torch.utils.model_zoo import HASH_REGEX +except: + from torch.hub import _download_url_to_file + from torch.hub import urlparse + from torch.hub import HASH_REGEX from maskrcnn_benchmark.utils.comm import is_main_process from maskrcnn_benchmark.utils.comm import synchronize From 4466eb5a21cdc13a239a4f405ca3b39c86f5284a Mon Sep 17 00:00:00 2001 From: zimenglan Date: Sat, 20 Apr 2019 19:08:15 +0800 Subject: [PATCH 12/52] add documentation for finetuning cityscapes (#697) * make pixel indexes 0-based for bounding box in pascal voc dataset * replacing all instances of torch.distributed.deprecated with torch.distributed * replacing all instances of torch.distributed.deprecated with torch.distributed * add GroupNorm * add GroupNorm -- sort out yaml files * use torch.nn.GroupNorm instead, replace 'use_gn' with 'conv_block' and use 'BaseStem'&'Bottleneck' to simply codes * modification on 'group_norm' and 'conv_with_kaiming_uniform' function * modification on yaml files in configs/gn_baselines/ and reduce the amount of indentation and code duplication * use 'kaiming_uniform' to initialize resnet, disable gn after fc layer, and add dilation into ResNetHead * agnostic-regression for bbox * please set 'STRIDE_IN_1X1' to be 'False' when backbone use GN * add README.md for GN * add dcn from mmdetection * add documentation for finetuning cityscapes * add documentation for finetuning cityscapes * add documentation for finetuning cityscapes --- configs/cityscapes/README.md | 213 +++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 configs/cityscapes/README.md diff --git a/configs/cityscapes/README.md b/configs/cityscapes/README.md new file mode 100644 index 000000000..8785cdb93 --- /dev/null +++ b/configs/cityscapes/README.md @@ -0,0 +1,213 @@ +### Paper +1 [mask-rcnn](https://arxiv.org/pdf/1703.06870.pdf) + + +### dataset +1 [cityscapesScripts](https://github.com/mcordts/cityscapesScripts) + + +### Performance (from paper) +| case | training data | im/gpu | mask AP[val] | mask AP [test] | mask AP50 [test] | +|--------------|:-------------:|:------:|:------------:|:--------------:|-----------------:| +| R-50-FPN | fine | 8/8 | 31.5 | 26.2 | 49.9 | +| R-50-FPN | fine + COCO | 8/8 | 36.4 | 32.0 | 58.1 | + + +### Note (from paper) +We apply our Mask R-CNN models with the ResNet-FPN-50 backbone; we found the 101-layer counterpart performs similarly due to the small dataset size. We train with image scale (shorter side) randomly sampled from [800, 1024], which reduces overfitting; inference is on a single scale of 1024 pixels. We use a mini-batch size of 1 image per GPU (so 8 on 8 GPUs) and train the model for 24k iterations, starting from a learning rate of 0.01 and reducing it to 0.001 at 18k iterations. It takes ∼4 hours of training on a single 8-GPU machine under this setting. + + +### Implemetation (for finetuning from coco trained model) +Step 1: download trained model on coco dataset from [model zoo](https://download.pytorch.org/models/maskrcnn/e2e_mask_rcnn_R_50_FPN_1x.pth) +Step 2: do the model surgery on the trained model as below and use it as `pretrained model` for finetuning: +```python +def clip_weights_from_pretrain_of_coco_to_cityscapes(f, out_file): + """""" + # COCO categories for pretty print + COCO_CATEGORIES = [ + "__background__", + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", + ] + # Cityscapes of fine categories for pretty print + CITYSCAPES_FINE_CATEGORIES = [ + "__background__", + "person", + "rider", + "car", + "truck", + "bus", + "train", + "motorcycle", + "bicycle", + ] + coco_cats = COCO_CATEGORIES + cityscapes_cats = CITYSCAPES_FINE_CATEGORIES + coco_cats_to_inds = dict(zip(coco_cats, range(len(coco_cats)))) + cityscapes_cats_to_inds = dict( + zip(cityscapes_cats, range(len(cityscapes_cats))) + ) + + checkpoint = torch.load(f) + m = checkpoint['model'] + + weight_names = { + "cls_score": "module.roi_heads.box.predictor.cls_score.weight", + "bbox_pred": "module.roi_heads.box.predictor.bbox_pred.weight", + "mask_fcn_logits": "module.roi_heads.mask.predictor.mask_fcn_logits.weight", + } + bias_names = { + "cls_score": "module.roi_heads.box.predictor.cls_score.bias", + "bbox_pred": "module.roi_heads.box.predictor.bbox_pred.bias", + "mask_fcn_logits": "module.roi_heads.mask.predictor.mask_fcn_logits.bias", + } + + representation_size = m[weight_names["cls_score"]].size(1) + cls_score = nn.Linear(representation_size, len(cityscapes_cats)) + nn.init.normal_(cls_score.weight, std=0.01) + nn.init.constant_(cls_score.bias, 0) + + representation_size = m[weight_names["bbox_pred"]].size(1) + class_agnostic = m[weight_names["bbox_pred"]].size(0) != len(coco_cats) * 4 + num_bbox_reg_classes = 2 if class_agnostic else len(cityscapes_cats) + bbox_pred = nn.Linear(representation_size, num_bbox_reg_classes * 4) + nn.init.normal_(bbox_pred.weight, std=0.001) + nn.init.constant_(bbox_pred.bias, 0) + + dim_reduced = m[weight_names["mask_fcn_logits"]].size(1) + mask_fcn_logits = Conv2d(dim_reduced, len(cityscapes_cats), 1, 1, 0) + nn.init.constant_(mask_fcn_logits.bias, 0) + nn.init.kaiming_normal_( + mask_fcn_logits.weight, mode="fan_out", nonlinearity="relu" + ) + + def _copy_weight(src_weight, dst_weight): + for ix, cat in enumerate(cityscapes_cats): + if cat not in coco_cats: + continue + jx = coco_cats_to_inds[cat] + dst_weight[ix] = src_weight[jx] + return dst_weight + + def _copy_bias(src_bias, dst_bias, class_agnostic=False): + if class_agnostic: + return dst_bias + return _copy_weight(src_bias, dst_bias) + + m[weight_names["cls_score"]] = _copy_weight( + m[weight_names["cls_score"]], cls_score.weight + ) + m[weight_names["bbox_pred"]] = _copy_weight( + m[weight_names["bbox_pred"]], bbox_pred.weight + ) + m[weight_names["mask_fcn_logits"]] = _copy_weight( + m[weight_names["mask_fcn_logits"]], mask_fcn_logits.weight + ) + + m[bias_names["cls_score"]] = _copy_bias( + m[bias_names["cls_score"]], cls_score.bias + ) + m[bias_names["bbox_pred"]] = _copy_bias( + m[bias_names["bbox_pred"]], bbox_pred.bias, class_agnostic + ) + m[bias_names["mask_fcn_logits"]] = _copy_bias( + m[bias_names["mask_fcn_logits"]], mask_fcn_logits.bias + ) + + print("f: {}\nout_file: {}".format(f, out_file)) + torch.save(m, out_file) +``` +Step 3: modify the `input&solver` configuration in the `yaml` file, like this: +``` +INPUT: + MIN_SIZE_TRAIN: (800, 832, 863, 896, 928, 960, 992, 1024, 1024) + MAX_SIZE_TRAIN: 2048 + MIN_SIZE_TEST: 1024 + MAX_SIZE_TEST: 2048 + +SOLVER: + BASE_LR: 0.01 + IMS_PER_BATCH: 8 + WEIGHT_DECAY: 0.0001 + STEPS: (3000,) + MAX_ITER: 4000 +``` + From b318c3ece6960d2884ddfec8bd912a870c99f6c4 Mon Sep 17 00:00:00 2001 From: Rodrigo Berriel Date: Sat, 20 Apr 2019 13:10:14 +0200 Subject: [PATCH 13/52] Add a switch for POST_NMS per batch/image during training (#695) --- maskrcnn_benchmark/config/defaults.py | 3 +++ maskrcnn_benchmark/modeling/rpn/inference.py | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index 4e2fe1acc..ddc2c4e21 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -165,6 +165,9 @@ # all FPN levels _C.MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN = 2000 _C.MODEL.RPN.FPN_POST_NMS_TOP_N_TEST = 2000 +# Apply the post NMS per batch (default) or per image during training +# (default is True to be consistent with Detectron, see Issue #672) +_C.MODEL.RPN.FPN_POST_NMS_PER_BATCH = True # Custom rpn head, empty to use default conv or separable conv _C.MODEL.RPN.RPN_HEAD = "SingleConvRPNHead" diff --git a/maskrcnn_benchmark/modeling/rpn/inference.py b/maskrcnn_benchmark/modeling/rpn/inference.py index 556082645..9fd23c5bc 100644 --- a/maskrcnn_benchmark/modeling/rpn/inference.py +++ b/maskrcnn_benchmark/modeling/rpn/inference.py @@ -24,6 +24,7 @@ def __init__( min_size, box_coder=None, fpn_post_nms_top_n=None, + fpn_post_nms_per_batch=True, ): """ Arguments: @@ -47,6 +48,7 @@ def __init__( if fpn_post_nms_top_n is None: fpn_post_nms_top_n = post_nms_top_n self.fpn_post_nms_top_n = fpn_post_nms_top_n + self.fpn_post_nms_per_batch = fpn_post_nms_per_batch def add_gt_proposals(self, proposals, targets): """ @@ -154,9 +156,9 @@ def select_over_all_levels(self, boxlists): # different behavior during training and during testing: # during training, post_nms_top_n is over *all* the proposals combined, while # during testing, it is over the proposals for each image - # TODO resolve this difference and make it consistent. It should be per image, - # and not per batch - if self.training: + # NOTE: it should be per image, and not per batch. However, to be consistent + # with Detectron, the default is per batch (see Issue #672) + if self.training and self.fpn_post_nms_per_batch: objectness = torch.cat( [boxlist.get_field("objectness") for boxlist in boxlists], dim=0 ) @@ -189,6 +191,7 @@ def make_rpn_postprocessor(config, rpn_box_coder, is_train): if not is_train: pre_nms_top_n = config.MODEL.RPN.PRE_NMS_TOP_N_TEST post_nms_top_n = config.MODEL.RPN.POST_NMS_TOP_N_TEST + fpn_post_nms_per_batch = config.MODEL.RPN.FPN_POST_NMS_PER_BATCH nms_thresh = config.MODEL.RPN.NMS_THRESH min_size = config.MODEL.RPN.MIN_SIZE box_selector = RPNPostProcessor( @@ -198,5 +201,6 @@ def make_rpn_postprocessor(config, rpn_box_coder, is_train): min_size=min_size, box_coder=rpn_box_coder, fpn_post_nms_top_n=fpn_post_nms_top_n, + fpn_post_nms_per_batch=fpn_post_nms_per_batch, ) return box_selector From 66c3e56cb677a02d82ae63db1101dc4c6d51ce6d Mon Sep 17 00:00:00 2001 From: Rodrigo Berriel Date: Sat, 20 Apr 2019 13:17:10 +0200 Subject: [PATCH 14/52] Add OpenCV in the INSTALL instructions (#694) --- INSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index c56cc9f2c..37bc0c9a0 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -24,7 +24,7 @@ conda activate maskrcnn_benchmark conda install ipython # maskrcnn_benchmark and coco api dependencies -pip install ninja yacs cython matplotlib tqdm +pip install ninja yacs cython matplotlib tqdm opencv-python # follow PyTorch installation in https://pytorch.org/get-started/locally/ # we give the instructions for CUDA 9.0 @@ -39,7 +39,7 @@ cd cocoapi/PythonAPI python setup.py build_ext install # install apex -cd ~github +cd $INSTALL_DIR git clone https://github.com/NVIDIA/apex.git cd apex python setup.py install --cuda_ext --cpp_ext From b3cab7fc2de9bee09a6bcb3977da161f6f7529b8 Mon Sep 17 00:00:00 2001 From: zimenglan Date: Sat, 20 Apr 2019 23:35:37 +0800 Subject: [PATCH 15/52] add 'once_differentiable' for dcn and modify 'configs/cityscapes/README.md' (#701) * make pixel indexes 0-based for bounding box in pascal voc dataset * replacing all instances of torch.distributed.deprecated with torch.distributed * replacing all instances of torch.distributed.deprecated with torch.distributed * add GroupNorm * add GroupNorm -- sort out yaml files * use torch.nn.GroupNorm instead, replace 'use_gn' with 'conv_block' and use 'BaseStem'&'Bottleneck' to simply codes * modification on 'group_norm' and 'conv_with_kaiming_uniform' function * modification on yaml files in configs/gn_baselines/ and reduce the amount of indentation and code duplication * use 'kaiming_uniform' to initialize resnet, disable gn after fc layer, and add dilation into ResNetHead * agnostic-regression for bbox * please set 'STRIDE_IN_1X1' to be 'False' when backbone use GN * add README.md for GN * add dcn from mmdetection * add documentation for finetuning cityscapes * add documentation for finetuning cityscapes * add documentation for finetuning cityscapes * add 'once_differentiable' for dcn and modify 'configs/cityscapes/README.md' --- configs/cityscapes/README.md | 8 ++++++-- maskrcnn_benchmark/layers/dcn/deform_conv_func.py | 3 +++ maskrcnn_benchmark/layers/dcn/deform_pool_func.py | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/configs/cityscapes/README.md b/configs/cityscapes/README.md index 8785cdb93..b8595f79c 100644 --- a/configs/cityscapes/README.md +++ b/configs/cityscapes/README.md @@ -195,10 +195,13 @@ def clip_weights_from_pretrain_of_coco_to_cityscapes(f, out_file): print("f: {}\nout_file: {}".format(f, out_file)) torch.save(m, out_file) ``` -Step 3: modify the `input&solver` configuration in the `yaml` file, like this: +Step 3: modify the `input&weight&solver` configuration in the `yaml` file, like this: ``` +MODEL: + WEIGHT: "xxx.pth" # the model u save from above code + INPUT: - MIN_SIZE_TRAIN: (800, 832, 863, 896, 928, 960, 992, 1024, 1024) + MIN_SIZE_TRAIN: (800, 832, 864, 896, 928, 960, 992, 1024, 1024) MAX_SIZE_TRAIN: 2048 MIN_SIZE_TEST: 1024 MAX_SIZE_TEST: 2048 @@ -210,4 +213,5 @@ SOLVER: STEPS: (3000,) MAX_ITER: 4000 ``` +Step 4: train the model. diff --git a/maskrcnn_benchmark/layers/dcn/deform_conv_func.py b/maskrcnn_benchmark/layers/dcn/deform_conv_func.py index ddc92bb0c..a276a05fe 100644 --- a/maskrcnn_benchmark/layers/dcn/deform_conv_func.py +++ b/maskrcnn_benchmark/layers/dcn/deform_conv_func.py @@ -1,5 +1,6 @@ import torch from torch.autograd import Function +from torch.autograd.function import once_differentiable from torch.nn.modules.utils import _pair from maskrcnn_benchmark import _C @@ -67,6 +68,7 @@ def forward( return output @staticmethod + @once_differentiable def backward(ctx, grad_output): input, offset, weight = ctx.saved_tensors @@ -201,6 +203,7 @@ def forward( return output @staticmethod + @once_differentiable def backward(ctx, grad_output): if not grad_output.is_cuda: raise NotImplementedError diff --git a/maskrcnn_benchmark/layers/dcn/deform_pool_func.py b/maskrcnn_benchmark/layers/dcn/deform_pool_func.py index f18fdd4cb..2f7810b23 100644 --- a/maskrcnn_benchmark/layers/dcn/deform_pool_func.py +++ b/maskrcnn_benchmark/layers/dcn/deform_pool_func.py @@ -1,5 +1,6 @@ import torch from torch.autograd import Function +from torch.autograd.function import once_differentiable from maskrcnn_benchmark import _C @@ -60,6 +61,7 @@ def forward( return output @staticmethod + @once_differentiable def backward(ctx, grad_output): if not grad_output.is_cuda: raise NotImplementedError From ff5903e0203b4f12b557d0f85df74fd5142b3317 Mon Sep 17 00:00:00 2001 From: Ren Jin Date: Sun, 21 Apr 2019 02:10:08 +0800 Subject: [PATCH 16/52] color jitter only during training (#703) --- maskrcnn_benchmark/data/transforms/build.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/maskrcnn_benchmark/data/transforms/build.py b/maskrcnn_benchmark/data/transforms/build.py index 825c3ee42..88aa975b6 100644 --- a/maskrcnn_benchmark/data/transforms/build.py +++ b/maskrcnn_benchmark/data/transforms/build.py @@ -7,21 +7,29 @@ def build_transforms(cfg, is_train=True): min_size = cfg.INPUT.MIN_SIZE_TRAIN max_size = cfg.INPUT.MAX_SIZE_TRAIN flip_prob = 0.5 # cfg.INPUT.FLIP_PROB_TRAIN + brightness = cfg.INPUT.BRIGHTNESS + contrast = cfg.INPUT.CONTRAST + saturation = cfg.INPUT.SATURATION + hue = cfg.INPUT.HUE else: min_size = cfg.INPUT.MIN_SIZE_TEST max_size = cfg.INPUT.MAX_SIZE_TEST flip_prob = 0 + brightness = 0.0 + contrast = 0.0 + saturation = 0.0 + hue = 0.0 to_bgr255 = cfg.INPUT.TO_BGR255 normalize_transform = T.Normalize( mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr255=to_bgr255 ) color_jitter = T.ColorJitter( - brightness=cfg.INPUT.BRIGHTNESS, - contrast=cfg.INPUT.CONTRAST, - saturation=cfg.INPUT.SATURATION, - hue=cfg.INPUT.HUE, - ) + brightness=brightness, + contrast=contrast, + saturation=saturation, + hue=hue, + ) transform = T.Compose( [ From 91d9fb9ffb1722252cdf02d4b2a0a24ce5d10883 Mon Sep 17 00:00:00 2001 From: Rodrigo Berriel Date: Sat, 20 Apr 2019 20:11:28 +0200 Subject: [PATCH 17/52] Update documentation and remove a generic except (#702) * Add ImportError to generic except * Black formatter * OpenCV is no longer optional * Update README w.r.t. FPN_POST_NMS_PER_BATCH --- INSTALL.md | 2 +- README.md | 4 ++-- maskrcnn_benchmark/utils/model_zoo.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 37bc0c9a0..2bdfb7fe3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -7,7 +7,7 @@ - yacs - matplotlib - GCC >= 4.9 -- (optional) OpenCV for the webcam demo +- OpenCV ### Option 1: Step-by-step installation diff --git a/README.md b/README.md index 5c68de5f0..b7cda0a62 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ and we have divided the learning rate by 8x. We also changed the batch size during testing, but that is generally not necessary because testing requires much less memory than training. -Furthermore, we set ```MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN 2000``` as the proposals are selected for per the batch rather than per image. The value is calculated by **1000 x images-per-gpu**. Here we have 2 images per GPU, therefore we set the number as 1000 x 2 = 2000. If we have 8 images per GPU, the value should be set as 8000. See [#672](https://github.com/facebookresearch/maskrcnn-benchmark/issues/672) for more details. +Furthermore, we set `MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN 2000` as the proposals are selected for per the batch rather than per image in the default training. The value is calculated by **1000 x images-per-gpu**. Here we have 2 images per GPU, therefore we set the number as 1000 x 2 = 2000. If we have 8 images per GPU, the value should be set as 8000. Note that this does not apply if `MODEL.RPN.FPN_POST_NMS_PER_BATCH` is set to `False` during training. See [#672](https://github.com/facebookresearch/maskrcnn-benchmark/issues/672) for more details. ### Multi-GPU training We use internally `torch.distributed.launch` in order to launch @@ -150,7 +150,7 @@ process will only use a single GPU. export NGPUS=8 python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_benchmark/tools/train_net.py --config-file "path/to/config/file.yaml" MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN images_per_gpu x 1000 ``` -Note we should set ```MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN``` follow the rule in Single-GPU training. +Note we should set `MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN` follow the rule in Single-GPU training. ## Abstractions For more information on some of the main abstractions in our implementation, see [ABSTRACTIONS.md](ABSTRACTIONS.md). diff --git a/maskrcnn_benchmark/utils/model_zoo.py b/maskrcnn_benchmark/utils/model_zoo.py index bd644d7cc..2128ad7bb 100644 --- a/maskrcnn_benchmark/utils/model_zoo.py +++ b/maskrcnn_benchmark/utils/model_zoo.py @@ -3,13 +3,13 @@ import sys try: - from torch.utils.model_zoo import _download_url_to_file - from torch.utils.model_zoo import urlparse - from torch.utils.model_zoo import HASH_REGEX -except: from torch.hub import _download_url_to_file from torch.hub import urlparse from torch.hub import HASH_REGEX +except ImportError: + from torch.utils.model_zoo import _download_url_to_file + from torch.utils.model_zoo import urlparse + from torch.utils.model_zoo import HASH_REGEX from maskrcnn_benchmark.utils.comm import is_main_process from maskrcnn_benchmark.utils.comm import synchronize @@ -35,8 +35,8 @@ def cache_url(url, model_dir=None, progress=True): >>> cached_file = maskrcnn_benchmark.utils.model_zoo.cache_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth') """ if model_dir is None: - torch_home = os.path.expanduser(os.getenv('TORCH_HOME', '~/.torch')) - model_dir = os.getenv('TORCH_MODEL_ZOO', os.path.join(torch_home, 'models')) + torch_home = os.path.expanduser(os.getenv("TORCH_HOME", "~/.torch")) + model_dir = os.getenv("TORCH_MODEL_ZOO", os.path.join(torch_home, "models")) if not os.path.exists(model_dir): os.makedirs(model_dir) parts = urlparse(url) From dbec62be96c68e9d09bc502dcb86cf0c5f75b4bb Mon Sep 17 00:00:00 2001 From: Jacobew <1003380053@qq.com> Date: Mon, 22 Apr 2019 19:30:45 +0800 Subject: [PATCH 18/52] Evaluation details (#705) * Update documentation for testing and calculating mAP for each class * Update documentation for testing and calculating mAP for each class --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index b7cda0a62..baa7c6bc4 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,14 @@ python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_ben ``` Note we should set `MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN` follow the rule in Single-GPU training. +## Evaluation +You can test your model directly on single or multiple gpus. Here is an example for Mask R-CNN R-50 FPN with the 1x schedule on 8 GPUS: +```bash +export NGPUS=8 +python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_benchmark/tools/test_net.py --config-file "configs/e2e_mask_rcnn_R_50_FPN_1x.yaml" TEST.IMS_PER_BATCH 16 +``` +To calculate mAP for each class, you can simply modify a few lines in [coco_eval.py](https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py). See [#524](https://github.com/facebookresearch/maskrcnn-benchmark/issues/524#issuecomment-475118810) for more details. + ## Abstractions For more information on some of the main abstractions in our implementation, see [ABSTRACTIONS.md](ABSTRACTIONS.md). From d44c3fd86220d6b1ca5fe61e7f258fecabec396d Mon Sep 17 00:00:00 2001 From: Thibault FEVRY Date: Tue, 23 Apr 2019 12:44:39 -0400 Subject: [PATCH 19/52] Quickfix for outdated paras (#710) --- maskrcnn_benchmark/config/defaults.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index ddc2c4e21..37e362cd7 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -10,9 +10,9 @@ # Whenever an argument can be either used for training or for testing, the # corresponding name will be post-fixed by a _TRAIN for a training parameter, # or _TEST for a test-specific parameter. -# For example, the number of images during training will be -# IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be -# IMAGES_PER_BATCH_TEST +# For example, the maximum image side during training will be +# INPUT.MAX_SIZE_TRAIN, while for testing it will be +# INPUT.MAX_SIZE_TEST # ----------------------------------------------------------------------------- # Config definition From eb4d3352be7e968b96260c8999a331e8431da95f Mon Sep 17 00:00:00 2001 From: zimenglan Date: Thu, 25 Apr 2019 21:11:07 +0800 Subject: [PATCH 20/52] rectify the 'padding' for 'DFConv2d' (#717) * make pixel indexes 0-based for bounding box in pascal voc dataset * replacing all instances of torch.distributed.deprecated with torch.distributed * replacing all instances of torch.distributed.deprecated with torch.distributed * add GroupNorm * add GroupNorm -- sort out yaml files * use torch.nn.GroupNorm instead, replace 'use_gn' with 'conv_block' and use 'BaseStem'&'Bottleneck' to simply codes * modification on 'group_norm' and 'conv_with_kaiming_uniform' function * modification on yaml files in configs/gn_baselines/ and reduce the amount of indentation and code duplication * use 'kaiming_uniform' to initialize resnet, disable gn after fc layer, and add dilation into ResNetHead * agnostic-regression for bbox * please set 'STRIDE_IN_1X1' to be 'False' when backbone use GN * add README.md for GN * add dcn from mmdetection * add documentation for finetuning cityscapes * add documentation for finetuning cityscapes * add documentation for finetuning cityscapes * add 'once_differentiable' for dcn and modify 'configs/cityscapes/README.md' * rectify the 'padding' for 'DFConv2d' --- maskrcnn_benchmark/layers/misc.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/maskrcnn_benchmark/layers/misc.py b/maskrcnn_benchmark/layers/misc.py index b3bca0db5..b64f23840 100644 --- a/maskrcnn_benchmark/layers/misc.py +++ b/maskrcnn_benchmark/layers/misc.py @@ -127,9 +127,18 @@ def __init__( ): super(DFConv2d, self).__init__() if isinstance(kernel_size, (list, tuple)): + assert isinstance(stride, (list, tuple)) + assert isinstance(dilation, (list, tuple)) assert len(kernel_size) == 2 + assert len(stride) == 2 + assert len(dilation) == 2 + padding = ( + dilation[0] * (kernel_size[0] - 1) // 2, + dilation[1] * (kernel_size[1] - 1) // 2 + ) offset_base_channels = kernel_size[0] * kernel_size[1] else: + padding = dilation * (kernel_size - 1) // 2 offset_base_channels = kernel_size * kernel_size if with_modulated_dcn: from maskrcnn_benchmark.layers import ModulatedDeformConv @@ -143,8 +152,8 @@ def __init__( in_channels, deformable_groups * offset_channels, kernel_size=kernel_size, - stride= stride, - padding= dilation, + stride=stride, + padding=padding, groups=1, dilation=dilation ) @@ -155,8 +164,8 @@ def __init__( in_channels, out_channels, kernel_size=kernel_size, - stride= stride, - padding=dilation, + stride=stride, + padding=padding, dilation=dilation, groups=groups, deformable_groups=deformable_groups, @@ -165,7 +174,7 @@ def __init__( self.with_modulated_dcn = with_modulated_dcn self.kernel_size = kernel_size self.stride = stride - self.padding = dilation + self.padding = padding self.dilation = dilation def forward(self, x): From 1127bdd368613f320f7b113320e62994c0baa216 Mon Sep 17 00:00:00 2001 From: Qizhu Li Date: Fri, 3 May 2019 05:21:39 +0800 Subject: [PATCH 21/52] Renames the `transforms` attribute of COCODataset (#744) Avoids naming clash with torchvision CocoDetection class --- maskrcnn_benchmark/data/datasets/coco.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/data/datasets/coco.py b/maskrcnn_benchmark/data/datasets/coco.py index d0e42b437..313924413 100644 --- a/maskrcnn_benchmark/data/datasets/coco.py +++ b/maskrcnn_benchmark/data/datasets/coco.py @@ -61,7 +61,7 @@ def __init__( v: k for k, v in self.json_category_id_to_contiguous_id.items() } self.id_to_img_map = {k: v for k, v in enumerate(self.ids)} - self.transforms = transforms + self._transforms = transforms def __getitem__(self, idx): img, anno = super(COCODataset, self).__getitem__(idx) @@ -90,8 +90,8 @@ def __getitem__(self, idx): target = target.clip_to_image(remove_empty=True) - if self.transforms is not None: - img, target = self.transforms(img, target) + if self._transforms is not None: + img, target = self._transforms(img, target) return img, target, idx From 16b8b09aea1b7ebee52bf8bf33098bdcf7b1cdf2 Mon Sep 17 00:00:00 2001 From: Qi Cai Date: Mon, 20 May 2019 14:34:39 +0800 Subject: [PATCH 22/52] change torch.IntTensor to torch.LongTensor for storing large tensor numel (#799) --- maskrcnn_benchmark/utils/comm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maskrcnn_benchmark/utils/comm.py b/maskrcnn_benchmark/utils/comm.py index 46d7c55ce..669f208ad 100644 --- a/maskrcnn_benchmark/utils/comm.py +++ b/maskrcnn_benchmark/utils/comm.py @@ -63,8 +63,8 @@ def all_gather(data): tensor = torch.ByteTensor(storage).to("cuda") # obtain Tensor size of each rank - local_size = torch.IntTensor([tensor.numel()]).to("cuda") - size_list = [torch.IntTensor([0]).to("cuda") for _ in range(world_size)] + local_size = torch.LongTensor([tensor.numel()]).to("cuda") + size_list = [torch.LongTensor([0]).to("cuda") for _ in range(world_size)] dist.all_gather(size_list, local_size) size_list = [int(size.item()) for size in size_list] max_size = max(size_list) From 38d08ca9982c1cd812e2c7a37c6e0070f1769392 Mon Sep 17 00:00:00 2001 From: CoinCheung <867153576@qq.com> Date: Fri, 24 May 2019 17:39:00 +0800 Subject: [PATCH 23/52] modify senario where dataset has no mask provided (#814) * modify senario where dataset has no mask provided * change to check segmentation mask in the way of keypoint --- maskrcnn_benchmark/data/datasets/coco.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/data/datasets/coco.py b/maskrcnn_benchmark/data/datasets/coco.py index 313924413..cd9fc835e 100644 --- a/maskrcnn_benchmark/data/datasets/coco.py +++ b/maskrcnn_benchmark/data/datasets/coco.py @@ -79,9 +79,10 @@ def __getitem__(self, idx): classes = torch.tensor(classes) target.add_field("labels", classes) - masks = [obj["segmentation"] for obj in anno] - masks = SegmentationMask(masks, img.size, mode='poly') - target.add_field("masks", masks) + if anno and "segmentation" in anno[0]: + masks = [obj["segmentation"] for obj in anno] + masks = SegmentationMask(masks, img.size, mode='poly') + target.add_field("masks", masks) if anno and "keypoints" in anno[0]: keypoints = [obj["keypoints"] for obj in anno] From 95521b646c486faaf6c20af6a3ef08c41fa8f67b Mon Sep 17 00:00:00 2001 From: Linus Date: Fri, 24 May 2019 18:39:59 +0900 Subject: [PATCH 24/52] some protection code in bounding box operations (#811) * max proposal protect and box convert to xyxy in iou function * remove unnecessary lines --- maskrcnn_benchmark/structures/boxlist_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/structures/boxlist_ops.py b/maskrcnn_benchmark/structures/boxlist_ops.py index dc51212f4..02dcaf121 100644 --- a/maskrcnn_benchmark/structures/boxlist_ops.py +++ b/maskrcnn_benchmark/structures/boxlist_ops.py @@ -67,7 +67,8 @@ def boxlist_iou(boxlist1, boxlist2): if boxlist1.size != boxlist2.size: raise RuntimeError( "boxlists should have same image size, got {}, {}".format(boxlist1, boxlist2)) - + boxlist1 = boxlist1.convert("xyxy") + boxlist2 = boxlist2.convert("xyxy") N = len(boxlist1) M = len(boxlist2) From 5eca57b47927b0d628a864e0dc3e998839fbed86 Mon Sep 17 00:00:00 2001 From: Idolized22 <41726905+Idolized22@users.noreply.github.com> Date: Fri, 24 May 2019 12:44:15 +0300 Subject: [PATCH 25/52] Windows10 installation fixes ver2 (#741) * Update SigmoidFocalLoss_cuda.cu line 120 converted to long to support windows10 * Update SigmoidFocalLoss_cuda.cu removed old line and converted to long to support windows10 * Update SigmoidFocalLoss_cuda.cu Converted to long to support windows10 * Update INSTALL.md Step_By_Step Guide installation on windows * Update INSTALL.md updated Clone of architecture to be the one of this repo which has the fixes needed for Windows10 * Update INSTALL.md How to check what cuda version to install with pytorch * Update INSTALL.md --- INSTALL.md | 51 +++++++++++++++++++ .../csrc/cuda/SigmoidFocalLoss_cuda.cu | 5 +- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 2bdfb7fe3..b1bfaa293 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -61,7 +61,58 @@ unset INSTALL_DIR # or if you are on macOS # MACOSX_DEPLOYMENT_TARGET=10.9 CC=clang CXX=clang++ python setup.py build develop ``` +#### Windows 10 +```bash +open a cmd and change to desired installation directory +from now on will be refered as INSTALL_DIR +conda create --name maskrcnn_benchmark +conda activate maskrcnn_benchmark + +# this installs the right pip and dependencies for the fresh python +conda install ipython + +# maskrcnn_benchmark and coco api dependencies +pip install ninja yacs cython matplotlib tqdm opencv-python + +# follow PyTorch installation in https://pytorch.org/get-started/locally/ +# we give the instructions for CUDA 9.0 +## Important : check the cuda version installed on your computer by running the command in the cmd : +nvcc -- version +conda install -c pytorch pytorch-nightly torchvision cudatoolkit=9.0 + +git clone https://github.com/cocodataset/cocoapi.git + + #To prevent installation error do the following after commiting cocooapi : + #using file explorer naviagate to cocoapi\PythonAPI\setup.py and change line 14 from: + #extra_compile_args=['-Wno-cpp', '-Wno-unused-function', '-std=c99'], + #to + #extra_compile_args={'gcc': ['/Qstd=c99']}, + #Based on https://github.com/cocodataset/cocoapi/issues/51 + +cd cocoapi/PythonAPI +python setup.py build_ext install + +# navigate back to INSTALL_DIR +cd .. +cd .. +# install apex + +git clone https://github.com/NVIDIA/apex.git +cd apex +python setup.py install --cuda_ext --cpp_ext +# navigate back to INSTALL_DIR +cd .. +# install PyTorch Detection +git clone https://github.com/Idolized22/maskrcnn-benchmark.git +cd maskrcnn-benchmark + +# the following will install the lib with +# symbolic links, so that you can modify +# the files if you want and won't need to +# re-build it +python setup.py build develop +``` ### Option 2: Docker Image (Requires CUDA, Linux only) Build image with defaults (`CUDA=9.0`, `CUDNN=7`, `FORCE_CUDA=1`): diff --git a/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu b/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu index 7d40767bb..456a5f235 100644 --- a/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu +++ b/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu @@ -117,7 +117,8 @@ at::Tensor SigmoidFocalLoss_forward_cuda( auto losses_size = num_samples * logits.size(1); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); - dim3 grid(std::min(THCCeilDiv(losses_size, 512L), 4096L)); + dim3 grid(std::min(THCCeilDiv((long)losses_size, 512L), 4096L)); + dim3 block(512); if (losses.numel() == 0) { @@ -161,7 +162,7 @@ at::Tensor SigmoidFocalLoss_backward_cuda( auto d_logits_size = num_samples * logits.size(1); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); - dim3 grid(std::min(THCCeilDiv(d_logits_size, 512L), 4096L)); + dim3 grid(std::min(THCCeilDiv((long)d_logits_size, 512L), 4096L)); dim3 block(512); if (d_logits.numel() == 0) { From 7a9b18574d8cd6106e0c922d182a21d38a4b098c Mon Sep 17 00:00:00 2001 From: Hongliang Yuan Date: Fri, 24 May 2019 17:55:16 +0800 Subject: [PATCH 26/52] Multi-scale testing (#804) * Implement multi-scale testing(bbox aug) like Detectron. * Add comment. * Fix missing cfg after merge. --- .../e2e_mask_rcnn_R_50_FPN_1x.yaml | 48 +++++++ maskrcnn_benchmark/config/defaults.py | 21 ++++ maskrcnn_benchmark/data/build.py | 8 +- maskrcnn_benchmark/data/collate_batch.py | 12 ++ .../data/transforms/transforms.py | 8 +- maskrcnn_benchmark/engine/bbox_aug.py | 118 ++++++++++++++++++ maskrcnn_benchmark/engine/inference.py | 8 +- .../modeling/roi_heads/box_head/inference.py | 11 +- 8 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 configs/test_time_aug/e2e_mask_rcnn_R_50_FPN_1x.yaml create mode 100644 maskrcnn_benchmark/engine/bbox_aug.py diff --git a/configs/test_time_aug/e2e_mask_rcnn_R_50_FPN_1x.yaml b/configs/test_time_aug/e2e_mask_rcnn_R_50_FPN_1x.yaml new file mode 100644 index 000000000..d1e4a75b6 --- /dev/null +++ b/configs/test_time_aug/e2e_mask_rcnn_R_50_FPN_1x.yaml @@ -0,0 +1,48 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + WEIGHT: "catalog://ImageNetPretrained/MSRA/R-50" + BACKBONE: + CONV_BODY: "R-50-FPN" + RESNETS: + BACKBONE_OUT_CHANNELS: 256 + RPN: + USE_FPN: True + ANCHOR_STRIDE: (4, 8, 16, 32, 64) + PRE_NMS_TOP_N_TRAIN: 2000 + PRE_NMS_TOP_N_TEST: 1000 + POST_NMS_TOP_N_TEST: 1000 + FPN_POST_NMS_TOP_N_TEST: 1000 + ROI_HEADS: + USE_FPN: True + ROI_BOX_HEAD: + POOLER_RESOLUTION: 7 + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + POOLER_SAMPLING_RATIO: 2 + FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor" + PREDICTOR: "FPNPredictor" + ROI_MASK_HEAD: + POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125) + FEATURE_EXTRACTOR: "MaskRCNNFPNFeatureExtractor" + PREDICTOR: "MaskRCNNC4Predictor" + POOLER_RESOLUTION: 14 + POOLER_SAMPLING_RATIO: 2 + RESOLUTION: 28 + SHARE_BOX_FEATURE_EXTRACTOR: False + MASK_ON: True +DATASETS: + TRAIN: ("coco_2014_train", "coco_2014_valminusminival") + TEST: ("coco_2014_minival",) +DATALOADER: + SIZE_DIVISIBILITY: 32 +SOLVER: + BASE_LR: 0.02 + WEIGHT_DECAY: 0.0001 + STEPS: (60000, 80000) + MAX_ITER: 90000 +TEST: + BBOX_AUG: + ENABLED: True + H_FLIP: True + SCALES: (400, 500, 600, 700, 900, 1000, 1100, 1200) + MAX_SIZE: 2000 + SCALE_H_FLIP: True diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index 37e362cd7..6a8d62ea4 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -427,6 +427,27 @@ # Number of detections per image _C.TEST.DETECTIONS_PER_IMG = 100 +# ---------------------------------------------------------------------------- # +# Test-time augmentations for bounding box detection +# See configs/test_time_aug/e2e_mask_rcnn_R-50-FPN_1x.yaml for an example +# ---------------------------------------------------------------------------- # +_C.TEST.BBOX_AUG = CN() + +# Enable test-time augmentation for bounding box detection if True +_C.TEST.BBOX_AUG.ENABLED = False + +# Horizontal flip at the original scale (id transform) +_C.TEST.BBOX_AUG.H_FLIP = False + +# Each scale is the pixel size of an image's shortest side +_C.TEST.BBOX_AUG.SCALES = () + +# Max pixel size of the longer side +_C.TEST.BBOX_AUG.MAX_SIZE = 4000 + +# Horizontal flip at each scale +_C.TEST.BBOX_AUG.SCALE_H_FLIP = False + # ---------------------------------------------------------------------------- # # Misc options diff --git a/maskrcnn_benchmark/data/build.py b/maskrcnn_benchmark/data/build.py index d2895fd7e..7572fd335 100644 --- a/maskrcnn_benchmark/data/build.py +++ b/maskrcnn_benchmark/data/build.py @@ -10,7 +10,7 @@ from . import datasets as D from . import samplers -from .collate_batch import BatchCollator +from .collate_batch import BatchCollator, BBoxAugCollator from .transforms import build_transforms @@ -150,7 +150,8 @@ def make_data_loader(cfg, is_train=True, is_distributed=False, start_iter=0): DatasetCatalog = paths_catalog.DatasetCatalog dataset_list = cfg.DATASETS.TRAIN if is_train else cfg.DATASETS.TEST - transforms = build_transforms(cfg, is_train) + # If bbox aug is enabled in testing, simply set transforms to None and we will apply transforms later + transforms = None if not is_train and cfg.TEST.BBOX_AUG.ENABLED else build_transforms(cfg, is_train) datasets = build_dataset(dataset_list, transforms, DatasetCatalog, is_train) data_loaders = [] @@ -159,7 +160,8 @@ def make_data_loader(cfg, is_train=True, is_distributed=False, start_iter=0): batch_sampler = make_batch_data_sampler( dataset, sampler, aspect_grouping, images_per_gpu, num_iters, start_iter ) - collator = BatchCollator(cfg.DATALOADER.SIZE_DIVISIBILITY) + collator = BBoxAugCollator() if not is_train and cfg.TEST.BBOX_AUG.ENABLED else \ + BatchCollator(cfg.DATALOADER.SIZE_DIVISIBILITY) num_workers = cfg.DATALOADER.NUM_WORKERS data_loader = torch.utils.data.DataLoader( dataset, diff --git a/maskrcnn_benchmark/data/collate_batch.py b/maskrcnn_benchmark/data/collate_batch.py index a7f034167..56571f18c 100644 --- a/maskrcnn_benchmark/data/collate_batch.py +++ b/maskrcnn_benchmark/data/collate_batch.py @@ -18,3 +18,15 @@ def __call__(self, batch): targets = transposed_batch[1] img_ids = transposed_batch[2] return images, targets, img_ids + + +class BBoxAugCollator(object): + """ + From a list of samples from the dataset, + returns the images and targets. + Images should be converted to batched images in `im_detect_bbox_aug` + """ + + def __call__(self, batch): + return list(zip(*batch)) + diff --git a/maskrcnn_benchmark/data/transforms/transforms.py b/maskrcnn_benchmark/data/transforms/transforms.py index 1c322f8ba..fa1d93934 100644 --- a/maskrcnn_benchmark/data/transforms/transforms.py +++ b/maskrcnn_benchmark/data/transforms/transforms.py @@ -54,9 +54,11 @@ def get_size(self, image_size): return (oh, ow) - def __call__(self, image, target): + def __call__(self, image, target=None): size = self.get_size(image.size) image = F.resize(image, size) + if target is None: + return image target = target.resize(image.size) return image, target @@ -101,8 +103,10 @@ def __init__(self, mean, std, to_bgr255=True): self.std = std self.to_bgr255 = to_bgr255 - def __call__(self, image, target): + def __call__(self, image, target=None): if self.to_bgr255: image = image[[2, 1, 0]] * 255 image = F.normalize(image, mean=self.mean, std=self.std) + if target is None: + return image return image, target diff --git a/maskrcnn_benchmark/engine/bbox_aug.py b/maskrcnn_benchmark/engine/bbox_aug.py new file mode 100644 index 000000000..444416538 --- /dev/null +++ b/maskrcnn_benchmark/engine/bbox_aug.py @@ -0,0 +1,118 @@ +import torch +import torchvision.transforms as TT + +from maskrcnn_benchmark.config import cfg +from maskrcnn_benchmark.data import transforms as T +from maskrcnn_benchmark.structures.image_list import to_image_list +from maskrcnn_benchmark.structures.bounding_box import BoxList +from maskrcnn_benchmark.modeling.roi_heads.box_head.inference import make_roi_box_post_processor + + +def im_detect_bbox_aug(model, images, device): + # Collect detections computed under different transformations + boxlists_ts = [] + for _ in range(len(images)): + boxlists_ts.append([]) + + def add_preds_t(boxlists_t): + for i, boxlist_t in enumerate(boxlists_t): + if len(boxlists_ts[i]) == 0: + # The first one is identity transform, no need to resize the boxlist + boxlists_ts[i].append(boxlist_t) + else: + # Resize the boxlist as the first one + boxlists_ts[i].append(boxlist_t.resize(boxlists_ts[i][0].size)) + + # Compute detections for the original image (identity transform) + boxlists_i = im_detect_bbox( + model, images, cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MAX_SIZE_TEST, device + ) + add_preds_t(boxlists_i) + + # Perform detection on the horizontally flipped image + if cfg.TEST.BBOX_AUG.H_FLIP: + boxlists_hf = im_detect_bbox_hflip( + model, images, cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MAX_SIZE_TEST, device + ) + add_preds_t(boxlists_hf) + + # Compute detections at different scales + for scale in cfg.TEST.BBOX_AUG.SCALES: + max_size = cfg.TEST.BBOX_AUG.MAX_SIZE + boxlists_scl = im_detect_bbox_scale( + model, images, scale, max_size, device + ) + add_preds_t(boxlists_scl) + + if cfg.TEST.BBOX_AUG.SCALE_H_FLIP: + boxlists_scl_hf = im_detect_bbox_scale( + model, images, scale, max_size, device, hflip=True + ) + add_preds_t(boxlists_scl_hf) + + # Merge boxlists detected by different bbox aug params + boxlists = [] + for i, boxlist_ts in enumerate(boxlists_ts): + bbox = torch.cat([boxlist_t.bbox for boxlist_t in boxlist_ts]) + scores = torch.cat([boxlist_t.get_field('scores') for boxlist_t in boxlist_ts]) + boxlist = BoxList(bbox, boxlist_ts[0].size, boxlist_ts[0].mode) + boxlist.add_field('scores', scores) + boxlists.append(boxlist) + + # Apply NMS and limit the final detections + results = [] + post_processor = make_roi_box_post_processor(cfg) + for boxlist in boxlists: + results.append(post_processor.filter_results(boxlist, cfg.MODEL.ROI_BOX_HEAD.NUM_CLASSES)) + + return results + + +def im_detect_bbox(model, images, target_scale, target_max_size, device): + """ + Performs bbox detection on the original image. + """ + transform = TT.Compose([ + T.Resize(target_scale, target_max_size), + TT.ToTensor(), + T.Normalize( + mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr255=cfg.INPUT.TO_BGR255 + ) + ]) + images = [transform(image) for image in images] + images = to_image_list(images, cfg.DATALOADER.SIZE_DIVISIBILITY) + return model(images.to(device)) + + +def im_detect_bbox_hflip(model, images, target_scale, target_max_size, device): + """ + Performs bbox detection on the horizontally flipped image. + Function signature is the same as for im_detect_bbox. + """ + transform = TT.Compose([ + T.Resize(target_scale, target_max_size), + TT.RandomHorizontalFlip(1.0), + TT.ToTensor(), + T.Normalize( + mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr255=cfg.INPUT.TO_BGR255 + ) + ]) + images = [transform(image) for image in images] + images = to_image_list(images, cfg.DATALOADER.SIZE_DIVISIBILITY) + boxlists = model(images.to(device)) + + # Invert the detections computed on the flipped image + boxlists_inv = [boxlist.transpose(0) for boxlist in boxlists] + return boxlists_inv + + +def im_detect_bbox_scale(model, images, target_scale, target_max_size, device, hflip=False): + """ + Computes bbox detections at the given scale. + Returns predictions in the scaled image space. + """ + if hflip: + boxlists_scl = im_detect_bbox_hflip(model, images, target_scale, target_max_size, device) + else: + boxlists_scl = im_detect_bbox(model, images, target_scale, target_max_size, device) + return boxlists_scl diff --git a/maskrcnn_benchmark/engine/inference.py b/maskrcnn_benchmark/engine/inference.py index e125cb877..4c83ed4c5 100644 --- a/maskrcnn_benchmark/engine/inference.py +++ b/maskrcnn_benchmark/engine/inference.py @@ -6,11 +6,13 @@ import torch from tqdm import tqdm +from maskrcnn_benchmark.config import cfg from maskrcnn_benchmark.data.datasets.evaluation import evaluate from ..utils.comm import is_main_process, get_world_size from ..utils.comm import all_gather from ..utils.comm import synchronize from ..utils.timer import Timer, get_time_str +from .bbox_aug import im_detect_bbox_aug def compute_on_dataset(model, data_loader, device, timer=None): @@ -19,11 +21,13 @@ def compute_on_dataset(model, data_loader, device, timer=None): cpu_device = torch.device("cpu") for _, batch in enumerate(tqdm(data_loader)): images, targets, image_ids = batch - images = images.to(device) with torch.no_grad(): if timer: timer.tic() - output = model(images) + if cfg.TEST.BBOX_AUG.ENABLED: + output = im_detect_bbox_aug(model, images, device) + else: + output = model(images.to(device)) if timer: torch.cuda.synchronize() timer.toc() diff --git a/maskrcnn_benchmark/modeling/roi_heads/box_head/inference.py b/maskrcnn_benchmark/modeling/roi_heads/box_head/inference.py index 595a2e616..cc2f4fa85 100644 --- a/maskrcnn_benchmark/modeling/roi_heads/box_head/inference.py +++ b/maskrcnn_benchmark/modeling/roi_heads/box_head/inference.py @@ -22,7 +22,8 @@ def __init__( nms=0.5, detections_per_img=100, box_coder=None, - cls_agnostic_bbox_reg=False + cls_agnostic_bbox_reg=False, + bbox_aug_enabled=False ): """ Arguments: @@ -39,6 +40,7 @@ def __init__( box_coder = BoxCoder(weights=(10., 10., 5., 5.)) self.box_coder = box_coder self.cls_agnostic_bbox_reg = cls_agnostic_bbox_reg + self.bbox_aug_enabled = bbox_aug_enabled def forward(self, x, boxes): """ @@ -79,7 +81,8 @@ def forward(self, x, boxes): ): boxlist = self.prepare_boxlist(boxes_per_img, prob, image_shape) boxlist = boxlist.clip_to_image(remove_empty=False) - boxlist = self.filter_results(boxlist, num_classes) + if not self.bbox_aug_enabled: # If bbox aug is enabled, we will do it later + boxlist = self.filter_results(boxlist, num_classes) results.append(boxlist) return results @@ -156,12 +159,14 @@ def make_roi_box_post_processor(cfg): nms_thresh = cfg.MODEL.ROI_HEADS.NMS detections_per_img = cfg.MODEL.ROI_HEADS.DETECTIONS_PER_IMG cls_agnostic_bbox_reg = cfg.MODEL.CLS_AGNOSTIC_BBOX_REG + bbox_aug_enabled = cfg.TEST.BBOX_AUG.ENABLED postprocessor = PostProcessor( score_thresh, nms_thresh, detections_per_img, box_coder, - cls_agnostic_bbox_reg + cls_agnostic_bbox_reg, + bbox_aug_enabled ) return postprocessor From cd63feb916fc331a22060c6552bfef0b529a9ad6 Mon Sep 17 00:00:00 2001 From: Csaba Botos Date: Fri, 24 May 2019 12:01:44 +0200 Subject: [PATCH 27/52] support empty BinaryMaskList initialization (#780) * fix the bug in segmentation_mask when initializing BinaryMaskList with an empty list * Add empty binary mask handling * Strict indexing in BinaryMaskList - raise error for empty indexing --- .../structures/segmentation_mask.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 060d512b3..364d01eb5 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -52,7 +52,9 @@ def __init__(self, masks, size): # The raw data representation is passed as argument masks = masks.clone() elif isinstance(masks, (list, tuple)): - if isinstance(masks[0], torch.Tensor): + if len(masks) == 0: + masks = torch.empty([0, size[1], size[0]]) # num_instances = 0! + elif isinstance(masks[0], torch.Tensor): masks = torch.stack(masks, dim=2).clone() elif isinstance(masks[0], dict) and "counts" in masks[0]: # RLE interpretation @@ -122,7 +124,7 @@ def resize(self, size): assert height > 0 # Height comes first here! - resized_masks = torch.nn.functional.interpolate( + resized_masks = interpolate( input=self.masks[None].float(), size=(height, width), mode="bilinear", @@ -132,6 +134,9 @@ def resize(self, size): return BinaryMaskList(resized_masks, resized_size) def convert_to_polygon(self): + if self.masks.numel() == 0: + return PolygonList([], self.size) + contours = self._findContours() return PolygonList(contours, self.size) @@ -159,10 +164,9 @@ def __len__(self): return len(self.masks) def __getitem__(self, index): - # Probably it can cause some overhead - # but preserves consistency - masks = self.masks[index].clone() - return BinaryMaskList(masks, self.size) + if self.masks.numel() == 0: + raise RuntimeError("Indexing empty BinaryMaskList") + return BinaryMaskList(self.masks[index], self.size) def __iter__(self): return iter(self.masks) @@ -308,7 +312,7 @@ def __repr__(self): s = self.__class__.__name__ + "(" s += "num_groups={}, ".format(len(self.polygons)) s += "image_width={}, ".format(self.size[0]) - s += "image_height={}, ".format(self.size[1]) + s += "image_height={})".format(self.size[1]) return s From 2ff23c4db4d6541f96faf1f532e8bb6eac53abcd Mon Sep 17 00:00:00 2001 From: Zhang Liliang Date: Fri, 24 May 2019 18:03:06 +0800 Subject: [PATCH 28/52] Update defaults.py (#774) remove `_C.MODEL.BACKBONE.USE_GN = False`, since it is not used in the code. ``` liliang@liliang-ubuntu:~/liliang_learning/maskrcnn-benchmark/maskrcnn_benchmark$ grep -r "USE_GN" Binary file config/__pycache__/defaults.cpython-37.pyc matches config/defaults.py:_C.MODEL.BACKBONE.USE_GN = False config/defaults.py:_C.MODEL.FPN.USE_GN = False config/defaults.py:_C.MODEL.ROI_BOX_HEAD.USE_GN = False config/defaults.py:_C.MODEL.ROI_MASK_HEAD.USE_GN = False Binary file modeling/backbone/__pycache__/backbone.cpython-37.pyc matches modeling/backbone/backbone.py: cfg.MODEL.FPN.USE_GN, cfg.MODEL.FPN.USE_RELU modeling/backbone/backbone.py: cfg.MODEL.FPN.USE_GN, cfg.MODEL.FPN.USE_RELU Binary file modeling/roi_heads/mask_head/__pycache__/roi_mask_feature_extractors.cpython-37.pyc matches modeling/roi_heads/mask_head/roi_mask_feature_extractors.py: use_gn = cfg.MODEL.ROI_MASK_HEAD.USE_GN Binary file modeling/roi_heads/box_head/__pycache__/roi_box_feature_extractors.cpython-37.pyc matches modeling/roi_heads/box_head/roi_box_feature_extractors.py: use_gn = cfg.MODEL.ROI_BOX_HEAD.USE_GN modeling/roi_heads/box_head/roi_box_feature_extractors.py: use_gn = cfg.MODEL.ROI_BOX_HEAD.USE_GN ``` --- maskrcnn_benchmark/config/defaults.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index 6a8d62ea4..beae4070a 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -98,8 +98,6 @@ # Add StopGrad at a specified stage so the bottom layers are frozen _C.MODEL.BACKBONE.FREEZE_CONV_BODY_AT = 2 -# GN for backbone -_C.MODEL.BACKBONE.USE_GN = False # ---------------------------------------------------------------------------- # From 3d2e2b3b97e9fb28d2e7cc0315cbd1a45d517bfc Mon Sep 17 00:00:00 2001 From: hcx1231 <32597852+hcx1231@users.noreply.github.com> Date: Fri, 24 May 2019 18:08:37 +0800 Subject: [PATCH 29/52] keep the resize function the same in test time the same with training time (#798) --- demo/predictor.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/demo/predictor.py b/demo/predictor.py index b152fda8d..4de83d067 100644 --- a/demo/predictor.py +++ b/demo/predictor.py @@ -10,7 +10,38 @@ from maskrcnn_benchmark import layers as L from maskrcnn_benchmark.utils import cv2_util +class Resize(object): + def __init__(self, min_size, max_size): + self.min_size = min_size + self.max_size = max_size + + # modified from torchvision to add support for max size + def get_size(self, image_size): + w, h = image_size + size = self.min_size + max_size = self.max_size + if max_size is not None: + min_original_size = float(min((w, h))) + max_original_size = float(max((w, h))) + if max_original_size / min_original_size * size > max_size: + size = int(round(max_size * min_original_size / max_original_size)) + + if (w <= h and w == size) or (h <= w and h == size): + return (h, w) + + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + + return (oh, ow) + def __call__(self, image): + size = self.get_size(image.size) + image = F.resize(image, size) + return image class COCODemo(object): # COCO categories for pretty print CATEGORIES = [ @@ -147,11 +178,12 @@ def build_transform(self): normalize_transform = T.Normalize( mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD ) - + min_size = cfg.INPUT.MIN_SIZE_TEST + max_size = cfg.INPUT.MAX_SIZE_TEST transform = T.Compose( [ T.ToPILImage(), - T.Resize(self.min_image_size), + Resize(min_size, max_size), T.ToTensor(), to_bgr_transform, normalize_transform, From 44fb065b9d5faeb9ca8157389c499e006de2a8c9 Mon Sep 17 00:00:00 2001 From: AnnaAraslanova Date: Fri, 24 May 2019 13:09:52 +0300 Subject: [PATCH 30/52] Add requests for compatibility with the demo notebook (#655) * Add requests for compatibility with the demo notebook * Add requests for compatibility with the demo notebook --- docker/Dockerfile | 2 +- docker/docker-jupyter/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4242117db..762441fe2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,7 +29,7 @@ ENV PATH=$CONDA_PREFIX/bin:$PATH ENV CONDA_AUTO_UPDATE_CONDA=false RUN conda install -y ipython -RUN pip install ninja yacs cython matplotlib opencv-python tqdm +RUN pip install requests ninja yacs cython matplotlib opencv-python tqdm # Install PyTorch 1.0 Nightly ARG CUDA diff --git a/docker/docker-jupyter/Dockerfile b/docker/docker-jupyter/Dockerfile index 323727195..bf957b636 100644 --- a/docker/docker-jupyter/Dockerfile +++ b/docker/docker-jupyter/Dockerfile @@ -28,7 +28,7 @@ ENV PATH=$CONDA_PREFIX/bin:$PATH ENV CONDA_AUTO_UPDATE_CONDA=false RUN conda install -y ipython -RUN pip install ninja yacs cython matplotlib jupyter +RUN pip install requests ninja yacs cython matplotlib jupyter # Install PyTorch 1.0 Nightly and OpenCV RUN conda install -y pytorch-nightly -c pytorch \ From c58550cf9a5bb0fcc253bfa75e50c54e6cf19651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernhard=20Sch=C3=A4fer?= Date: Fri, 24 May 2019 14:09:35 +0300 Subject: [PATCH 31/52] minor fix: incorrect assert string message formatting (#631) * do not split strings so that format() works as expected * address flake8 indentation issue --- .gitignore | 3 +++ maskrcnn_benchmark/data/build.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 4124b2bf1..d52f92b83 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ dist/ # Pycharm editor settings .idea +# MacOS +.DS_Store + # project dirs /datasets /models diff --git a/maskrcnn_benchmark/data/build.py b/maskrcnn_benchmark/data/build.py index 7572fd335..b0ce3c348 100644 --- a/maskrcnn_benchmark/data/build.py +++ b/maskrcnn_benchmark/data/build.py @@ -110,8 +110,8 @@ def make_data_loader(cfg, is_train=True, is_distributed=False, start_iter=0): images_per_batch = cfg.SOLVER.IMS_PER_BATCH assert ( images_per_batch % num_gpus == 0 - ), "SOLVER.IMS_PER_BATCH ({}) must be divisible by the number " - "of GPUs ({}) used.".format(images_per_batch, num_gpus) + ), "SOLVER.IMS_PER_BATCH ({}) must be divisible by the number of GPUs ({}) used.".format( + images_per_batch, num_gpus) images_per_gpu = images_per_batch // num_gpus shuffle = True num_iters = cfg.SOLVER.MAX_ITER @@ -119,8 +119,8 @@ def make_data_loader(cfg, is_train=True, is_distributed=False, start_iter=0): images_per_batch = cfg.TEST.IMS_PER_BATCH assert ( images_per_batch % num_gpus == 0 - ), "TEST.IMS_PER_BATCH ({}) must be divisible by the number " - "of GPUs ({}) used.".format(images_per_batch, num_gpus) + ), "TEST.IMS_PER_BATCH ({}) must be divisible by the number of GPUs ({}) used.".format( + images_per_batch, num_gpus) images_per_gpu = images_per_batch // num_gpus shuffle = False if not is_distributed else True num_iters = None From cd35ff062f883bf3ae6f886f96d266da31b7310e Mon Sep 17 00:00:00 2001 From: Ammon Perkes Date: Fri, 24 May 2019 16:33:15 -0400 Subject: [PATCH 32/52] Update predictor.py (#820) Fixed missing line importing F (torchvision.transforms.functional) used in line 43 --- demo/predictor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/predictor.py b/demo/predictor.py index 4de83d067..fa663c7e4 100644 --- a/demo/predictor.py +++ b/demo/predictor.py @@ -2,7 +2,7 @@ import cv2 import torch from torchvision import transforms as T - +from torchvision.transforms import functional as F from maskrcnn_benchmark.modeling.detector import build_detection_model from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer from maskrcnn_benchmark.structures.image_list import to_image_list From d7e3c65177a399049026bed6bb88a3f7364f72fa Mon Sep 17 00:00:00 2001 From: kangsir Date: Sat, 25 May 2019 15:00:10 +0800 Subject: [PATCH 33/52] enable inference on cpu (#821) now can inference on cpu by setting MODEL.DEVICE cpu --- maskrcnn_benchmark/engine/inference.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/engine/inference.py b/maskrcnn_benchmark/engine/inference.py index 4c83ed4c5..82d0abb6d 100644 --- a/maskrcnn_benchmark/engine/inference.py +++ b/maskrcnn_benchmark/engine/inference.py @@ -29,7 +29,8 @@ def compute_on_dataset(model, data_loader, device, timer=None): else: output = model(images.to(device)) if timer: - torch.cuda.synchronize() + if not cfg.MODEL.DEVICE == 'cpu': + torch.cuda.synchronize() timer.toc() output = [o.to(cpu_device) for o in output] results_dict.update( From 4c6cd1a4bb55c656cd853ca6b41691587a72ee2d Mon Sep 17 00:00:00 2001 From: hcx1231 <32597852+hcx1231@users.noreply.github.com> Date: Wed, 29 May 2019 03:56:36 +0800 Subject: [PATCH 34/52] add vertical flip (#818) * keep the resize function the same in test time the same with training time * add vertical flip --- maskrcnn_benchmark/config/defaults.py | 1 + maskrcnn_benchmark/data/transforms/build.py | 9 ++++++--- maskrcnn_benchmark/data/transforms/transforms.py | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/config/defaults.py b/maskrcnn_benchmark/config/defaults.py index beae4070a..65fbdaddd 100644 --- a/maskrcnn_benchmark/config/defaults.py +++ b/maskrcnn_benchmark/config/defaults.py @@ -60,6 +60,7 @@ _C.INPUT.SATURATION = 0.0 _C.INPUT.HUE = 0.0 +_C.INPUT.VERTICAL_FLIP_PROB_TRAIN = 0.0 # ----------------------------------------------------------------------------- # Dataset diff --git a/maskrcnn_benchmark/data/transforms/build.py b/maskrcnn_benchmark/data/transforms/build.py index 88aa975b6..52385ea7d 100644 --- a/maskrcnn_benchmark/data/transforms/build.py +++ b/maskrcnn_benchmark/data/transforms/build.py @@ -6,7 +6,8 @@ def build_transforms(cfg, is_train=True): if is_train: min_size = cfg.INPUT.MIN_SIZE_TRAIN max_size = cfg.INPUT.MAX_SIZE_TRAIN - flip_prob = 0.5 # cfg.INPUT.FLIP_PROB_TRAIN + flip_horizontal_prob = 0.5 # cfg.INPUT.FLIP_PROB_TRAIN + flip_vertical_prob = cfg.INPUT.VERTICAL_FLIP_PROB_TRAIN brightness = cfg.INPUT.BRIGHTNESS contrast = cfg.INPUT.CONTRAST saturation = cfg.INPUT.SATURATION @@ -14,7 +15,8 @@ def build_transforms(cfg, is_train=True): else: min_size = cfg.INPUT.MIN_SIZE_TEST max_size = cfg.INPUT.MAX_SIZE_TEST - flip_prob = 0 + flip_horizontal_prob = 0.0 + flip_vertical_prob = 0.0 brightness = 0.0 contrast = 0.0 saturation = 0.0 @@ -35,7 +37,8 @@ def build_transforms(cfg, is_train=True): [ color_jitter, T.Resize(min_size, max_size), - T.RandomHorizontalFlip(flip_prob), + T.RandomHorizontalFlip(flip_horizontal_prob), + T.RandomVerticalFlip(flip_vertical_prob), T.ToTensor(), normalize_transform, ] diff --git a/maskrcnn_benchmark/data/transforms/transforms.py b/maskrcnn_benchmark/data/transforms/transforms.py index fa1d93934..2d37dc72f 100644 --- a/maskrcnn_benchmark/data/transforms/transforms.py +++ b/maskrcnn_benchmark/data/transforms/transforms.py @@ -73,6 +73,15 @@ def __call__(self, image, target): target = target.transpose(0) return image, target +class RandomVerticalFlip(object): + def __init__(self, prob=0.5): + self.prob = prob + + def __call__(self, image, target): + if random.random() < self.prob: + image = F.vflip(image) + target = target.transpose(1) + return image, target class ColorJitter(object): def __init__(self, From d2698472fc7b07ebf8580604bc86037855e68d45 Mon Sep 17 00:00:00 2001 From: Ouail Date: Wed, 29 May 2019 20:29:32 +0200 Subject: [PATCH 35/52] WorkAround for `ValueError: No ground-truth boxes available for one of the images during training` (#836) * Update setup.py * add a FORCE_CUDA flag Following discussion [here](https://github.com/facebookresearch/maskrcnn-benchmark/issues/167), this seemed the best solution * Update Dockerfile * Update setup.py * add FORCE_CUDA as an ARG * modified: docker/Dockerfile modified: setup.py * small fix to readme of demo * remove test print * keep ARG_CUDA * remove env value and use the one from ARG * keep same formatting as source * change proposed by @miguelvr * Update INSTALL.md * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * Update setup.py --- maskrcnn_benchmark/engine/trainer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/maskrcnn_benchmark/engine/trainer.py b/maskrcnn_benchmark/engine/trainer.py index 281d91339..560b63e1c 100644 --- a/maskrcnn_benchmark/engine/trainer.py +++ b/maskrcnn_benchmark/engine/trainer.py @@ -55,6 +55,10 @@ def do_train( start_training_time = time.time() end = time.time() for iteration, (images, targets, _) in enumerate(data_loader, start_iter): + + if any(len(target) < 1 for target in targets): + logger.error(f"Iteration={iteration + 1} || Image Ids used for training {_} || targets Length={[len(target) for target in targets]}" ) + continue data_time = time.time() - end iteration = iteration + 1 arguments["iteration"] = iteration From 7338be22f26a47ca0bb13ef7f24077964e28cc73 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 31 May 2019 20:02:17 +0200 Subject: [PATCH 36/52] Save full configuration in output dir (#835) * Merge branch 'master' of /home/braincreator/projects/maskrcnn-benchmark with conflicts. * update Dockerfile * save config in output dir * replace string format with os.path.join --- maskrcnn_benchmark/utils/miscellaneous.py | 7 +++++++ tools/train_net.py | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/utils/miscellaneous.py b/maskrcnn_benchmark/utils/miscellaneous.py index db9a8b367..ecd3ef6a2 100644 --- a/maskrcnn_benchmark/utils/miscellaneous.py +++ b/maskrcnn_benchmark/utils/miscellaneous.py @@ -1,6 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. import errno import os +from .comm import is_main_process def mkdir(path): @@ -9,3 +10,9 @@ def mkdir(path): except OSError as e: if e.errno != errno.EEXIST: raise + + +def save_config(cfg, path): + if is_main_process(): + with open(path, 'w') as f: + f.write(cfg.dump()) diff --git a/tools/train_net.py b/tools/train_net.py index 9f4761b3f..3468fbb4a 100644 --- a/tools/train_net.py +++ b/tools/train_net.py @@ -23,7 +23,7 @@ from maskrcnn_benchmark.utils.comm import synchronize, get_rank from maskrcnn_benchmark.utils.imports import import_file from maskrcnn_benchmark.utils.logger import setup_logger -from maskrcnn_benchmark.utils.miscellaneous import mkdir +from maskrcnn_benchmark.utils.miscellaneous import mkdir, save_config # See if we can use apex.DistributedDataParallel instead of the torch default, # and enable mixed-precision via apex.amp @@ -176,6 +176,11 @@ def main(): logger.info(config_str) logger.info("Running with config:\n{}".format(cfg)) + output_config_path = os.path.join(cfg.OUTPUT_DIR, 'config.yml') + logger.info("Saving config into: {}".format(output_config_path)) + # save overloaded model config in the output directory + save_config(cfg, output_config_path) + model = train(cfg, args.local_rank, args.distributed) if not args.skip_test: From d802413d37fa0503cfd1b7bd791aa76ffab489df Mon Sep 17 00:00:00 2001 From: Gu Wang Date: Mon, 3 Jun 2019 17:48:40 +0800 Subject: [PATCH 37/52] fix cv2 compatibility between versions 3 and 4; ignore vscode; minor flake8 fix (#846) --- .gitignore | 3 + .../docker-jupyter/jupyter_notebook_config.py | 16 +- maskrcnn_benchmark/data/datasets/voc.py | 8 +- maskrcnn_benchmark/layers/dcn/__init__.py | 4 +- .../layers/dcn/deform_conv_func.py | 156 +++++++++--------- .../layers/dcn/deform_pool_func.py | 42 ++--- maskrcnn_benchmark/layers/misc.py | 22 +-- .../modeling/backbone/resnet.py | 10 +- maskrcnn_benchmark/modeling/make_layers.py | 38 ++--- .../modeling/roi_heads/box_head/loss.py | 14 +- .../structures/segmentation_mask.py | 4 +- tests/test_segmentation_mask.py | 5 - 12 files changed, 160 insertions(+), 162 deletions(-) diff --git a/.gitignore b/.gitignore index d52f92b83..7fe9a046b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ dist/ # Pycharm editor settings .idea +# vscode editor settings +.vscode + # MacOS .DS_Store diff --git a/docker/docker-jupyter/jupyter_notebook_config.py b/docker/docker-jupyter/jupyter_notebook_config.py index bd5494812..e8fbe7de4 100644 --- a/docker/docker-jupyter/jupyter_notebook_config.py +++ b/docker/docker-jupyter/jupyter_notebook_config.py @@ -1,7 +1,7 @@ import os from IPython.lib import passwd -#c = c # pylint:disable=undefined-variable +# c = c # pylint:disable=undefined-variable c = get_config() c.NotebookApp.ip = '0.0.0.0' c.NotebookApp.port = int(os.getenv('PORT', 8888)) @@ -9,10 +9,10 @@ # sets a password if PASSWORD is set in the environment if 'PASSWORD' in os.environ: - password = os.environ['PASSWORD'] - if password: - c.NotebookApp.password = passwd(password) - else: - c.NotebookApp.password = '' - c.NotebookApp.token = '' - del os.environ['PASSWORD'] + password = os.environ['PASSWORD'] + if password: + c.NotebookApp.password = passwd(password) + else: + c.NotebookApp.password = '' + c.NotebookApp.token = '' + del os.environ['PASSWORD'] diff --git a/maskrcnn_benchmark/data/datasets/voc.py b/maskrcnn_benchmark/data/datasets/voc.py index 459985bd1..ad20a8721 100644 --- a/maskrcnn_benchmark/data/datasets/voc.py +++ b/maskrcnn_benchmark/data/datasets/voc.py @@ -89,7 +89,7 @@ def _preprocess_annotation(self, target): gt_classes = [] difficult_boxes = [] TO_REMOVE = 1 - + for obj in target.iter("object"): difficult = int(obj.find("difficult").text) == 1 if not self.keep_difficult and difficult: @@ -99,9 +99,9 @@ def _preprocess_annotation(self, target): # Make pixel indexes 0-based # Refer to "https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/datasets/pascal_voc.py#L208-L211" box = [ - bb.find("xmin").text, - bb.find("ymin").text, - bb.find("xmax").text, + bb.find("xmin").text, + bb.find("ymin").text, + bb.find("xmax").text, bb.find("ymax").text, ] bndbox = tuple( diff --git a/maskrcnn_benchmark/layers/dcn/__init__.py b/maskrcnn_benchmark/layers/dcn/__init__.py index 22fe18ff3..bb5af25d4 100644 --- a/maskrcnn_benchmark/layers/dcn/__init__.py +++ b/maskrcnn_benchmark/layers/dcn/__init__.py @@ -1,3 +1,3 @@ -# +# # Copied From [mmdetection](https://github.com/open-mmlab/mmdetection/tree/master/mmdet/ops/dcn) -# \ No newline at end of file +# diff --git a/maskrcnn_benchmark/layers/dcn/deform_conv_func.py b/maskrcnn_benchmark/layers/dcn/deform_conv_func.py index a276a05fe..388bacf12 100644 --- a/maskrcnn_benchmark/layers/dcn/deform_conv_func.py +++ b/maskrcnn_benchmark/layers/dcn/deform_conv_func.py @@ -10,15 +10,15 @@ class DeformConvFunction(Function): @staticmethod def forward( - ctx, - input, - offset, + ctx, + input, + offset, weight, - stride=1, - padding=0, - dilation=1, - groups=1, - deformable_groups=1, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, im2col_step=64 ): if input is not None and input.dim() != 4: @@ -47,21 +47,21 @@ def forward( assert (input.shape[0] % cur_im2col_step) == 0, 'im2col step must divide batchsize' _C.deform_conv_forward( - input, - weight, - offset, - output, - ctx.bufs_[0], + input, + weight, + offset, + output, + ctx.bufs_[0], ctx.bufs_[1], - weight.size(3), - weight.size(2), - ctx.stride[1], + weight.size(3), + weight.size(2), + ctx.stride[1], ctx.stride[0], - ctx.padding[1], - ctx.padding[0], + ctx.padding[1], + ctx.padding[0], ctx.dilation[1], - ctx.dilation[0], - ctx.groups, + ctx.dilation[0], + ctx.groups, ctx.deformable_groups, cur_im2col_step ) @@ -85,22 +85,22 @@ def backward(ctx, grad_output): grad_input = torch.zeros_like(input) grad_offset = torch.zeros_like(offset) _C.deform_conv_backward_input( - input, - offset, - grad_output, + input, + offset, + grad_output, grad_input, - grad_offset, - weight, - ctx.bufs_[0], + grad_offset, + weight, + ctx.bufs_[0], weight.size(3), - weight.size(2), - ctx.stride[1], + weight.size(2), + ctx.stride[1], ctx.stride[0], - ctx.padding[1], - ctx.padding[0], + ctx.padding[1], + ctx.padding[0], ctx.dilation[1], - ctx.dilation[0], - ctx.groups, + ctx.dilation[0], + ctx.groups, ctx.deformable_groups, cur_im2col_step ) @@ -108,22 +108,22 @@ def backward(ctx, grad_output): if ctx.needs_input_grad[2]: grad_weight = torch.zeros_like(weight) _C.deform_conv_backward_parameters( - input, - offset, + input, + offset, grad_output, - grad_weight, - ctx.bufs_[0], - ctx.bufs_[1], + grad_weight, + ctx.bufs_[0], + ctx.bufs_[1], weight.size(3), - weight.size(2), - ctx.stride[1], + weight.size(2), + ctx.stride[1], ctx.stride[0], - ctx.padding[1], - ctx.padding[0], + ctx.padding[1], + ctx.padding[0], ctx.dilation[1], - ctx.dilation[0], - ctx.groups, - ctx.deformable_groups, + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, 1, cur_im2col_step ) @@ -180,24 +180,24 @@ def forward( ModulatedDeformConvFunction._infer_shape(ctx, input, weight)) ctx._bufs = [input.new_empty(0), input.new_empty(0)] _C.modulated_deform_conv_forward( - input, - weight, - bias, - ctx._bufs[0], - offset, - mask, + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, output, - ctx._bufs[1], - weight.shape[2], - weight.shape[3], + ctx._bufs[1], + weight.shape[2], + weight.shape[3], + ctx.stride, ctx.stride, - ctx.stride, - ctx.padding, - ctx.padding, - ctx.dilation, + ctx.padding, + ctx.padding, ctx.dilation, - ctx.groups, - ctx.deformable_groups, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, ctx.with_bias ) return output @@ -214,29 +214,29 @@ def backward(ctx, grad_output): grad_weight = torch.zeros_like(weight) grad_bias = torch.zeros_like(bias) _C.modulated_deform_conv_backward( - input, - weight, - bias, - ctx._bufs[0], - offset, - mask, + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, ctx._bufs[1], - grad_input, - grad_weight, - grad_bias, - grad_offset, + grad_input, + grad_weight, + grad_bias, + grad_offset, grad_mask, - grad_output, - weight.shape[2], - weight.shape[3], + grad_output, + weight.shape[2], + weight.shape[3], + ctx.stride, ctx.stride, - ctx.stride, - ctx.padding, - ctx.padding, - ctx.dilation, + ctx.padding, + ctx.padding, + ctx.dilation, ctx.dilation, - ctx.groups, - ctx.deformable_groups, + ctx.groups, + ctx.deformable_groups, ctx.with_bias ) if not ctx.with_bias: diff --git a/maskrcnn_benchmark/layers/dcn/deform_pool_func.py b/maskrcnn_benchmark/layers/dcn/deform_pool_func.py index 2f7810b23..e083b002e 100644 --- a/maskrcnn_benchmark/layers/dcn/deform_pool_func.py +++ b/maskrcnn_benchmark/layers/dcn/deform_pool_func.py @@ -39,18 +39,18 @@ def forward( output = data.new_empty(n, out_channels, out_size, out_size) output_count = data.new_empty(n, out_channels, out_size, out_size) _C.deform_psroi_pooling_forward( - data, - rois, - offset, - output, - output_count, + data, + rois, + offset, + output, + output_count, ctx.no_trans, - ctx.spatial_scale, - ctx.out_channels, - ctx.group_size, + ctx.spatial_scale, + ctx.out_channels, + ctx.group_size, ctx.out_size, - ctx.part_size, - ctx.sample_per_part, + ctx.part_size, + ctx.sample_per_part, ctx.trans_std ) @@ -73,19 +73,19 @@ def backward(ctx, grad_output): grad_offset = torch.zeros_like(offset) _C.deform_psroi_pooling_backward( - grad_output, - data, - rois, - offset, - output_count, + grad_output, + data, + rois, + offset, + output_count, grad_input, - grad_offset, - ctx.no_trans, - ctx.spatial_scale, + grad_offset, + ctx.no_trans, + ctx.spatial_scale, ctx.out_channels, - ctx.group_size, - ctx.out_size, - ctx.part_size, + ctx.group_size, + ctx.out_size, + ctx.part_size, ctx.sample_per_part, ctx.trans_std ) diff --git a/maskrcnn_benchmark/layers/misc.py b/maskrcnn_benchmark/layers/misc.py index b64f23840..871132419 100644 --- a/maskrcnn_benchmark/layers/misc.py +++ b/maskrcnn_benchmark/layers/misc.py @@ -114,12 +114,12 @@ def _output_size(dim): class DFConv2d(nn.Module): """Deformable convolutional layer""" def __init__( - self, - in_channels, - out_channels, - with_modulated_dcn=True, - kernel_size=3, - stride=1, + self, + in_channels, + out_channels, + with_modulated_dcn=True, + kernel_size=3, + stride=1, groups=1, dilation=1, deformable_groups=1, @@ -156,7 +156,7 @@ def __init__( padding=padding, groups=1, dilation=dilation - ) + ) for l in [self.offset,]: nn.init.kaiming_uniform_(l.weight, a=1) torch.nn.init.constant_(l.bias, 0.) @@ -192,10 +192,10 @@ def forward(self, x): output_shape = [ (i + 2 * p - (di * (k - 1) + 1)) // d + 1 for i, p, di, k, d in zip( - x.shape[-2:], - self.padding, - self.dilation, - self.kernel_size, + x.shape[-2:], + self.padding, + self.dilation, + self.kernel_size, self.stride ) ] diff --git a/maskrcnn_benchmark/modeling/backbone/resnet.py b/maskrcnn_benchmark/modeling/backbone/resnet.py index fc02dc1e8..147531091 100644 --- a/maskrcnn_benchmark/modeling/backbone/resnet.py +++ b/maskrcnn_benchmark/modeling/backbone/resnet.py @@ -288,11 +288,11 @@ def __init__( deformable_groups = dcn_config.get("deformable_groups", 1) with_modulated_dcn = dcn_config.get("with_modulated_dcn", False) self.conv2 = DFConv2d( - bottleneck_channels, - bottleneck_channels, - with_modulated_dcn=with_modulated_dcn, - kernel_size=3, - stride=stride_3x3, + bottleneck_channels, + bottleneck_channels, + with_modulated_dcn=with_modulated_dcn, + kernel_size=3, + stride=stride_3x3, groups=num_groups, dilation=dilation, deformable_groups=deformable_groups, diff --git a/maskrcnn_benchmark/modeling/make_layers.py b/maskrcnn_benchmark/modeling/make_layers.py index 74e56b0e2..049aee6d1 100644 --- a/maskrcnn_benchmark/modeling/make_layers.py +++ b/maskrcnn_benchmark/modeling/make_layers.py @@ -34,29 +34,29 @@ def group_norm(out_channels, affine=True, divisor=1): num_groups = cfg.MODEL.GROUP_NORM.NUM_GROUPS // divisor eps = cfg.MODEL.GROUP_NORM.EPSILON # default: 1e-5 return torch.nn.GroupNorm( - get_group_gn(out_channels, dim_per_gp, num_groups), - out_channels, - eps, + get_group_gn(out_channels, dim_per_gp, num_groups), + out_channels, + eps, affine ) def make_conv3x3( - in_channels, - out_channels, - dilation=1, - stride=1, + in_channels, + out_channels, + dilation=1, + stride=1, use_gn=False, use_relu=False, kaiming_init=True ): conv = Conv2d( - in_channels, - out_channels, - kernel_size=3, - stride=stride, - padding=dilation, - dilation=dilation, + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, bias=False if use_gn else True ) if kaiming_init: @@ -97,12 +97,12 @@ def make_conv( in_channels, out_channels, kernel_size, stride=1, dilation=1 ): conv = Conv2d( - in_channels, - out_channels, - kernel_size=kernel_size, - stride=stride, - padding=dilation * (kernel_size - 1) // 2, - dilation=dilation, + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=dilation * (kernel_size - 1) // 2, + dilation=dilation, bias=False if use_gn else True ) # Caffe2 implementation uses XavierFill, which in fact diff --git a/maskrcnn_benchmark/modeling/roi_heads/box_head/loss.py b/maskrcnn_benchmark/modeling/roi_heads/box_head/loss.py index 9f2771d02..a1fdd2308 100644 --- a/maskrcnn_benchmark/modeling/roi_heads/box_head/loss.py +++ b/maskrcnn_benchmark/modeling/roi_heads/box_head/loss.py @@ -19,10 +19,10 @@ class FastRCNNLossComputation(object): """ def __init__( - self, - proposal_matcher, - fg_bg_sampler, - box_coder, + self, + proposal_matcher, + fg_bg_sampler, + box_coder, cls_agnostic_bbox_reg=False ): """ @@ -184,9 +184,9 @@ def make_roi_box_loss_evaluator(cfg): cls_agnostic_bbox_reg = cfg.MODEL.CLS_AGNOSTIC_BBOX_REG loss_evaluator = FastRCNNLossComputation( - matcher, - fg_bg_sampler, - box_coder, + matcher, + fg_bg_sampler, + box_coder, cls_agnostic_bbox_reg ) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 364d01eb5..4547398a0 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -3,7 +3,7 @@ import torch import numpy as np from maskrcnn_benchmark.layers.misc import interpolate - +from maskrcnn_benchmark.utils import cv2_util import pycocotools.mask as mask_utils # transpose @@ -148,7 +148,7 @@ def _findContours(self): masks = self.masks.detach().numpy() for mask in masks: mask = cv2.UMat(mask) - contour, hierarchy = cv2.findContours( + contour, hierarchy = cv2_util.findContours( mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_L1 ) diff --git a/tests/test_segmentation_mask.py b/tests/test_segmentation_mask.py index d01ed9452..3f70ed551 100644 --- a/tests/test_segmentation_mask.py +++ b/tests/test_segmentation_mask.py @@ -20,13 +20,11 @@ def __init__(self, method_name='runTest'): self.P = SegmentationMask(poly, size, 'poly') self.M = SegmentationMask(poly, size, 'poly').convert('mask') - def L1(self, A, B): diff = A.get_mask_tensor() - B.get_mask_tensor() diff = torch.sum(torch.abs(diff.float())).item() return diff - def test_convert(self): M_hat = self.M.convert('poly').convert('mask') P_hat = self.P.convert('mask').convert('poly') @@ -37,13 +35,11 @@ def test_convert(self): self.assertTrue(diff_mask <= 8169.) self.assertTrue(diff_poly <= 8169.) - def test_crop(self): box = [400, 250, 500, 300] # xyxy diff = self.L1(self.M.crop(box), self.P.crop(box)) self.assertTrue(diff <= 1.) - def test_resize(self): new_size = 50, 25 M_hat = self.M.resize(new_size) @@ -55,7 +51,6 @@ def test_resize(self): self.assertTrue(self.M.size != M_hat.size) self.assertTrue(diff <= 255.) - def test_transpose(self): FLIP_LEFT_RIGHT = 0 FLIP_TOP_BOTTOM = 1 From 614e42759652e66103c21808e82265b17cc97214 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 3 Jun 2019 11:49:17 +0200 Subject: [PATCH 38/52] Save labels in output directory (#842) * Merge branch 'master' of /home/braincreator/projects/maskrcnn-benchmark with conflicts. * update Dockerfile * save labels to output dir * save labels on main process only --- maskrcnn_benchmark/data/build.py | 5 +++++ maskrcnn_benchmark/data/datasets/coco.py | 2 ++ maskrcnn_benchmark/data/datasets/voc.py | 1 + maskrcnn_benchmark/utils/miscellaneous.py | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+) diff --git a/maskrcnn_benchmark/data/build.py b/maskrcnn_benchmark/data/build.py index b0ce3c348..8cf610ade 100644 --- a/maskrcnn_benchmark/data/build.py +++ b/maskrcnn_benchmark/data/build.py @@ -6,6 +6,7 @@ import torch.utils.data from maskrcnn_benchmark.utils.comm import get_world_size from maskrcnn_benchmark.utils.imports import import_file +from maskrcnn_benchmark.utils.miscellaneous import save_labels from . import datasets as D from . import samplers @@ -154,6 +155,10 @@ def make_data_loader(cfg, is_train=True, is_distributed=False, start_iter=0): transforms = None if not is_train and cfg.TEST.BBOX_AUG.ENABLED else build_transforms(cfg, is_train) datasets = build_dataset(dataset_list, transforms, DatasetCatalog, is_train) + if is_train: + # save category_id to label name mapping + save_labels(datasets, cfg.OUTPUT_DIR) + data_loaders = [] for dataset in datasets: sampler = make_data_sampler(dataset, shuffle, is_distributed) diff --git a/maskrcnn_benchmark/data/datasets/coco.py b/maskrcnn_benchmark/data/datasets/coco.py index cd9fc835e..cc10f29d1 100644 --- a/maskrcnn_benchmark/data/datasets/coco.py +++ b/maskrcnn_benchmark/data/datasets/coco.py @@ -54,6 +54,8 @@ def __init__( ids.append(img_id) self.ids = ids + self.categories = {cat['id']: cat['name'] for cat in self.coco.cats.values()} + self.json_category_id_to_contiguous_id = { v: i + 1 for i, v in enumerate(self.coco.getCatIds()) } diff --git a/maskrcnn_benchmark/data/datasets/voc.py b/maskrcnn_benchmark/data/datasets/voc.py index ad20a8721..ab4075ec5 100644 --- a/maskrcnn_benchmark/data/datasets/voc.py +++ b/maskrcnn_benchmark/data/datasets/voc.py @@ -57,6 +57,7 @@ def __init__(self, data_dir, split, use_difficult=False, transforms=None): cls = PascalVOCDataset.CLASSES self.class_to_ind = dict(zip(cls, range(len(cls)))) + self.categories = dict(zip(range(len(cls)), cls)) def __getitem__(self, index): img_id = self.ids[index] diff --git a/maskrcnn_benchmark/utils/miscellaneous.py b/maskrcnn_benchmark/utils/miscellaneous.py index ecd3ef6a2..ce1c279bf 100644 --- a/maskrcnn_benchmark/utils/miscellaneous.py +++ b/maskrcnn_benchmark/utils/miscellaneous.py @@ -1,5 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. import errno +import json +import logging import os from .comm import is_main_process @@ -12,6 +14,25 @@ def mkdir(path): raise +def save_labels(dataset_list, output_dir): + if is_main_process(): + logger = logging.getLogger(__name__) + + ids_to_labels = {} + for dataset in dataset_list: + if hasattr(dataset, 'categories'): + ids_to_labels.update(dataset.categories) + else: + logger.warning("Dataset [{}] has no categories attribute, labels.json file won't be created".format( + dataset.__class__.__name__)) + + if ids_to_labels: + labels_file = os.path.join(output_dir, 'labels.json') + logger.info("Saving labels mapping into {}".format(labels_file)) + with open(labels_file, 'w') as f: + json.dump(ids_to_labels, f, indent=2) + + def save_config(cfg, path): if is_main_process(): with open(path, 'w') as f: From 9e25cd5880250f68cfc1335721fb33ae766abf09 Mon Sep 17 00:00:00 2001 From: Khoa Ho <25312735+khoa-ho@users.noreply.github.com> Date: Mon, 3 Jun 2019 02:50:02 -0700 Subject: [PATCH 39/52] Add mixed precision training instruction (#837) * Add mixed precision training instruction Highlight the model support for mixed precision training and add instruction to enable it on README * Add link to in-depth MP training guide --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index baa7c6bc4..18382906f 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ creating detection and segmentation models using PyTorch 1.0. - **Very fast**: up to **2x** faster than [Detectron](https://github.com/facebookresearch/Detectron) and **30%** faster than [mmdetection](https://github.com/open-mmlab/mmdetection) during training. See [MODEL_ZOO.md](MODEL_ZOO.md) for more details. - **Memory efficient:** uses roughly 500MB less GPU memory than mmdetection during training - **Multi-GPU training and inference** +- **Mixed precision training:** trains faster with less GPU memory on [NVIDIA tensor cores](https://developer.nvidia.com/tensor-cores). - **Batched inference:** can perform inference using multiple images per batch per GPU - **CPU support for inference:** runs on CPU in inference time. See our [webcam demo](demo) for an example - Provides pre-trained models for almost all reference Mask R-CNN and Faster R-CNN configurations with 1x schedule. @@ -152,6 +153,15 @@ python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_ben ``` Note we should set `MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN` follow the rule in Single-GPU training. +### Mixed precision training +We currently use [APEX](https://github.com/NVIDIA/apex) to add [Automatic Mixed Precision](https://developer.nvidia.com/automatic-mixed-precision) support. To enable, just do Single-GPU or Multi-GPU training and set `DTYPE "float16"`. + +```bash +export NGPUS=8 +python -m torch.distributed.launch --nproc_per_node=$NGPUS /path_to_maskrcnn_benchmark/tools/train_net.py --config-file "path/to/config/file.yaml" MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN images_per_gpu x 1000 DTYPE "float16" +``` +If you want more verbose logging, set `AMP_VERBOSE True`. See [Mixed Precision Training guide](https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html) for more details. + ## Evaluation You can test your model directly on single or multiple gpus. Here is an example for Mask R-CNN R-50 FPN with the 1x schedule on 8 GPUS: ```bash From 2c025d9beacca96e2697ca73fcb7bd576dcb8b56 Mon Sep 17 00:00:00 2001 From: Karanbir Chahal Date: Mon, 3 Jun 2019 15:25:25 +0530 Subject: [PATCH 40/52] Shapes dataset demo (#849) --- demo/shapes_dataset_demo.ipynb | 3085 ++++++++++++++++++++++++++++++++ 1 file changed, 3085 insertions(+) create mode 100644 demo/shapes_dataset_demo.ipynb diff --git a/demo/shapes_dataset_demo.ipynb b/demo/shapes_dataset_demo.ipynb new file mode 100644 index 000000000..e073ec997 --- /dev/null +++ b/demo/shapes_dataset_demo.ipynb @@ -0,0 +1,3085 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "objdet.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [ + "xnr8tbDz7WjS", + "5DC0K7tW7d-M", + "BI2ncK7kATEh", + "hbzY16ocEdrg", + "If8z4OZfDHmC", + "mOo-0LGFEAmc", + "bbCBInqHFUg7", + "tAn3omCjTFGI", + "BTKsrHa-TkGr" + ] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "268x1mG64rCy", + "colab_type": "text" + }, + "source": [ + "# Installation" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "VNvKG2TF3Y0B", + "colab_type": "code", + "outputId": "64393040-91f6-49e8-8656-f8de9480b867", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + } + }, + "source": [ + "%%writefile setup.sh\n", + "\n", + "# maskrcnn_benchmark and coco api dependencies\n", + "pip install ninja yacs cython matplotlib tqdm opencv-python\n", + "\n", + "# follow PyTorch installation in https://pytorch.org/get-started/locally/\n", + "# we give the instructions for CUDA 9.0\n", + "pip install -c pytorch pytorch-nightly torchvision cudatoolkit=9.0\n", + "\n", + "\n", + "git clone https://github.com/cocodataset/cocoapi.git\n", + "cd cocoapi/PythonAPI\n", + "python setup.py build_ext install\n", + "cd ../../\n", + "\n", + "# install apex\n", + "rm -rf apex\n", + "git clone https://github.com/NVIDIA/apex.git\n", + "cd apex\n", + "git pull\n", + "python setup.py install --cuda_ext --cpp_ext\n", + "cd ../\n", + "\n", + "# install PyTorch Detection\n", + "git clone https://github.com/facebookresearch/maskrcnn-benchmark.git\n", + "cd maskrcnn-benchmark\n", + "\n", + "# the following will install the lib with\n", + "# symbolic links, so that you can modify\n", + "# the files if you want and won't need to\n", + "# re-build it\n", + "python setup.py build develop\n" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Writing setup.sh\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "NYzsp3Ng3mOy", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 13705 + }, + "outputId": "e1230ab4-a5b6-41b7-fa69-5b175a20b26d" + }, + "source": [ + "!sh setup.sh" + ], + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Collecting ninja\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/cc/bf/32e5dd5cce6543374e4050a7292099402ab80787eddf3732810a55b37763/ninja-1.9.0.post1-py3-none-manylinux1_x86_64.whl (98kB)\n", + "\u001b[K |████████████████████████████████| 102kB 32.5MB/s \n", + "\u001b[?25hCollecting yacs\n", + " Downloading https://files.pythonhosted.org/packages/2f/51/9d613d67a8561a0cdf696c3909870f157ed85617fea3cff769bb7de09ef7/yacs-0.1.6-py3-none-any.whl\n", + "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (0.29.9)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (3.0.3)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n", + "Requirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (3.4.5.20)\n", + "Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from yacs) (3.13)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (0.10.0)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.4.0)\n", + "Requirement already satisfied: numpy>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.16.4)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.1.0)\n", + "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.5.3)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib) (1.12.0)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib) (41.0.1)\n", + "Installing collected packages: ninja, yacs\n", + "Successfully installed ninja-1.9.0.post1 yacs-0.1.6\n", + "\u001b[31mERROR: Could not open requirements file: [Errno 2] No such file or directory: 'pytorch'\u001b[0m\n", + "Cloning into 'cocoapi'...\n", + "remote: Enumerating objects: 953, done.\u001b[K\n", + "remote: Total 953 (delta 0), reused 0 (delta 0), pack-reused 953\u001b[K\n", + "Receiving objects: 100% (953/953), 11.70 MiB | 6.53 MiB/s, done.\n", + "Resolving deltas: 100% (565/565), done.\n", + "running build_ext\n", + "cythoning pycocotools/_mask.pyx to pycocotools/_mask.c\n", + "/usr/local/lib/python3.6/dist-packages/Cython/Compiler/Main.py:367: FutureWarning: Cython directive 'language_level' not set, using 2 for now (Py2). This will change in a later release! File: /content/cocoapi/PythonAPI/pycocotools/_mask.pyx\n", + " tree = Parsing.p_module(s, pxd, full_module_name)\n", + "building 'pycocotools._mask' extension\n", + "creating build\n", + "creating build/common\n", + "creating build/temp.linux-x86_64-3.6\n", + "creating build/temp.linux-x86_64-3.6/pycocotools\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/numpy/core/include -I../common -I/usr/include/python3.6m -c ../common/maskApi.c -o build/temp.linux-x86_64-3.6/../common/maskApi.o -Wno-cpp -Wno-unused-function -std=c99\n", + "\u001b[01m\u001b[K../common/maskApi.c:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[KrleDecode\u001b[m\u001b[K’:\n", + "\u001b[01m\u001b[K../common/maskApi.c:46:7:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n", + " \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K( k=0; k2) x+=(long) cnts[m-2]; cnts[m++]=(uint) x;\n", + " \u001b[01;35m\u001b[K^~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K../common/maskApi.c:228:34:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kif\u001b[m\u001b[K’\n", + " if(m>2) x+=(long) cnts[m-2]; \u001b[01;36m\u001b[Kcnts\u001b[m\u001b[K[m++]=(uint) x;\n", + " \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K../common/maskApi.c:\u001b[m\u001b[K In function ‘\u001b[01m\u001b[KrleToBbox\u001b[m\u001b[K’:\n", + "\u001b[01m\u001b[K../common/maskApi.c:141:31:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kxp\u001b[m\u001b[K’ may be used uninitialized in this function [\u001b[01;35m\u001b[K-Wmaybe-uninitialized\u001b[m\u001b[K]\n", + " if(j%2==0) xp=x; else if\u001b[01;35m\u001b[K(\u001b[m\u001b[Kxp build/lib.linux-x86_64-3.6/pycocotools\n", + "copying pycocotools/mask.py -> build/lib.linux-x86_64-3.6/pycocotools\n", + "copying pycocotools/coco.py -> build/lib.linux-x86_64-3.6/pycocotools\n", + "copying pycocotools/cocoeval.py -> build/lib.linux-x86_64-3.6/pycocotools\n", + "creating build/bdist.linux-x86_64\n", + "creating build/bdist.linux-x86_64/egg\n", + "creating build/bdist.linux-x86_64/egg/pycocotools\n", + "copying build/lib.linux-x86_64-3.6/pycocotools/__init__.py -> build/bdist.linux-x86_64/egg/pycocotools\n", + "copying build/lib.linux-x86_64-3.6/pycocotools/mask.py -> build/bdist.linux-x86_64/egg/pycocotools\n", + "copying build/lib.linux-x86_64-3.6/pycocotools/_mask.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg/pycocotools\n", + "copying build/lib.linux-x86_64-3.6/pycocotools/coco.py -> build/bdist.linux-x86_64/egg/pycocotools\n", + "copying build/lib.linux-x86_64-3.6/pycocotools/cocoeval.py -> build/bdist.linux-x86_64/egg/pycocotools\n", + "byte-compiling build/bdist.linux-x86_64/egg/pycocotools/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/pycocotools/mask.py to mask.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/pycocotools/coco.py to coco.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/pycocotools/cocoeval.py to cocoeval.cpython-36.pyc\n", + "creating stub loader for pycocotools/_mask.cpython-36m-x86_64-linux-gnu.so\n", + "byte-compiling build/bdist.linux-x86_64/egg/pycocotools/_mask.py to _mask.cpython-36.pyc\n", + "creating build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying pycocotools.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying pycocotools.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying pycocotools.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying pycocotools.egg-info/requires.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying pycocotools.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt\n", + "zip_safe flag not set; analyzing archive contents...\n", + "pycocotools.__pycache__._mask.cpython-36: module references __file__\n", + "creating dist\n", + "creating 'dist/pycocotools-2.0-py3.6-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n", + "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n", + "Processing pycocotools-2.0-py3.6-linux-x86_64.egg\n", + "creating /usr/local/lib/python3.6/dist-packages/pycocotools-2.0-py3.6-linux-x86_64.egg\n", + "Extracting pycocotools-2.0-py3.6-linux-x86_64.egg to /usr/local/lib/python3.6/dist-packages\n", + "Adding pycocotools 2.0 to easy-install.pth file\n", + "\n", + "Installed /usr/local/lib/python3.6/dist-packages/pycocotools-2.0-py3.6-linux-x86_64.egg\n", + "Processing dependencies for pycocotools==2.0\n", + "Searching for matplotlib==3.0.3\n", + "Best match: matplotlib 3.0.3\n", + "Adding matplotlib 3.0.3 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for Cython==0.29.9\n", + "Best match: Cython 0.29.9\n", + "Adding Cython 0.29.9 to easy-install.pth file\n", + "Installing cygdb script to /usr/local/bin\n", + "Installing cython script to /usr/local/bin\n", + "Installing cythonize script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for setuptools==41.0.1\n", + "Best match: setuptools 41.0.1\n", + "Adding setuptools 41.0.1 to easy-install.pth file\n", + "Installing easy_install script to /usr/local/bin\n", + "Installing easy_install-3.6 script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for pyparsing==2.4.0\n", + "Best match: pyparsing 2.4.0\n", + "Adding pyparsing 2.4.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for kiwisolver==1.1.0\n", + "Best match: kiwisolver 1.1.0\n", + "Adding kiwisolver 1.1.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for python-dateutil==2.5.3\n", + "Best match: python-dateutil 2.5.3\n", + "Adding python-dateutil 2.5.3 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for numpy==1.16.4\n", + "Best match: numpy 1.16.4\n", + "Adding numpy 1.16.4 to easy-install.pth file\n", + "Installing f2py script to /usr/local/bin\n", + "Installing f2py3 script to /usr/local/bin\n", + "Installing f2py3.6 script to /usr/local/bin\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for cycler==0.10.0\n", + "Best match: cycler 0.10.0\n", + "Adding cycler 0.10.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Searching for six==1.12.0\n", + "Best match: six 1.12.0\n", + "Adding six 1.12.0 to easy-install.pth file\n", + "\n", + "Using /usr/local/lib/python3.6/dist-packages\n", + "Finished processing dependencies for pycocotools==2.0\n", + "Cloning into 'apex'...\n", + "remote: Enumerating objects: 28, done.\u001b[K\n", + "remote: Counting objects: 100% (28/28), done.\u001b[K\n", + "remote: Compressing objects: 100% (25/25), done.\u001b[K\n", + "remote: Total 4606 (delta 12), reused 7 (delta 3), pack-reused 4578\u001b[K\n", + "Receiving objects: 100% (4606/4606), 8.68 MiB | 6.00 MiB/s, done.\n", + "Resolving deltas: 100% (2982/2982), done.\n", + "Already up to date.\n", + "torch.__version__ = 1.1.0\n", + "\n", + "Compiling cuda extensions with\n", + "nvcc: NVIDIA (R) Cuda compiler driver\n", + "Copyright (c) 2005-2018 NVIDIA Corporation\n", + "Built on Sat_Aug_25_21:08:01_CDT_2018\n", + "Cuda compilation tools, release 10.0, V10.0.130\n", + "from /usr/local/cuda/bin\n", + "\n", + "running install\n", + "running bdist_egg\n", + "running egg_info\n", + "creating apex.egg-info\n", + "writing apex.egg-info/PKG-INFO\n", + "writing dependency_links to apex.egg-info/dependency_links.txt\n", + "writing top-level names to apex.egg-info/top_level.txt\n", + "writing manifest file 'apex.egg-info/SOURCES.txt'\n", + "writing manifest file 'apex.egg-info/SOURCES.txt'\n", + "installing library code to build/bdist.linux-x86_64/egg\n", + "running install_lib\n", + "running build_py\n", + "creating build\n", + "creating build/lib.linux-x86_64-3.6\n", + "creating build/lib.linux-x86_64-3.6/apex\n", + "copying apex/__init__.py -> build/lib.linux-x86_64-3.6/apex\n", + "creating build/lib.linux-x86_64-3.6/apex/reparameterization\n", + "copying apex/reparameterization/__init__.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n", + "copying apex/reparameterization/weight_norm.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n", + "copying apex/reparameterization/reparameterization.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\n", + "creating build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/__init__.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/LARC.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/multiproc.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/distributed.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/optimized_sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\n", + "creating build/lib.linux-x86_64-3.6/apex/fp16_utils\n", + "copying apex/fp16_utils/__init__.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n", + "copying apex/fp16_utils/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n", + "copying apex/fp16_utils/fp16util.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n", + "copying apex/fp16_utils/loss_scaler.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\n", + "creating build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/rnn_compat.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/frontend.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/wrap.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/amp.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/compat.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/_process_optimizer.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/__version__.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/opt.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/_initialize.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/_amp_state.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/utils.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/handle.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "copying apex/amp/scaler.py -> build/lib.linux-x86_64-3.6/apex/amp\n", + "creating build/lib.linux-x86_64-3.6/apex/normalization\n", + "copying apex/normalization/__init__.py -> build/lib.linux-x86_64-3.6/apex/normalization\n", + "copying apex/normalization/fused_layer_norm.py -> build/lib.linux-x86_64-3.6/apex/normalization\n", + "creating build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n", + "copying apex/multi_tensor_apply/__init__.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n", + "copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\n", + "creating build/lib.linux-x86_64-3.6/apex/optimizers\n", + "copying apex/optimizers/fused_adam.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n", + "copying apex/optimizers/__init__.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n", + "copying apex/optimizers/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/optimizers\n", + "creating build/lib.linux-x86_64-3.6/apex/RNN\n", + "copying apex/RNN/__init__.py -> build/lib.linux-x86_64-3.6/apex/RNN\n", + "copying apex/RNN/RNNBackend.py -> build/lib.linux-x86_64-3.6/apex/RNN\n", + "copying apex/RNN/models.py -> build/lib.linux-x86_64-3.6/apex/RNN\n", + "copying apex/RNN/cells.py -> build/lib.linux-x86_64-3.6/apex/RNN\n", + "creating build/lib.linux-x86_64-3.6/apex/amp/lists\n", + "copying apex/amp/lists/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n", + "copying apex/amp/lists/torch_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n", + "copying apex/amp/lists/functional_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n", + "copying apex/amp/lists/tensor_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\n", + "running build_ext\n", + "building 'apex_C' extension\n", + "creating build/temp.linux-x86_64-3.6\n", + "creating build/temp.linux-x86_64-3.6/csrc\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/include/python3.6m -c csrc/flatten_unflatten.cpp -o build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=apex_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -o build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so\n", + "building 'amp_C' extension\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/amp_C_frontend.cpp -o build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_scale_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_axpby_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_l2norm_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb_stage_1.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_1.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/multi_tensor_lamb_stage_2.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_2.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_1.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_lamb_stage_2.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so\n", + "building 'fused_adam_cuda' extension\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/fused_adam_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/fused_adam_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so\n", + "building 'syncbn' extension\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/syncbn.cpp -o build/temp.linux-x86_64-3.6/csrc/syncbn.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/welford.cu -o build/temp.linux-x86_64-3.6/csrc/welford.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/syncbn.o build/temp.linux-x86_64-3.6/csrc/welford.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so\n", + "building 'fused_layer_norm_cuda' extension\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/layer_norm_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o -O3 -DVERSION_GE_1_1 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c csrc/layer_norm_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -maxrregcount=50 -O3 --use_fast_math -DVERSION_GE_1_1 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so\n", + "creating build/bdist.linux-x86_64\n", + "creating build/bdist.linux-x86_64/egg\n", + "copying build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n", + "copying build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n", + "copying build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n", + "creating build/bdist.linux-x86_64/egg/apex\n", + "creating build/bdist.linux-x86_64/egg/apex/reparameterization\n", + "copying build/lib.linux-x86_64-3.6/apex/reparameterization/__init__.py -> build/bdist.linux-x86_64/egg/apex/reparameterization\n", + "copying build/lib.linux-x86_64-3.6/apex/reparameterization/weight_norm.py -> build/bdist.linux-x86_64/egg/apex/reparameterization\n", + "copying build/lib.linux-x86_64-3.6/apex/reparameterization/reparameterization.py -> build/bdist.linux-x86_64/egg/apex/reparameterization\n", + "copying build/lib.linux-x86_64-3.6/apex/__init__.py -> build/bdist.linux-x86_64/egg/apex\n", + "creating build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm_kernel.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/__init__.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/LARC.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/multiproc.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/distributed.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm_kernel.py -> build/bdist.linux-x86_64/egg/apex/parallel\n", + "creating build/bdist.linux-x86_64/egg/apex/fp16_utils\n", + "copying build/lib.linux-x86_64-3.6/apex/fp16_utils/__init__.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n", + "copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16_optimizer.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n", + "copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16util.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n", + "copying build/lib.linux-x86_64-3.6/apex/fp16_utils/loss_scaler.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n", + "creating build/bdist.linux-x86_64/egg/apex/amp\n", + "creating build/bdist.linux-x86_64/egg/apex/amp/lists\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/lists/__init__.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/lists/torch_overrides.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/lists/functional_overrides.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/lists/tensor_overrides.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/rnn_compat.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/frontend.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/wrap.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/__init__.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/amp.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/compat.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/_process_optimizer.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/__version__.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/opt.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/_initialize.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/_amp_state.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/utils.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/handle.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "copying build/lib.linux-x86_64-3.6/apex/amp/scaler.py -> build/bdist.linux-x86_64/egg/apex/amp\n", + "creating build/bdist.linux-x86_64/egg/apex/normalization\n", + "copying build/lib.linux-x86_64-3.6/apex/normalization/__init__.py -> build/bdist.linux-x86_64/egg/apex/normalization\n", + "copying build/lib.linux-x86_64-3.6/apex/normalization/fused_layer_norm.py -> build/bdist.linux-x86_64/egg/apex/normalization\n", + "creating build/bdist.linux-x86_64/egg/apex/multi_tensor_apply\n", + "copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/__init__.py -> build/bdist.linux-x86_64/egg/apex/multi_tensor_apply\n", + "copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/multi_tensor_apply.py -> build/bdist.linux-x86_64/egg/apex/multi_tensor_apply\n", + "creating build/bdist.linux-x86_64/egg/apex/optimizers\n", + "copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_adam.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n", + "copying build/lib.linux-x86_64-3.6/apex/optimizers/__init__.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n", + "copying build/lib.linux-x86_64-3.6/apex/optimizers/fp16_optimizer.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n", + "creating build/bdist.linux-x86_64/egg/apex/RNN\n", + "copying build/lib.linux-x86_64-3.6/apex/RNN/__init__.py -> build/bdist.linux-x86_64/egg/apex/RNN\n", + "copying build/lib.linux-x86_64-3.6/apex/RNN/RNNBackend.py -> build/bdist.linux-x86_64/egg/apex/RNN\n", + "copying build/lib.linux-x86_64-3.6/apex/RNN/models.py -> build/bdist.linux-x86_64/egg/apex/RNN\n", + "copying build/lib.linux-x86_64-3.6/apex/RNN/cells.py -> build/bdist.linux-x86_64/egg/apex/RNN\n", + "copying build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n", + "copying build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/reparameterization/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/reparameterization/weight_norm.py to weight_norm.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/reparameterization/reparameterization.py to reparameterization.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/sync_batchnorm_kernel.py to sync_batchnorm_kernel.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/sync_batchnorm.py to sync_batchnorm.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/LARC.py to LARC.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/multiproc.py to multiproc.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/distributed.py to distributed.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/optimized_sync_batchnorm.py to optimized_sync_batchnorm.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/optimized_sync_batchnorm_kernel.py to optimized_sync_batchnorm_kernel.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/fp16util.py to fp16util.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/loss_scaler.py to loss_scaler.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/torch_overrides.py to torch_overrides.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/functional_overrides.py to functional_overrides.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/tensor_overrides.py to tensor_overrides.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/rnn_compat.py to rnn_compat.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/frontend.py to frontend.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/wrap.py to wrap.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/amp.py to amp.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/compat.py to compat.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/_process_optimizer.py to _process_optimizer.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/__version__.py to __version__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/opt.py to opt.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/_initialize.py to _initialize.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/_amp_state.py to _amp_state.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/utils.py to utils.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/handle.py to handle.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/scaler.py to scaler.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/normalization/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/normalization/fused_layer_norm.py to fused_layer_norm.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/multi_tensor_apply/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/multi_tensor_apply/multi_tensor_apply.py to multi_tensor_apply.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/__init__.py to __init__.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/RNNBackend.py to RNNBackend.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/models.py to models.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/cells.py to cells.cpython-36.pyc\n", + "creating stub loader for apex_C.cpython-36m-x86_64-linux-gnu.so\n", + "creating stub loader for amp_C.cpython-36m-x86_64-linux-gnu.so\n", + "creating stub loader for fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so\n", + "creating stub loader for syncbn.cpython-36m-x86_64-linux-gnu.so\n", + "creating stub loader for fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so\n", + "byte-compiling build/bdist.linux-x86_64/egg/apex_C.py to apex_C.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/amp_C.py to amp_C.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/fused_adam_cuda.py to fused_adam_cuda.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/syncbn.py to syncbn.cpython-36.pyc\n", + "byte-compiling build/bdist.linux-x86_64/egg/fused_layer_norm_cuda.py to fused_layer_norm_cuda.cpython-36.pyc\n", + "creating build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying apex.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying apex.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying apex.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "copying apex.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n", + "writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt\n", + "zip_safe flag not set; analyzing archive contents...\n", + "__pycache__.amp_C.cpython-36: module references __file__\n", + "__pycache__.apex_C.cpython-36: module references __file__\n", + "__pycache__.fused_adam_cuda.cpython-36: module references __file__\n", + "__pycache__.fused_layer_norm_cuda.cpython-36: module references __file__\n", + "__pycache__.syncbn.cpython-36: module references __file__\n", + "creating dist\n", + "creating 'dist/apex-0.1-py3.6-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n", + "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n", + "Processing apex-0.1-py3.6-linux-x86_64.egg\n", + "creating /usr/local/lib/python3.6/dist-packages/apex-0.1-py3.6-linux-x86_64.egg\n", + "Extracting apex-0.1-py3.6-linux-x86_64.egg to /usr/local/lib/python3.6/dist-packages\n", + "Adding apex 0.1 to easy-install.pth file\n", + "\n", + "Installed /usr/local/lib/python3.6/dist-packages/apex-0.1-py3.6-linux-x86_64.egg\n", + "Processing dependencies for apex==0.1\n", + "Finished processing dependencies for apex==0.1\n", + "Cloning into 'maskrcnn-benchmark'...\n", + "remote: Enumerating objects: 1, done.\u001b[K\n", + "remote: Counting objects: 100% (1/1), done.\u001b[K\n", + "remote: Total 1524 (delta 0), reused 1 (delta 0), pack-reused 1523\u001b[K\n", + "Receiving objects: 100% (1524/1524), 6.31 MiB | 4.94 MiB/s, done.\n", + "Resolving deltas: 100% (917/917), done.\n", + "running build\n", + "running build_py\n", + "creating build\n", + "creating build/lib.linux-x86_64-3.6\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark\n", + "copying maskrcnn_benchmark/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/engine\n", + "copying maskrcnn_benchmark/engine/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/engine\n", + "copying maskrcnn_benchmark/engine/inference.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/engine\n", + "copying maskrcnn_benchmark/engine/bbox_aug.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/engine\n", + "copying maskrcnn_benchmark/engine/trainer.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/engine\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/solver\n", + "copying maskrcnn_benchmark/solver/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/solver\n", + "copying maskrcnn_benchmark/solver/build.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/solver\n", + "copying maskrcnn_benchmark/solver/lr_scheduler.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/solver\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/config\n", + "copying maskrcnn_benchmark/config/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/config\n", + "copying maskrcnn_benchmark/config/defaults.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/config\n", + "copying maskrcnn_benchmark/config/paths_catalog.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/config\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/sigmoid_focal_loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/misc.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/smooth_l1_loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/roi_align.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/_utils.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/nms.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/batch_norm.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "copying maskrcnn_benchmark/layers/roi_pool.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data\n", + "copying maskrcnn_benchmark/data/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data\n", + "copying maskrcnn_benchmark/data/collate_batch.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data\n", + "copying maskrcnn_benchmark/data/build.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "copying maskrcnn_benchmark/structures/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "copying maskrcnn_benchmark/structures/boxlist_ops.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "copying maskrcnn_benchmark/structures/image_list.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "copying maskrcnn_benchmark/structures/segmentation_mask.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "copying maskrcnn_benchmark/structures/keypoint.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "copying maskrcnn_benchmark/structures/bounding_box.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/structures\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/box_coder.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/matcher.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/registry.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/utils.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/poolers.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/balanced_positive_negative_sampler.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "copying maskrcnn_benchmark/modeling/make_layers.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/comm.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/checkpoint.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/metric_logger.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/model_serialization.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/miscellaneous.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/env.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/imports.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/timer.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/cv2_util.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/c2_model_loading.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/registry.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/collect_env.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/model_zoo.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "copying maskrcnn_benchmark/utils/logger.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/utils\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers/dcn\n", + "copying maskrcnn_benchmark/layers/dcn/deform_conv_module.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers/dcn\n", + "copying maskrcnn_benchmark/layers/dcn/deform_pool_module.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers/dcn\n", + "copying maskrcnn_benchmark/layers/dcn/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers/dcn\n", + "copying maskrcnn_benchmark/layers/dcn/deform_pool_func.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers/dcn\n", + "copying maskrcnn_benchmark/layers/dcn/deform_conv_func.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/layers/dcn\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/transforms\n", + "copying maskrcnn_benchmark/data/transforms/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/transforms\n", + "copying maskrcnn_benchmark/data/transforms/build.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/transforms\n", + "copying maskrcnn_benchmark/data/transforms/transforms.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/transforms\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/samplers\n", + "copying maskrcnn_benchmark/data/samplers/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/samplers\n", + "copying maskrcnn_benchmark/data/samplers/distributed.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/samplers\n", + "copying maskrcnn_benchmark/data/samplers/grouped_batch_sampler.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/samplers\n", + "copying maskrcnn_benchmark/data/samplers/iteration_based_batch_sampler.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/samplers\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets\n", + "copying maskrcnn_benchmark/data/datasets/voc.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets\n", + "copying maskrcnn_benchmark/data/datasets/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets\n", + "copying maskrcnn_benchmark/data/datasets/coco.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets\n", + "copying maskrcnn_benchmark/data/datasets/concat_dataset.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets\n", + "copying maskrcnn_benchmark/data/datasets/list_dataset.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation\n", + "copying maskrcnn_benchmark/data/datasets/evaluation/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation/coco\n", + "copying maskrcnn_benchmark/data/datasets/evaluation/coco/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation/coco\n", + "copying maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation/coco\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation/voc\n", + "copying maskrcnn_benchmark/data/datasets/evaluation/voc/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation/voc\n", + "copying maskrcnn_benchmark/data/datasets/evaluation/voc/voc_eval.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/data/datasets/evaluation/voc\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "copying maskrcnn_benchmark/modeling/rpn/anchor_generator.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "copying maskrcnn_benchmark/modeling/rpn/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "copying maskrcnn_benchmark/modeling/rpn/loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "copying maskrcnn_benchmark/modeling/rpn/inference.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "copying maskrcnn_benchmark/modeling/rpn/rpn.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "copying maskrcnn_benchmark/modeling/rpn/utils.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/detector\n", + "copying maskrcnn_benchmark/modeling/detector/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/detector\n", + "copying maskrcnn_benchmark/modeling/detector/detectors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/detector\n", + "copying maskrcnn_benchmark/modeling/detector/generalized_rcnn.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/detector\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads\n", + "copying maskrcnn_benchmark/modeling/roi_heads/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads\n", + "copying maskrcnn_benchmark/modeling/roi_heads/roi_heads.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/fbnet.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/fbnet_modeldef.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/fpn.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/fbnet_builder.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/backbone.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "copying maskrcnn_benchmark/modeling/backbone/resnet.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/backbone\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn/retinanet\n", + "copying maskrcnn_benchmark/modeling/rpn/retinanet/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn/retinanet\n", + "copying maskrcnn_benchmark/modeling/rpn/retinanet/loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn/retinanet\n", + "copying maskrcnn_benchmark/modeling/rpn/retinanet/inference.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn/retinanet\n", + "copying maskrcnn_benchmark/modeling/rpn/retinanet/retinanet.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/rpn/retinanet\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/box_head/roi_box_feature_extractors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/box_head/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/box_head/loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/box_head/roi_box_predictors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/box_head/box_head.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/box_head/inference.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/box_head\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/mask_head/roi_mask_feature_extractors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/mask_head/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/mask_head/mask_head.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/mask_head/inference.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/mask_head/roi_mask_predictors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/mask_head\n", + "creating build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/keypoint_head/keypoint_head.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/keypoint_head/__init__.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/keypoint_head/loss.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/keypoint_head/roi_keypoint_predictors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/keypoint_head/inference.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "copying maskrcnn_benchmark/modeling/roi_heads/keypoint_head/roi_keypoint_feature_extractors.py -> build/lib.linux-x86_64-3.6/maskrcnn_benchmark/modeling/roi_heads/keypoint_head\n", + "running build_ext\n", + "building 'maskrcnn_benchmark._C' extension\n", + "creating build/temp.linux-x86_64-3.6\n", + "creating build/temp.linux-x86_64-3.6/content\n", + "creating build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark\n", + "creating build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark\n", + "creating build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc\n", + "creating build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu\n", + "creating build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/vision.cpp -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/vision.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "In file included from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/ATen.h:9:0\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/types.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader_options.h:4\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/base.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/stateful.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/all.h:4\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/extension.h:4\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/vision.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp:2\u001b[m\u001b[K:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:71:52:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " at::ScalarType _st = ::detail::scalar_type(TYPE\u001b[01;35m\u001b[K)\u001b[m\u001b[K; \\\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp:242:3:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kin expansion of macro ‘\u001b[01m\u001b[KAT_DISPATCH_FLOATING_TYPES\u001b[m\u001b[K’\n", + " \u001b[01;36m\u001b[KAT_DISPATCH_FLOATING_TYPES\u001b[m\u001b[K(input.type(), \"ROIAlign_forward\", [&] {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:23:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " inline at::ScalarType \u001b[01;36m\u001b[Kscalar_type\u001b[m\u001b[K(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/nms_cpu.cpp -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/nms_cpu.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "In file included from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/ATen.h:9:0\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/types.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader_options.h:4\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/base.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/stateful.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data/dataloader.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/data.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include/torch/all.h:4\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/torch/extension.h:4\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/vision.h:3\u001b[m\u001b[K,\n", + " from \u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/nms_cpu.cpp:2\u001b[m\u001b[K:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/nms_cpu.cpp:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:71:52:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " at::ScalarType _st = ::detail::scalar_type(TYPE\u001b[01;35m\u001b[K)\u001b[m\u001b[K; \\\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/nms_cpu.cpp:71:3:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kin expansion of macro ‘\u001b[01m\u001b[KAT_DISPATCH_FLOATING_TYPES\u001b[m\u001b[K’\n", + " \u001b[01;36m\u001b[KAT_DISPATCH_FLOATING_TYPES\u001b[m\u001b[K(dets.type(), \"nms\", [&] {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~~~~~~~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:23:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " inline at::ScalarType \u001b[01;36m\u001b[Kscalar_type\u001b[m\u001b[K(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(83): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lowest\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(84): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"max\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(85): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lower_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(86): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"upper_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:266:124:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:360:126:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:458:126:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:788:124:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:820:126:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.cu:853:126:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/nms.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/nms.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(83): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lowest\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(84): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"max\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(85): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lower_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(86): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"upper_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.cu:283:120:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES(input.type(), \"ROIAlign_forward\", [&] {\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.cu:329:118:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES(grad.type(), \"ROIAlign_backward\", [&] {\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(83): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lowest\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(84): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"max\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(85): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lower_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(86): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"upper_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.cu:137:120:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES(input.type(), \"ROIPool_forward\", [&] {\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.cu:185:118:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES(grad.type(), \"ROIPool_backward\", [&] {\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(83): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lowest\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(84): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"max\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(85): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lower_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(86): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"upper_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu:292:118:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.cu:343:126:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES_AND_HALF(\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "/usr/local/cuda/bin/nvcc -DWITH_CUDA -I/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc -I/usr/local/lib/python3.6/dist-packages/torch/include -I/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.6/dist-packages/torch/include/TH -I/usr/local/lib/python3.6/dist-packages/torch/include/THC -I/usr/local/cuda/include -I/usr/include/python3.6m -c /content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu -o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DCUDA_HAS_FP16=1 -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(83): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lowest\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(84): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"max\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(85): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"lower_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "/usr/local/lib/python3.6/dist-packages/torch/include/ATen/cuda/NumericLimits.cuh(86): warning: calling a constexpr __host__ function(\"from_bits\") from a __host__ __device__ function(\"upper_bound\") is not allowed. The experimental flag '--expt-relaxed-constexpr' can be used to allow this.\n", + "\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu:129:122:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES(logits.type(), \"SigmoidFocalLoss_forward\", [&] {\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu:\u001b[m\u001b[K In lambda function:\n", + "\u001b[01m\u001b[K/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.cu:173:122:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[K‘\u001b[01m\u001b[Kc10::ScalarType detail::scalar_type(const at::DeprecatedTypeProperties&)\u001b[m\u001b[K’ is deprecated [\u001b[01;35m\u001b[K-Wdeprecated-declarations\u001b[m\u001b[K]\n", + " AT_DISPATCH_FLOATING_TYPES(logits.type(), \"SigmoidFocalLoss_backward\", [&] {\n", + " \u001b[01;35m\u001b[K^\u001b[m\u001b[K\n", + "\u001b[01m\u001b[K/usr/local/lib/python3.6/dist-packages/torch/include/ATen/Dispatch.h:47:1:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[Kdeclared here\n", + " \u001b[01;36m\u001b[Kinline at::\u001b[m\u001b[KScalarType scalar_type(const at::DeprecatedTypeProperties &t) {\n", + " \u001b[01;36m\u001b[K^~~~~~~~~~~\u001b[m\u001b[K\n", + "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/vision.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cpu/nms_cpu.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_kernel_cuda.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/nms.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIAlign_cuda.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_conv_cuda.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_cuda.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/ROIPool_cuda.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/deform_pool_kernel_cuda.o build/temp.linux-x86_64-3.6/content/maskrcnn-benchmark/maskrcnn_benchmark/csrc/cuda/SigmoidFocalLoss_cuda.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/maskrcnn_benchmark/_C.cpython-36m-x86_64-linux-gnu.so\n", + "running develop\n", + "running egg_info\n", + "creating maskrcnn_benchmark.egg-info\n", + "writing maskrcnn_benchmark.egg-info/PKG-INFO\n", + "writing dependency_links to maskrcnn_benchmark.egg-info/dependency_links.txt\n", + "writing top-level names to maskrcnn_benchmark.egg-info/top_level.txt\n", + "writing manifest file 'maskrcnn_benchmark.egg-info/SOURCES.txt'\n", + "writing manifest file 'maskrcnn_benchmark.egg-info/SOURCES.txt'\n", + "running build_ext\n", + "copying build/lib.linux-x86_64-3.6/maskrcnn_benchmark/_C.cpython-36m-x86_64-linux-gnu.so -> maskrcnn_benchmark\n", + "Creating /usr/local/lib/python3.6/dist-packages/maskrcnn-benchmark.egg-link (link to .)\n", + "Adding maskrcnn-benchmark 0.1 to easy-install.pth file\n", + "\n", + "Installed /content/maskrcnn-benchmark\n", + "Processing dependencies for maskrcnn-benchmark==0.1\n", + "Finished processing dependencies for maskrcnn-benchmark==0.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1uoPMGDl49Wk", + "colab_type": "text" + }, + "source": [ + "### Checking our Installation\n", + "\n", + "If a module not found error appears, restart the runtime. The libraries should be loaded after restarting" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "3q-n76S95KA3", + "colab_type": "code", + "colab": {} + }, + "source": [ + "import maskrcnn_benchmark" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-N9mxq4OX6Yc", + "colab_type": "text" + }, + "source": [ + "# Imports" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "kLzesfGNX9O2", + "colab_type": "code", + "colab": {} + }, + "source": [ + "import torch\n", + "\n", + "# Set up custom environment before nearly anything else is imported\n", + "# NOTE: this should be the first import (no not reorder)\n", + "from maskrcnn_benchmark.utils.env import setup_environment # noqa F401 isort:skip\n", + "\n", + "from maskrcnn_benchmark.data.build import *\n", + "from maskrcnn_benchmark.structures.bounding_box import BoxList\n", + "from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask\n", + "from maskrcnn_benchmark.modeling.detector import build_detection_model\n", + "from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer\n", + "from maskrcnn_benchmark.structures.image_list import to_image_list\n", + "from maskrcnn_benchmark.modeling.roi_heads.mask_head.inference import Masker\n", + "from maskrcnn_benchmark import layers as L\n", + "from maskrcnn_benchmark.utils import cv2_util\n", + "from maskrcnn_benchmark.utils.miscellaneous import mkdir\n", + "from maskrcnn_benchmark.utils.logger import setup_logger\n", + "from maskrcnn_benchmark.utils.comm import synchronize, get_rank\n", + "from maskrcnn_benchmark.config import cfg\n", + "from maskrcnn_benchmark.config import cfg\n", + "from maskrcnn_benchmark.data import make_data_loader\n", + "from maskrcnn_benchmark.solver import make_lr_scheduler\n", + "from maskrcnn_benchmark.solver import make_optimizer\n", + "from maskrcnn_benchmark.engine.inference import inference\n", + "from maskrcnn_benchmark.engine.trainer import do_train\n", + "from maskrcnn_benchmark.modeling.detector import build_detection_model\n", + "from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer\n", + "from maskrcnn_benchmark.utils.collect_env import collect_env_info\n", + "from maskrcnn_benchmark.utils.comm import synchronize, get_rank\n", + "from maskrcnn_benchmark.utils.imports import import_file\n", + "from maskrcnn_benchmark.data.datasets.evaluation import evaluate\n", + "from maskrcnn_benchmark.utils.comm import is_main_process, get_world_size\n", + "from maskrcnn_benchmark.utils.comm import all_gather\n", + "from maskrcnn_benchmark.utils.timer import Timer, get_time_str\n", + "from maskrcnn_benchmark.engine.inference import compute_on_dataset, _accumulate_predictions_from_multiple_gpus\n", + "from maskrcnn_benchmark.data.datasets.evaluation.coco import coco_evaluation\n", + "\n", + "from PIL import Image\n", + "import json\n", + "import logging\n", + "import torch\n", + "import numpy as np\n", + "import skimage.draw as draw\n", + "import tempfile\n", + "from pycocotools.coco import COCO\n", + "import os\n", + "import sys\n", + "import random\n", + "import math\n", + "import re\n", + "import time\n", + "import cv2\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "from tqdm import tqdm\n", + "\n", + "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.\n", + "from torchvision import transforms as T\n", + "from torchvision.transforms import functional as F\n", + "from google.colab.patches import cv2_imshow\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DvU-NYKJ3uzb", + "colab_type": "text" + }, + "source": [ + "# Loading Our Dataset\n", + "\n", + "To train a network using the MaskRCNN repo, we first need to define our dataset. The dataset needs to a class of type object and should extend 3 things. \n", + "\n", + "1. **__getitem__(self, idx)**: This function should return a PIL Image, a BoxList and the idx. The Boxlist is an abstraction for our bounding boxes, segmentation masks, class lables and also people keypoints. Please check ABSTRACTIONS.ms for more details on this. \n", + "\n", + "2. **__len__()**: returns the length of the dataset. \n", + "\n", + "3. **get_img_info(self, idx)**: Return a dict of img info with the fields \"height\" and \"width\" filled in with the idx's image's height and width.\n", + "\n", + "4. **self.coco**: Should be a variable that holds the COCO object for your annotations so that you can perform evaluations of your dataset. \n", + "\n", + "5. **self.id_to_img_map**: Is a dictionary that maps the ids to coco image ids. Almost in all cases just map the idxs to idxs. This is simply a requirement for the coco evaluation. \n", + "\n", + "6. **self.contiguous_category_id_to_json_id**: Another requirement for coco evaluation. It maps the categpry to json category id. Again, for almost all purposes category id and json id should be same. \n", + "\n", + "Given below is a sample fo a dataset. It is the Shape Dataset taken from the Matterport Mask RCNN Repo. One important detail is that the constructor if the dataset should have the variable transforms that is set inside the constructor. It should thgen be used inside **__get__item(idx)** as shown below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xnr8tbDz7WjS", + "colab_type": "text" + }, + "source": [ + "## Helper Functions" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "tb_5MERf7c_1", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Helper Functions for the Shapes Dataset\n", + "\n", + "def non_max_suppression(boxes, scores, threshold):\n", + " \"\"\"Performs non-maximum suppression and returns indices of kept boxes.\n", + " boxes: [N, (y1, x1, y2, x2)]. Notice that (y2, x2) lays outside the box.\n", + " scores: 1-D array of box scores.\n", + " threshold: Float. IoU threshold to use for filtering.\n", + " \"\"\"\n", + " assert boxes.shape[0] > 0\n", + " if boxes.dtype.kind != \"f\":\n", + " boxes = boxes.astype(np.float32)\n", + "\n", + " # Compute box areas\n", + " y1 = boxes[:, 0]\n", + " x1 = boxes[:, 1]\n", + " y2 = boxes[:, 2]\n", + " x2 = boxes[:, 3]\n", + " area = (y2 - y1) * (x2 - x1)\n", + "\n", + " # Get indicies of boxes sorted by scores (highest first)\n", + " ixs = scores.argsort()[::-1]\n", + "\n", + " pick = []\n", + " while len(ixs) > 0:\n", + " # Pick top box and add its index to the list\n", + " i = ixs[0]\n", + " pick.append(i)\n", + " # Compute IoU of the picked box with the rest\n", + " iou = compute_iou(boxes[i], boxes[ixs[1:]], area[i], area[ixs[1:]])\n", + " # Identify boxes with IoU over the threshold. This\n", + " # returns indices into ixs[1:], so add 1 to get\n", + " # indices into ixs.\n", + " remove_ixs = np.where(iou > threshold)[0] + 1\n", + " # Remove indices of the picked and overlapped boxes.\n", + " ixs = np.delete(ixs, remove_ixs)\n", + " ixs = np.delete(ixs, 0)\n", + " return np.array(pick, dtype=np.int32)\n", + "\n", + "def compute_iou(box, boxes, box_area, boxes_area):\n", + " \"\"\"Calculates IoU of the given box with the array of the given boxes.\n", + " box: 1D vector [y1, x1, y2, x2]\n", + " boxes: [boxes_count, (y1, x1, y2, x2)]\n", + " box_area: float. the area of 'box'\n", + " boxes_area: array of length boxes_count.\n", + " Note: the areas are passed in rather than calculated here for\n", + " efficiency. Calculate once in the caller to avoid duplicate work.\n", + " \"\"\"\n", + " # Calculate intersection areas\n", + " y1 = np.maximum(box[0], boxes[:, 0])\n", + " y2 = np.minimum(box[2], boxes[:, 2])\n", + " x1 = np.maximum(box[1], boxes[:, 1])\n", + " x2 = np.minimum(box[3], boxes[:, 3])\n", + " intersection = np.maximum(x2 - x1, 0) * np.maximum(y2 - y1, 0)\n", + " union = box_area + boxes_area[:] - intersection[:]\n", + " iou = intersection / union\n", + " return iou" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5DC0K7tW7d-M", + "colab_type": "text" + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WhG_Tu9ELAsj", + "colab_type": "code", + "colab": {} + }, + "source": [ + "class ShapeDataset(object):\n", + " \n", + " def __init__(self, num_examples, transforms=None):\n", + " \n", + " self.height = 128\n", + " self.width = 128\n", + " \n", + " self.num_examples = num_examples\n", + " self.transforms = transforms # IMPORTANT, DON'T MISS\n", + " self.image_info = []\n", + " self.logger = logging.getLogger(__name__)\n", + " \n", + " # Class Names: Note that the ids start fromm 1 not 0. This repo uses the 0 index for background\n", + " self.class_names = {\"square\": 1, \"circle\": 2, \"triangle\": 3}\n", + " \n", + " # Add images\n", + " # Generate random specifications of images (i.e. color and\n", + " # list of shapes sizes and locations). This is more compact than\n", + " # actual images. Images are generated on the fly in load_image().\n", + " for i in range(num_examples):\n", + " bg_color, shapes = self.random_image(self.height, self.width)\n", + " self.image_info.append({ \"path\":None,\n", + " \"width\": self.width, \"height\": self.height,\n", + " \"bg_color\": bg_color, \"shapes\": shapes\n", + " })\n", + " \n", + " # Fills in the self.coco varibale for evaluation.\n", + " self.get_gt()\n", + " \n", + " # Variables needed for coco mAP evaluation\n", + " self.id_to_img_map = {}\n", + " for i, _ in enumerate(self.image_info):\n", + " self.id_to_img_map[i] = i\n", + "\n", + " self.contiguous_category_id_to_json_id = { 0:0 ,1:1, 2:2, 3:3 }\n", + " \n", + "\n", + " def random_shape(self, height, width):\n", + " \"\"\"Generates specifications of a random shape that lies within\n", + " the given height and width boundaries.\n", + " Returns a tuple of three valus:\n", + " * The shape name (square, circle, ...)\n", + " * Shape color: a tuple of 3 values, RGB.\n", + " * Shape dimensions: A tuple of values that define the shape size\n", + " and location. Differs per shape type.\n", + " \"\"\"\n", + " # Shape\n", + " shape = random.choice([\"square\", \"circle\", \"triangle\"])\n", + " # Color\n", + " color = tuple([random.randint(0, 255) for _ in range(3)])\n", + " # Center x, y\n", + " buffer = 20\n", + " y = random.randint(buffer, height - buffer - 1)\n", + " x = random.randint(buffer, width - buffer - 1)\n", + " # Size\n", + " s = random.randint(buffer, height//4)\n", + " return shape, color, (x, y, s)\n", + "\n", + " def random_image(self, height, width):\n", + " \"\"\"Creates random specifications of an image with multiple shapes.\n", + " Returns the background color of the image and a list of shape\n", + " specifications that can be used to draw the image.\n", + " \"\"\"\n", + " # Pick random background color\n", + " bg_color = np.array([random.randint(0, 255) for _ in range(3)])\n", + " # Generate a few random shapes and record their\n", + " # bounding boxes\n", + " shapes = []\n", + " boxes = []\n", + " N = random.randint(1, 4)\n", + " labels = {}\n", + " for _ in range(N):\n", + " shape, color, dims = self.random_shape(height, width)\n", + " shapes.append((shape, color, dims))\n", + " x, y, s = dims\n", + " boxes.append([y-s, x-s, y+s, x+s])\n", + "\n", + " # Apply non-max suppression wit 0.3 threshold to avoid\n", + " # shapes covering each other\n", + " keep_ixs = non_max_suppression(np.array(boxes), np.arange(N), 0.3)\n", + " shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]\n", + " \n", + " return bg_color, shapes\n", + " \n", + " \n", + " def draw_shape(self, image, shape, dims, color):\n", + " \"\"\"Draws a shape from the given specs.\"\"\"\n", + " # Get the center x, y and the size s\n", + " x, y, s = dims\n", + " if shape == 'square':\n", + " cv2.rectangle(image, (x-s, y-s), (x+s, y+s), color, -1)\n", + " elif shape == \"circle\":\n", + " cv2.circle(image, (x, y), s, color, -1)\n", + " elif shape == \"triangle\":\n", + " points = np.array([[(x, y-s),\n", + " (x-s/math.sin(math.radians(60)), y+s),\n", + " (x+s/math.sin(math.radians(60)), y+s),\n", + " ]], dtype=np.int32)\n", + " cv2.fillPoly(image, points, color)\n", + " return image, [ x-s, y-s, x+s, y+s]\n", + "\n", + "\n", + " def load_mask(self, image_id):\n", + " \"\"\"\n", + " Generates instance masks for shapes of the given image ID.\n", + " \"\"\"\n", + " info = self.image_info[image_id]\n", + " shapes = info['shapes']\n", + " count = len(shapes)\n", + " mask = np.zeros([info['height'], info['width'], count], dtype=np.uint8)\n", + " \n", + " boxes = []\n", + " \n", + " for i, (shape, _, dims) in enumerate(info['shapes']):\n", + " mask[:, :, i:i+1], box = self.draw_shape(mask[:, :, i:i+1].copy(),\n", + " shape, dims, 1)\n", + " boxes.append(box)\n", + " \n", + " # Handle occlusions\n", + " occlusion = np.logical_not(mask[:, :, -1]).astype(np.uint8)\n", + " for i in range(count-2, -1, -1):\n", + " mask[:, :, i] = mask[:, :, i] * occlusion\n", + " occlusion = np.logical_and(occlusion, np.logical_not(mask[:, :, i]))\n", + " # Map class names to class IDs.\n", + " class_ids = np.array([self.class_names[s[0]] for s in shapes])\n", + " return mask.astype(np.uint8), class_ids.astype(np.int32), boxes\n", + " \n", + " def load_image(self, image_id):\n", + " \"\"\"Generate an image from the specs of the given image ID.\n", + " Typically this function loads the image from a file, but\n", + " in this case it generates the image on the fly from the\n", + " specs in image_info.\n", + " \"\"\"\n", + " info = self.image_info[image_id]\n", + " bg_color = np.array(info['bg_color']).reshape([1, 1, 3])\n", + " image = np.ones([info['height'], info['width'], 3], dtype=np.uint8)\n", + " image = image * bg_color.astype(np.uint8)\n", + " for shape, color, dims in info['shapes']:\n", + " image, _ = self.draw_shape(image, shape, dims, color)\n", + " return image\n", + " \n", + " def __getitem__(self, idx):\n", + " \n", + " \"\"\"Generate an image from the specs of the given image ID.\n", + " Typically this function loads the image from a file, but\n", + " in this case it generates the image on the fly from the\n", + " specs in image_info.\n", + " \"\"\"\n", + " image = Image.fromarray(self.load_image(idx))\n", + " masks, labels, boxes = self.load_mask(idx)\n", + " \n", + " # create a BoxList from the boxes\n", + " boxlist = BoxList(boxes, image.size, mode=\"xyxy\")\n", + "\n", + " # add the labels to the boxlist\n", + " boxlist.add_field(\"labels\", torch.tensor(labels))\n", + "\n", + " # Add masks to the boxlist\n", + " masks = np.transpose(masks, (2,0,1))\n", + " masks = SegmentationMask(torch.tensor(masks), image.size, \"mask\")\n", + " boxlist.add_field(\"masks\", masks)\n", + " \n", + " # Important line! dont forget to add this\n", + " if self.transforms:\n", + " image, boxlist = self.transforms(image, boxlist)\n", + "\n", + " # return the image, the boxlist and the idx in your dataset\n", + " return image, boxlist, idx\n", + " \n", + " \n", + " def __len__(self):\n", + " return self.num_examples\n", + " \n", + "\n", + " def get_img_info(self, idx):\n", + " # get img_height and img_width. This is used if\n", + " # we want to split the batches according to the aspect ratio\n", + " # of the image, as it can be more efficient than loading the\n", + " # image from disk\n", + "\n", + " return {\"height\": self.height, \"width\": self.width}\n", + " \n", + " def get_gt(self):\n", + " # Prepares dataset for coco eval\n", + " \n", + " \n", + " images = []\n", + " annotations = []\n", + " results = []\n", + " \n", + " # Define categories\n", + " categories = [ {\"id\": 1, \"name\": \"square\"}, {\"id\": 2, \"name\": \"circle\"}, {\"id\": 3, \"name\": \"triangle\"}]\n", + "\n", + "\n", + " i = 1\n", + " ann_id = 0\n", + "\n", + " for img_id, d in enumerate(self.image_info):\n", + "\n", + " images.append( {\"id\": img_id, 'height': self.height, 'width': self.width } )\n", + "\n", + " for (shape, color, dims) in d['shapes']:\n", + " \n", + " if shape == \"square\":\n", + " category_id = 1\n", + " elif shape == \"circle\":\n", + " category_id = 2\n", + " elif shape == \"triangle\":\n", + " category_id = 3\n", + " \n", + " x, y, s = dims\n", + " bbox = [ x - s, y - s, x+s, y +s ] \n", + " area = (bbox[0] - bbox[2]) * (bbox[1] - bbox[3])\n", + " \n", + " # Format for COCOC\n", + " annotations.append( {\n", + " \"id\": int(ann_id),\n", + " \"category_id\": category_id,\n", + " \"image_id\": int(img_id),\n", + " \"area\" : float(area),\n", + " \"bbox\": [ float(bbox[0]), float(bbox[1]), float(bbox[2]) - float(bbox[0]) + 1, float(bbox[3]) - float(bbox[1]) + 1 ], # note that the bboxes are in x, y , width, height format\n", + " \"iscrowd\" : 0\n", + " } )\n", + "\n", + " ann_id += 1\n", + "\n", + " # Save ground truth file\n", + " \n", + " with open(\"tmp_gt.json\", \"w\") as f:\n", + " json.dump({\"images\": images, \"annotations\": annotations, \"categories\": categories }, f)\n", + "\n", + " # Load gt for coco eval\n", + " self.coco = COCO(\"tmp_gt.json\") \n", + " " + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2hpTvuSp830x", + "colab_type": "text" + }, + "source": [ + "## Visualise Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BI2ncK7kATEh", + "colab_type": "text" + }, + "source": [ + "### Load" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6nsO_MRUbBpk", + "colab_type": "code", + "outputId": "db8c6102-8663-4bb5-e9e9-c9f305f014f7", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 105 + } + }, + "source": [ + "train_dt = ShapeDataset(100)\n", + "im, boxlist, idx = train_dt[0]" + ], + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "text": [ + "loading annotations into memory...\n", + "Done (t=0.00s)\n", + "creating index...\n", + "index created!\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F9njOSX0AU5-", + "colab_type": "text" + }, + "source": [ + "### Display some sample Images" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nMXB9sAW994F", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 486 + }, + "outputId": "6de8c71d-c656-42a0-eeb0-24982fcad9de" + }, + "source": [ + "rows = 2\n", + "cols = 2\n", + "fig = plt.figure(figsize=(8, 8))\n", + "for i in range(1, rows*cols+1):\n", + " im, boxlist, idx = train_dt[i]\n", + " fig.add_subplot(rows, cols, i)\n", + " plt.imshow(im)\n", + "plt.show()\n", + " " + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAHVCAYAAADLvzPyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+wXHV9//Hnm4QEiZLww6YxoRI0\n/kCnrUyG0tFSp/QHUmtwdCiMrVEzZjpfbLW2Y6M4g3/IjNRWqzOtThQktpQfBRky/WorpVqnnS9o\nQORXBGIESb6BqPzyaxxC5P39Y8/VzeXe5N495+z57O7zMXPn7p7dvfvez+ad1/18zrlnIzORJEnl\nOKLrAiRJ0sEMZ0mSCmM4S5JUGMNZkqTCGM6SJBXGcJYkqTCthXNEnBUR90bEjojY1NbzSGqXvSwN\nX7Txd84RsQC4D/gdYBfwDeD8zLyn8SeT1Bp7WepGWzPn04AdmbkzM/cDVwHrWnouSe2xl6UOLGzp\n564EHuq7vgv4tdnuvODIo3Lh4iUtlSKNpv0/fvQHmfn8jsuYVy8DLDl2US57wXNaLUoaJY//35/w\n48f2x3we01Y4H1ZEbAQ2AixYvIQVv3p2V6VIRXrwf/7pwa5rmKv+fl664ij+11Wv6bgiqRz/cN5/\nz/sxbS1r7wZO7Lu+qtr2M5m5OTPXZubaBQsXt1SGpJoO28twcD8vOXbR0IqTxlVb4fwNYE1ErI6I\nRcB5wNaWnktSe+xlqQOtLGtn5oGIeBfw78AC4LLMvLuN55LUHntZ6kZr+5wz84vAF9v6+ZKGw16W\nhs8zhEmSVBjDWZKkwhjOkiQVxnCWJKkwhrMkSYUxnCVJKozhLElSYQxnSZIKYzhLklQYw1mSpMIY\nzpIkFcZwliSpMIazJEmFMZwlSSqM4SxJUmEMZ0mSCmM4S9IEe+emF/HOTS/qugxNs7DrAiRJzRok\nbOfzmM985Dvz/vmaH2fOkiQVxpmzJI24YS9LT38+Z9LNM5wlaQSVtJ+4vxaDuhkua0uSVJiBZ84R\ncSLweWA5kMDmzPxERBwHXA2cBDwAnJuZj9UvVVJb7OfRUNJseTbOoptRZ+Z8APiLzDwFOB24ICJO\nATYBN2XmGuCm6rqkstnPBRvVP3ca1bpLMHA4Z+aezLytuvwjYDuwElgHbKnutgU4p26RktplP0tl\naWSfc0ScBLwKuAVYnpl7qpseprdMJmlE2M9lGYeZpzPo+at9tHZEPBe4DnhPZj4ZET+7LTMzInKW\nx20ENgIsWLykbhmSGtBEPy9dcdQwSh1b4xxi7o+eu1oz54g4kl4jX5GZX6g2PxIRK6rbVwB7Z3ps\nZm7OzLWZuXbBwsV1ypDUgKb6ecmxi4ZTsDTGBg7n6P1KfSmwPTM/1nfTVmB9dXk9cMPg5UkaBvu5\nDOM8a55ukl7rIOosa78a+GPgzoi4vdr2AeAjwDURsQF4EDi3XomShsB+lgoycDhn5n8DMcvNZw76\ncyUNn/3crUmdRU69bvc/P5tnCJMkqTCGsyRJhTGcJalDk7qk3c8xeDY/lUqSOmAgHcz9zwdz5ixJ\nUmEMZ0kaMmfNs3NsegxnSZIKYzhLklQYDwiTpCFxyXZuPDjMmbMkScUxnCVJKozhLElD4JL2/E3y\nmBnOkiQVxnCWJKkwhrMkSYUxnCVJKox/5yxJLZrkg5qaMKl/8+zMWZKkwhjOkiQVxnCWJKkwhrMk\nSYUxnCVJKkztcI6IBRHxzYj41+r66oi4JSJ2RMTVEbGofpmShsF+lsrQxMz53cD2vuuXAB/PzBcD\njwEbGngOScNhP0sFqBXOEbEK+H3gs9X1AH4LuLa6yxbgnDrPIWk47GepHHVnzn8HvA94prp+PPB4\nZh6oru8CVs70wIjYGBHbImLbTw88VbMMSQ1opJ9//Nj+9iuVxtzA4RwRrwf2Zuatgzw+Mzdn5trM\nXLtg4eJBy5DUgCb7ecmx7pae4tnBmvPOTS+aqPGsc/rOVwNviIizgaOAY4BPAMsiYmH12/YqYHf9\nMiW1zH6WCjLwzDkz35+ZqzLzJOA84D8z8y3AV4A3V3dbD9xQu0pJrbKf2zFp54Nu02c+8p2JGs82\n/s75r4D3RsQOevusLm3hOSQNh/0sdaCRT6XKzK8CX60u7wROa+LnSho++1nqnmcIkySpMIazJEmF\nMZwlSSqM4SxJUmEMZ0mSCmM4S5JUGMNZqjzz8jU88/I1XZchSYazJEmlaeQkJFLJ5jsbns/9j9h+\n/3zL0YSZOuXkJH1oQ5Mm6ZSd/QxnjZVhL0tPfz7DWlITXNaWJKkwzpw1sko8eGummpxNS5ovZ84a\nKVNHVJcYzLMZxZrVvEndd1rHJI+Z4SxJUmEMZ42McZh5OoOWNBeGsyRJhfGAMBVtXGeZ/a/LA8Ym\nh3/zPDeTvK95ijNnFWmSln8n5XVKmjvDWZKkwhjOKsokzZj7TerrnlQu287Oselxn7OKYTj1xsB9\n0JPB/c8HM5QP5sxZkqTC1Jo5R8Qy4LPAK4EE3gHcC1wNnAQ8AJybmY/VqlJjzRnzwX42Hv8z3Ocd\ntX5eePrNXZfQiM9xM29/7Vu6LqNTzpqfre7M+RPAv2Xmy4BfAbYDm4CbMnMNcFN1XZqRwVwU+1kq\nxMDhHBFLgTOASwEyc39mPg6sA7ZUd9sCnFO3SEntsp+lstRZ1l4NfB/4XET8CnAr8G5geWbuqe7z\nMLB8pgdHxEZgI8CCxUtqlKHTNvSG+OuXPtJxJXPnjLk4jfXz0hVHtV/tmPncV68AmLjlbZezZ1cn\nnBcCpwJ/mpm3RMQnmLbklZkZETnTgzNzM7AZYPFzj5/xPvp58DZ13xIC3GAuUmP9vPIVS+3nAX3u\nq1dMTEAbzIdWZ5/zLmBXZt5SXb+WXnM/EhErAKrve+uVKGkI7GepIAPPnDPz4Yh4KCJempn3AmcC\n91Rf64GPVN9vaKTSCTKf2XKdn13CLFplsJ/LMbXEPWWcZtLOlueu7klI/hS4IiIWATuBt9ObjV8T\nERuAB4Fzaz7HRGgzkOfynMMIapezi2c/F2gclroN5fmrFc6ZeTuwdoabzqzzcyUNn/0slcPTd3ao\ni9nybKbX0vRM2lmzNLhRPZr7/Y/uAuDkjusYRZ6+U5Kkwjhz7kBJM+bZnLZhuQeMSYXpP1is1Fn0\n1GxZ9RjOQzYKwTyliZObuJwttaOkoD5cIO/cuBiAkzc/NYxyxoLL2pIkFcaZ85CM0ox5Ope4pbL1\nz6IP3Hx6658RPejS9c6Ni509z5HhPASjHMxTRvH83dKkms/fFU8F+WyPmVqS1nC5rC1JUmGcOas1\nHgwmlW/YM2YPDpsbZ84tOm3D8rFY0u43bq9H0rO5lN09w1mSpMIYzpKkoXN2fmjuc27JOC//Hu7I\nbfc1S6PJwCyHM2dJkgpjOEuSOrFz42Jn67MwnBs2jkdoz2ZSXqc0CboMSQP62QxnSZIKYzhLklQY\nw1mSpML4p1SSNMFK2d/raT0P5sxZkiZUKcGsZzOcJUkqTK1wjog/j4i7I+KuiLgyIo6KiNURcUtE\n7IiIqyNiUVPFSmqP/awSOJvvGTicI2Il8GfA2sx8JbAAOA+4BPh4Zr4YeAzY0EShKt8zL1/jqTtH\nlP08WTz5R/nqLmsvBJ4TEQuBo4E9wG8B11a3bwHOqfkckobDfpYKMfDR2pm5OyL+Bvge8BPgy8Ct\nwOOZeaC62y5g5UyPj4iNwEaABYuXDFpGMSb1bFn9H4JxxPb7AT/4YhQ12c9LVxzVfsEaax65XW9Z\n+1hgHbAaeAGwBDhrro/PzM2ZuTYz1y5YOPrLK1+/9JFZP6VpnE3q6x43TfbzkmPdLV0yl7NHQ51l\n7d8GvpuZ38/Mp4EvAK8GllXLYgCrgN01a5TUPvtZKkidcP4ecHpEHB0RAZwJ3AN8BXhzdZ/1wA31\nSpQ0BPazijPJs/w6+5xviYhrgduAA8A3gc3A/wauiogPV9subaJQSe2xn8ffqAbdpO5/rnX6zsy8\nCLho2uadwGl1fq6k4bOfpXJ4hjBJGnOjOmueZIazJKl4k3biFMNZkqTC+JGRkjSmJmmmOW6cOUuS\nVBjDuWGTdMas2V7nEdvv/9mpPCWpSZOyGuCytiSNmUkJsHHmzFmSpMIYzi0Z56XtSVq6l0bNJMya\nJ+HPqgxntcb9zpLaNM4BbThLklQYw7lF47j8O26vR5JKZDhLkkbWuO5/NpwlSSqM4TwE47AUPOgS\nvSckkaT5M5yHZJT3P49q3ZImx7gtbRvOkiQVxnAeslGahTY523dpW5LmznDuwCgscbdRn/ufJbVp\nnI7cNpwlSSqMn0rVof7Z6WkblndYyWgtt0vSuHPmLElSYQ4bzhFxWUTsjYi7+rYdFxE3RsT91fdj\nq+0REZ+MiB0RcUdEnNpm8eNkaj/0MGewXTwneHBYl+xnTYJx2O88l5nz5cBZ07ZtAm7KzDXATdV1\ngNcBa6qvjcCnmilzsrQZml0F8nQeHNaZy7GfNQFG/eCww4ZzZn4NeHTa5nXAluryFuCcvu2fz56b\ngWURsaKpYiXVYz9Lo2HQA8KWZ+ae6vLDwNTRTCuBh/rut6vatodpImIjvd/GWbB4yYBljL+5zHCn\nDibrejY8iCO2388zL1/TdRmTrtF+XrriqPYqlSZE7aO1MzMjIgd43GZgM8Di5x4/78fr50YxlPtN\nLW8b0j1dLvc30c8rX7HUflYxppa2T978VMeVzM+gR2s/MrW8VX3fW23fDZzYd79V1TZJ5bKfpcIM\nGs5bgfXV5fXADX3b31od5Xk68ETfcpl0SB4g1tkY2M9SYQ67rB0RVwKvBU6IiF3ARcBHgGsiYgPw\nIHBudfcvAmcDO4B9wNtbqFljbFKXuIcVyuPUzwduPr3rEoZilI84LsnOjYtHamn7sOGcmefPctOZ\nM9w3gQvqFiWpHfazNBo8Q5iKNEl/Bz0pr1PS3HlubRWtP7jGaanbQNbhuJzdvFE6ctuZsyRJhTGc\nNTLGYbY5Scv1UqlGYVXCcJYkqTDuc9ZIGcV90M6UNR+jMKtT+wxnjayZQq/rwDaIpdFQ+sFhLmtL\nklQYZ84aK8OeTTtTVpNc0tYUw1ljb64BOhXiBq6GzVDuTqmn9XRZW5KkwjhzlirOmCWVwpmzJHXI\nJe3u7dy4uLj3wXCWJKkwhrMkSYUxnCWpAyUupU66kt4Pw1mSpMIYzpI0ZCXN0HSwUlY0DGdJkgpj\nOEuSVBjDWZKGpJQlUx1e1++T4SxJUmEOG84RcVlE7I2Iu/q2fTQivh0Rd0TE9RGxrO+290fEjoi4\nNyJ+r63CJc2f/SyNhrnMnC8Hzpq27UbglZn5y8B9wPsBIuIU4DzgFdVj/iEiFjRWraS6Lsd+7kTX\ny6Savy53Qxw2nDPza8Cj07Z9OTMPVFdvBlZVl9cBV2XmU5n5XWAHcFqD9UqqwX6WRkMT+5zfAXyp\nurwSeKjvtl3VtmeJiI0RsS0itv30QHmfpSlNqNr9/OPH9rdcojT+an1kZERcCBwArpjvYzNzM7AZ\nYPFzj886dUiqr6l+XvmKpfZzH5ezR9/Ue3jy5uFNJAcO54h4G/B64MzMnGrG3cCJfXdbVW2TVDD7\nWSrLQOEcEWcB7wN+MzP39d20FfjniPgY8AJgDfD12lVKao393K5hzrY0Pg4bzhFxJfBa4ISI2AVc\nRO9ozsXAjREBcHNm/klm3h0R1wD30FseuyAzf9pW8ZLmx36WRsNhwzkzz59h86WHuP/FwMV1ipLU\nDvtZGg2eIUySpMIYzpIkFcZwliSpMIazJEmFMZwlSSqM4SxJUmHi5ycD6rCIiO8DPwZ+0HUtsziB\nMmsrtS4ot7ZS64Jn1/bCzHx+V8UMKiJ+BNzbdR2zGKX3vxSl1gWjU9u8e7mIcAaIiG2ZubbrOmZS\nam2l1gXl1lZqXVB2bfNR8uuwtvkrtS4Y79pc1pYkqTCGsyRJhSkpnDd3XcAhlFpbqXVBubWVWheU\nXdt8lPw6rG3+Sq0Lxri2YvY5S5KknpJmzpIkiQLCOSLOioh7I2JHRGzquJYTI+IrEXFPRNwdEe+u\ntn8oInZHxO3V19kd1fdARNxZ1bCt2nZcRNwYEfdX348dck0v7RuX2yPiyYh4T1djFhGXRcTeiLir\nb9uMYxQ9n6z+7d0REad2UNtHI+Lb1fNfHxHLqu0nRcRP+sbv023W1pRS+tleHrgu+3nwuprt5czs\n7AtYAHwHOBlYBHwLOKXDelYAp1aXnwfcB5wCfAj4yy7HqqrpAeCEadv+GthUXd4EXNLx+/kw8MKu\nxgw4AzgVuOtwYwScDXwJCOB04JYOavtdYGF1+ZK+2k7qv98ofJXUz/ZyY++n/Tz3uhrt5a5nzqcB\nOzJzZ2buB64C1nVVTGbuyczbqss/ArYDK7uqZ47WAVuqy1uAczqs5UzgO5n5YFcFZObXgEenbZ5t\njNYBn8+em4FlEbFimLVl5pcz80B19WZgVVvPPwTF9LO93Aj7eR51Nd3LXYfzSuChvuu7KKSBIuIk\n4FXALdWmd1XLFZd1sdxUSeDLEXFrRGysti3PzD3V5YeB5d2UBsB5wJV910sYM5h9jEr79/cOer/5\nT1kdEd+MiP+KiN/oqqh5KG08AXu5Bvt5cLV7uetwLlJEPBe4DnhPZj4JfAp4EfCrwB7gbzsq7TWZ\neSrwOuCCiDij/8bsraF0cvh9RCwC3gD8S7WplDE7SJdjdCgRcSFwALii2rQH+KXMfBXwXuCfI+KY\nruobVfbyYOznwTXVy12H827gxL7rq6ptnYmII+k18xWZ+QWAzHwkM3+amc8An6G3fDd0mbm7+r4X\nuL6q45GppZvq+94uaqP3n8xtmflIVWMRY1aZbYyK+PcXEW8DXg+8pfrPhsx8KjN/WF2+ld6+3JcM\nu7Z5KmI8p9jLtdjPA2iyl7sO528AayJidfWb2nnA1q6KiYgALgW2Z+bH+rb377d4I3DX9McOobYl\nEfG8qcv0Dj64i954ra/uth64Ydi1Vc6nbwmshDHrM9sYbQXeWh3leTrwRN9y2VBExFnA+4A3ZOa+\nvu3Pj4gF1eWTgTXAzmHWNoBi+tlers1+nqfGe7mto9nm+kXvCLv76P02cWHHtbyG3hLJHcDt1dfZ\nwD8Cd1bbtwIrOqjtZHpHv34LuHtqrIDjgZuA+4H/AI7roLYlwA+BpX3bOhkzev+h7AGeprfPacNs\nY0TvqM6/r/7t3Qms7aC2HfT2k039e/t0dd83Ve/z7cBtwB8M+30d8DUW0c/2cq367OfB6mq0lz1D\nmCRJhel6WVuSJE1jOEuSVBjDWZKkwhjOkiQVxnCWJKkwhrMkSYUxnCVJKozhLElSYQxnSZIKYzhL\nklQYw1mSpMIYzpIkFcZwliSpMIazJEmFMZwlSSqM4SxJUmEMZ0mSCmM4S5JUGMNZkqTCGM6SJBXG\ncJYkqTCthXNEnBUR90bEjojY1NbzSGqXvSwNX2Rm8z80YgFwH/A7wC7gG8D5mXlP408mqTX2stSN\nhS393NOAHZm5EyAirgLWATM29NIjn5PLj1raUinSaLr//z3yg8x8fsdlzKuXAY5Ztih/4RePHlJ5\n42f3omO7LmHeVu5/rOsSirb34X08+fj+mM9j2grnlcBDfdd3Ab/Wf4eI2AhsBPiFxc/jk2vf0lIp\n0mh63Vc/9mDXNTCHXoaD+/n5y5/D33zmN4dT3Rj64Oo3dl3CvH34u9d3XULR/vKd/zXvx7QVzoeV\nmZuBzQAved4vNr+2Lmlo+vv5xS9bZj/P0ygGcr/++g3qZrR1QNhu4MS+66uqbZJGi73cslEP5uk+\nuPqNY/eautBWOH8DWBMRqyNiEXAesLWl55LUHntZ6kAry9qZeSAi3gX8O7AAuCwz727juSS1x15u\nz7jPLj+4+o0ucdfQ2j7nzPwi8MW2fr6k4bCXmzPugTzd9NdrWM+dZwiTJKkwhrMkDcGkzZpn4hjM\nneEsSVJhDGdJkgrT2UlIJGkSuJR7sKnx8OCwQ3PmLGmsXH3xdVx98XVdlwEYzIfi2Bya4SxJUmFc\n1pY0NvpnzFOX//DCNw29DmeFc+MS9+ycOUuSVBjDWZKkwhjOkkbeoQ4CG+bBYX4i02Acs2cznCVJ\nKozhLElSYQxnSSNtLsvWw/jbZ5dm63GXwMEMZ0mSCmM4S5JUGMNZ0kgaZKm6jaVtl2Ob5Vj2GM6S\nJBXGcJY0UUr6YAxpNoazpJFjuGrcGc6SJBXGcJY0Mppckm7i53jwUjs8yK5GOEfEiRHxlYi4JyLu\njoh3V9uPi4gbI+L+6vuxzZUrqQ32s1SWOjPnA8BfZOYpwOnABRFxCrAJuCkz1wA3Vdcllc1+lgoy\ncDhn5p7MvK26/CNgO7ASWAdsqe62BTinbpGS2jUK/dzGQWAeua1SNbLPOSJOAl4F3AIsz8w91U0P\nA8tneczGiNgWEdueeHpfE2VIakDdfn7y8f1DqVMaZ7XDOSKeC1wHvCczn+y/LTMTyJkel5mbM3Nt\nZq5deuTRdcuQ1IAm+vmYZYuGUKk03mqFc0QcSa+Rr8jML1SbH4mIFdXtK4C99UqUNAyl9vMwlp5d\n2lZp6hytHcClwPbM/FjfTVuB9dXl9cANg5cnaRjsZ6ksC2s89tXAHwN3RsTt1bYPAB8BromIDcCD\nwLn1SpQ0BPazVJCBwzkz/xuIWW4+c9CfK2n4Su3nYS43Tz3XH174pqE9pzQbzxAmSVJhDGdJ6jOX\n2bqnlxyOSR7jOvucJakVpR89/eHvXg9MdngMw9Q4TyJnzpIkFcaZs6RilDJj9uAwdc2ZsyRJhTGc\nJUkqjOEsqQilLGn381Or1BXDWZKkwhjOkiQVxnCW1KlRWDouvT6NH8NZkuZgFH6J0PgwnCVJKozh\nLKkzoz4TneTTS7bpw9+9fuLH1nCWJKkwhrMkzcOoz/Y1Gjy3tqShM+CkQ3PmLElSYQxnSZqn/j+r\n8uClZjmWPYazpKFySVs6PMNZkqTC1A7niFgQEd+MiH+trq+OiFsiYkdEXB0Ri+qXKWkY2uzncTzD\nVv/rcTm2HncPHKyJmfO7ge191y8BPp6ZLwYeAzY08ByShsN+lgpQK5wjYhXw+8Bnq+sB/BZwbXWX\nLcA5dZ5D0nDYz1I56s6c/w54H/BMdf144PHMPFBd3wWsrPkckoajtX4et+Xsfh65XZ9j9mwDh3NE\nvB7Ym5m3Dvj4jRGxLSK2PfH0vkHLkNSAJvv5ycf3N1zdaBjnX0A0fHXOEPZq4A0RcTZwFHAM8Alg\nWUQsrH7bXgXsnunBmbkZ2Azwkuf9YtaoQ1J9jfXzi1+2zH6Wahp45pyZ78/MVZl5EnAe8J+Z+Rbg\nK8Cbq7utB26oXaWkVrXVz+N4hPZcuLw9sw+ccfA/H8dpdm38nfNfAe+NiB309lld2sJzSBoO+1nq\nQCMffJGZXwW+Wl3eCZzWxM+VNHz28+CmVgn+8MI3Ab2Z4QdXv7HLkorQP2Oeunz0lmdmu7vwDGGS\nWjKJy9kzcelWgzCcJUkqjJ/nLEkNm768PammHwDWb9/6I1zaPgRnzpIa9eieF7mkPQOXtjUfhrMk\nSYUxnCWpJdNXEKYODpuEWfShlrSn7Ft/BPvWG0MzcVQkqUWznYhlUkJagzGcJUkqjOEsSR0at9nz\nB864YU5L2v1c2n42/5RKkobg6ouvm/VPq/oDehTPKDZVvyHbHEdSkqTCOHOWpIJMX+YudSbd9HL8\n1KzbE5P0GM6SNCSDnDmspCXv2QK5yeVszxzW47K2JEmFceYsSSNimEve43YU+ahx5ixJUmGcOUvS\nkB3qz6rmYz6z26lZdlsHcrXxMyd537PhLEkdGPbHSo5CKOvnHF1JkgpjOEuSijTJs/PJfeWSVICZ\nPrGqdJMcmsPiCEuSVJha4RwRyyLi2oj4dkRsj4hfj4jjIuLGiLi/+n5sU8VKao/9rBLtW3/ERM7U\n677iTwD/lpkvA34F2A5sAm7KzDXATdV1SeWznzty9cXXjcTydpdBOWkhPfArjYilwBnApQCZuT8z\nHwfWAVuqu20BzqlbpKR22c9SWer8GrIa+D7wuYj4ZkR8NiKWAMszc091n4eB5XWLlNQ6+1kqSJ1w\nXgicCnwqM18F/JhpS16ZmUDO9OCI2BgR2yJi2xNP76tRhqQGNNbPT+3/QevFjquSl7ZLWVIupY62\n1XmVu4BdmXlLdf1aes39SESsAKi+753pwZm5OTPXZubapUceXaMMSQ1orJ8XLzphKAVL42zgcM7M\nh4GHIuKl1aYzgXuArcD6att64IZaFUpqnf2s2UzagVilqHtu7T8FroiIRcBO4O30Av+aiNgAPAic\nW/M5JA2H/VyAYZ9zexRNwgdj1ArnzLwdWDvDTWfW+bmShs9+lsrhp1JJUoG6nkG7lN0tR1+SpMIY\nzpKkkTTOs/vxfWWSNAa6+NvncQ69UeE7IElSYTwgTJIEjOaMeVz/rGr03glJmjCj8qlVXRrFXywO\nZbxejSRJY8BwliSN3cxz1PluSNKIcGn70MbpPODj8SokSRojHq0tqVHHrfiOH9owQsZlpjlufFck\nSWNlHH7hGP1XIEnSmDGcJWlCjcMMc1y5z7kD/+efTuq6hE79+h890HUJ0kSbhFAe9TOHjf87JEnS\niDGcJUkqjOEsSVJhDGdJmiCTsL+536ieNWz0KpYkaZ5GLaBHq1pJkiZArXCOiD+PiLsj4q6IuDIi\njoqI1RFxS0TsiIirI2JRU8VKao/9PN5GdXl3Ug38TkXESuDPgLWZ+UpgAXAecAnw8cx8MfAYsKGJ\nQiW1x37WJBilX1DqVrkQeE5ELASOBvYAvwVcW92+BTin5nNIGg77WSrEwOGcmbuBvwG+R6+JnwBu\nBR7PzAPV3XYBK2d6fERsjIhtEbHtiaf3DVqGpAY02c9PPr5/GCVrHkZltqifq7OsfSywDlgNvABY\nApw118dn5ubMXJuZa5ceefSgZUhqQJP9fMwyd0uXYpSWcYdpFMakToW/DXw3M7+fmU8DXwBeDSyr\nlsUAVgG7a9YoqX32s1SQOuH8PeD0iDg6IgI4E7gH+Arw5uo+64Eb6pUoaQjsZ6kgdfY530LvQJHb\ngDurn7UZ+CvgvRGxAzgeuLSpnrMjAAAIr0lEQVSBOiW1yH4eLy5nH17pY1TrIyMz8yLgommbdwKn\n1fm5kobPfpbKUe6vDZIktazU2XOZVUmSBlJq2Gh+fBclSSpMrX3OkqQyOGMe3NTYHb3lmY4r+Tnf\nTUmSCmM4S5JUGMNZkkacS9rNKGkcy6lEkiQBhrMkScXxaG1JGlElLcOOi1KO3PadlSSpMIazJEmF\nMZwlaQS5pN2ursfXd1eSpMJ4QJgkjaCuD1hSu5w5S5JUGMNZkqTCGM6SJBXGcJYkqTCGsyRJhTGc\nJUkqjOEsSVJhDhvOEXFZROyNiLv6th0XETdGxP3V92Or7RERn4yIHRFxR0Sc2mbxkubHfpZGw1xm\nzpcDZ03btgm4KTPXADdV1wFeB6ypvjYCn2qmTEkNuRz7WSreYcM5M78GPDpt8zpgS3V5C3BO3/bP\nZ8/NwLKIWNFUsZLqsZ+l0TDoPuflmbmnuvwwsLy6vBJ4qO9+u6ptzxIRGyNiW0Rse+LpfQOWIakB\njfbzk4/vb69SaULUPiAsMxPIAR63OTPXZubapUceXbcMSQ1oop+PWbaohcqkyTJoOD8ytbxVfd9b\nbd8NnNh3v1XVNknlsp+lwgwazluB9dXl9cANfdvfWh3leTrwRN9ymaQy2c9SYQ77kZERcSXwWuCE\niNgFXAR8BLgmIjYADwLnVnf/InA2sAPYB7y9hZolDch+lkbDYcM5M8+f5aYzZ7hvAhfULUpSO+xn\naTR4hjBJkgpjOEuSVBjDWZKkwhjOkiQVxnCWJKkwhrMkSYUxnCVJKozhLElSYQxnSZIKYzhLklQY\nw1mSpMIYzpIkFcZwliSpMIazJEmFMZwlSSqM4SxJUmEMZ0mSCrOw6wIm0a//0QNdlyBJKpgzZ0mS\nCmM4S5JUGMNZkqTCGM6SJBXmsOEcEZdFxN6IuKtv20cj4tsRcUdEXB8Ry/pue39E7IiIeyPi99oq\nXNL82c/SaJjLzPly4Kxp224EXpmZvwzcB7wfICJOAc4DXlE95h8iYkFj1Uqq63LsZ6l4hw3nzPwa\n8Oi0bV/OzAPV1ZuBVdXldcBVmflUZn4X2AGc1mC9kmqwn6XR0MQ+53cAX6ourwQe6rttV7XtWSJi\nY0Rsi4htTzy9r4EyJDWgdj8/+fj+lkuUxl+tcI6IC4EDwBXzfWxmbs7MtZm5dumRR9cpQ1IDmurn\nY5Ytar44acIMfIawiHgb8HrgzMzMavNu4MS+u62qtkkqmP0slWWgmXNEnAW8D3hDZvavSW8FzouI\nxRGxGlgDfL1+mZLaYj9L5TnszDkirgReC5wQEbuAi+gdzbkYuDEiAG7OzD/JzLsj4hrgHnrLYxdk\n5k/bKl7S/NjP0mg4bDhn5vkzbL70EPe/GLi4TlGS2mE/S6PBM4RJklQYw1mSpMIYzpIkFcZwliSp\nMIazJEmFMZwlSSpM/PxkQB0WEfF94MfAD7quZRYnUGZtpdYF5dZWal3w7NpemJnP76qYQUXEj4B7\nu65jFqP0/pei1LpgdGqbdy8XEc4AEbEtM9d2XcdMSq2t1Lqg3NpKrQvKrm0+Sn4d1jZ/pdYF412b\ny9qSJBXGcJYkqTAlhfPmrgs4hFJrK7UuKLe2UuuCsmubj5Jfh7XNX6l1wRjXVsw+Z0mS1FPSzFmS\nJGE4S5JUnM7DOSLOioh7I2JHRGzquJYTI+IrEXFPRNwdEe+utn8oInZHxO3V19kd1fdARNxZ1bCt\n2nZcRNwYEfdX348dck0v7RuX2yPiyYh4T1djFhGXRcTeiLirb9uMYxQ9n6z+7d0REad2UNtHI+Lb\n1fNfHxHLqu0nRcRP+sbv023W1pRS+tleHrgu+3nwuprt5czs7AtYAHwHOBlYBHwLOKXDelYAp1aX\nnwfcB5wCfAj4yy7HqqrpAeCEadv+GthUXd4EXNLx+/kw8MKuxgw4AzgVuOtwYwScDXwJCOB04JYO\navtdYGF1+ZK+2k7qv98ofJXUz/ZyY++n/Tz3uhrt5a5nzqcBOzJzZ2buB64C1nVVTGbuyczbqss/\nArYDK7uqZ47WAVuqy1uAczqs5UzgO5n5YFcFZObXgEenbZ5tjNYBn8+em4FlEbFimLVl5pcz80B1\n9WZgVVvPPwTF9LO93Aj7eR51Nd3LXYfzSuChvuu7KKSBIuIk4FXALdWmd1XLFZd1sdxUSeDLEXFr\nRGysti3PzD3V5YeB5d2UBsB5wJV910sYM5h9jEr79/cOer/5T1kdEd+MiP+KiN/oqqh5KG08AXu5\nBvt5cLV7uetwLlJEPBe4DnhPZj4JfAp4EfCrwB7gbzsq7TWZeSrwOuCCiDij/8bsraF08rdxEbEI\neAPwL9WmUsbsIF2O0aFExIXAAeCKatMe4Jcy81XAe4F/johjuqpvVNnLg7GfB9dUL3cdzruBE/uu\nr6q2dSYijqTXzFdk5hcAMvORzPxpZj4DfIbe8t3QZebu6vte4Pqqjkemlm6q73u7qI3efzK3ZeYj\nVY1FjFlltjEq4t9fRLwNeD3wluo/GzLzqcz8YXX5Vnr7cl8y7NrmqYjxnGIv12I/D6DJXu46nL8B\nrImI1dVvaucBW7sqJiICuBTYnpkf69vev9/ijcBd0x87hNqWRMTzpi7TO/jgLnrjtb6623rghmHX\nVjmfviWwEsasz2xjtBV4a3WU5+nAE33LZUMREWcB7wPekJn7+rY/PyIWVJdPBtYAO4dZ2wCK6Wd7\nuTb7eZ4a7+W2jmab6xe9I+zuo/fbxIUd1/IaekskdwC3V19nA/8I3Flt3wqs6KC2k+kd/fot4O6p\nsQKOB24C7gf+Aziug9qWAD8ElvZt62TM6P2Hsgd4mt4+pw2zjRG9ozr/vvq3dyewtoPadtDbTzb1\n7+3T1X3fVL3PtwO3AX8w7Pd1wNdYRD/by7Xqs58Hq6vRXvb0nZIkFabrZW1JkjSN4SxJUmEMZ0mS\nCmM4S5JUGMNZkqTCGM6SJBXGcJYkqTD/HyG3uDbwtsBXAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ORQXaa6k30yD", + "colab_type": "text" + }, + "source": [ + "# Training a Model\n", + "\n", + "Now we move on to training our very own model. Here we will be finetuning a Mask RCNN on this dataset. To do this we need\n", + "\n", + "1. A base model that has the same amount of output classes as our dataset. In this case, we have need for only 3 classes instead of COCO's 80. Hence , we first need to do some model trimming. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SVaNqbpiAzwx", + "colab_type": "text" + }, + "source": [ + "## Model Trimming" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hbzY16ocEdrg", + "colab_type": "text" + }, + "source": [ + "### Helper Functions for Visualising Detections" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "yk5a6RpsEdIt", + "colab_type": "code", + "colab": {} + }, + "source": [ + "class Resize(object):\n", + " def __init__(self, min_size, max_size):\n", + " self.min_size = min_size\n", + " self.max_size = max_size\n", + "\n", + " # modified from torchvision to add support for max size\n", + " def get_size(self, image_size):\n", + " w, h = image_size\n", + " size = self.min_size\n", + " max_size = self.max_size\n", + " if max_size is not None:\n", + " min_original_size = float(min((w, h)))\n", + " max_original_size = float(max((w, h)))\n", + " if max_original_size / min_original_size * size > max_size:\n", + " size = int(round(max_size * min_original_size / max_original_size))\n", + "\n", + " if (w <= h and w == size) or (h <= w and h == size):\n", + " return (h, w)\n", + "\n", + " if w < h:\n", + " ow = size\n", + " oh = int(size * h / w)\n", + " else:\n", + " oh = size\n", + " ow = int(size * w / h)\n", + "\n", + " return (oh, ow)\n", + "\n", + " def __call__(self, image):\n", + " size = self.get_size(image.size)\n", + " image = F.resize(image, size)\n", + " return image\n", + " \n", + " \n", + "class COCODemo(object):\n", + " \n", + " def __init__(\n", + " self,\n", + " cfg,\n", + " confidence_threshold=0.7,\n", + " show_mask_heatmaps=False,\n", + " masks_per_dim=2,\n", + " min_image_size=224,\n", + " ):\n", + " self.cfg = cfg.clone()\n", + " self.model = build_detection_model(cfg)\n", + " self.model.eval()\n", + " self.device = torch.device(cfg.MODEL.DEVICE)\n", + " self.model.to(self.device)\n", + " self.min_image_size = min_image_size\n", + "\n", + " save_dir = cfg.OUTPUT_DIR\n", + " checkpointer = DetectronCheckpointer(cfg, self.model, save_dir=save_dir)\n", + " _ = checkpointer.load(cfg.MODEL.WEIGHT)\n", + "\n", + " self.transforms = self.build_transform()\n", + "\n", + " mask_threshold = -1 if show_mask_heatmaps else 0.5\n", + " self.masker = Masker(threshold=mask_threshold, padding=1)\n", + "\n", + " # used to make colors for each class\n", + " self.palette = torch.tensor([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])\n", + "\n", + " self.cpu_device = torch.device(\"cpu\")\n", + " self.confidence_threshold = confidence_threshold\n", + " self.show_mask_heatmaps = show_mask_heatmaps\n", + " self.masks_per_dim = masks_per_dim\n", + "\n", + " def build_transform(self):\n", + " \"\"\"\n", + " Creates a basic transformation that was used to train the models\n", + " \"\"\"\n", + " cfg = self.cfg\n", + "\n", + " # we are loading images with OpenCV, so we don't need to convert them\n", + " # to BGR, they are already! So all we need to do is to normalize\n", + " # by 255 if we want to convert to BGR255 format, or flip the channels\n", + " # if we want it to be in RGB in [0-1] range.\n", + " if cfg.INPUT.TO_BGR255:\n", + " to_bgr_transform = T.Lambda(lambda x: x * 255)\n", + " else:\n", + " to_bgr_transform = T.Lambda(lambda x: x[[2, 1, 0]])\n", + "\n", + " normalize_transform = T.Normalize(\n", + " mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD\n", + " )\n", + " min_size = cfg.INPUT.MIN_SIZE_TEST\n", + " max_size = cfg.INPUT.MAX_SIZE_TEST\n", + " transform = T.Compose(\n", + " [\n", + " T.ToPILImage(),\n", + " Resize(min_size, max_size),\n", + " T.ToTensor(),\n", + " to_bgr_transform,\n", + " normalize_transform,\n", + " ]\n", + " )\n", + " return transform\n", + "\n", + " def run_on_opencv_image(self, image):\n", + " \"\"\"\n", + " Arguments:\n", + " image (np.ndarray): an image as returned by OpenCV\n", + " Returns:\n", + " prediction (BoxList): the detected objects. Additional information\n", + " of the detection properties can be found in the fields of\n", + " the BoxList via `prediction.fields()`\n", + " \"\"\"\n", + " predictions = self.compute_prediction(image)\n", + " top_predictions = self.select_top_predictions(predictions)\n", + "\n", + " result = image.copy()\n", + " if self.show_mask_heatmaps:\n", + " return self.create_mask_montage(result, top_predictions)\n", + " result = self.overlay_boxes(result, top_predictions)\n", + " if self.cfg.MODEL.MASK_ON:\n", + " result = self.overlay_mask(result, top_predictions)\n", + " if self.cfg.MODEL.KEYPOINT_ON:\n", + " result = self.overlay_keypoints(result, top_predictions)\n", + " result = self.overlay_class_names(result, top_predictions)\n", + "\n", + " return result\n", + "\n", + " def compute_prediction(self, original_image):\n", + " \"\"\"\n", + " Arguments:\n", + " original_image (np.ndarray): an image as returned by OpenCV\n", + " Returns:\n", + " prediction (BoxList): the detected objects. Additional information\n", + " of the detection properties can be found in the fields of\n", + " the BoxList via `prediction.fields()`\n", + " \"\"\"\n", + " # apply pre-processing to image\n", + " image = self.transforms(original_image)\n", + " # convert to an ImageList, padded so that it is divisible by\n", + " # cfg.DATALOADER.SIZE_DIVISIBILITY\n", + " image_list = to_image_list(image, self.cfg.DATALOADER.SIZE_DIVISIBILITY)\n", + " image_list = image_list.to(self.device)\n", + " # compute predictions\n", + " with torch.no_grad():\n", + " predictions = self.model(image_list)\n", + " predictions = [o.to(self.cpu_device) for o in predictions]\n", + "\n", + " # always single image is passed at a time\n", + " prediction = predictions[0]\n", + "\n", + " # reshape prediction (a BoxList) into the original image size\n", + " height, width = original_image.shape[:-1]\n", + " prediction = prediction.resize((width, height))\n", + "\n", + " if prediction.has_field(\"mask\"):\n", + " # if we have masks, paste the masks in the right position\n", + " # in the image, as defined by the bounding boxes\n", + " masks = prediction.get_field(\"mask\")\n", + " # always single image is passed at a time\n", + " masks = self.masker([masks], [prediction])[0]\n", + " prediction.add_field(\"mask\", masks)\n", + " return prediction\n", + "\n", + " def select_top_predictions(self, predictions):\n", + " \"\"\"\n", + " Select only predictions which have a `score` > self.confidence_threshold,\n", + " and returns the predictions in descending order of score\n", + " Arguments:\n", + " predictions (BoxList): the result of the computation by the model.\n", + " It should contain the field `scores`.\n", + " Returns:\n", + " prediction (BoxList): the detected objects. Additional information\n", + " of the detection properties can be found in the fields of\n", + " the BoxList via `prediction.fields()`\n", + " \"\"\"\n", + " scores = predictions.get_field(\"scores\")\n", + " keep = torch.nonzero(scores > self.confidence_threshold).squeeze(1)\n", + " predictions = predictions[keep]\n", + " scores = predictions.get_field(\"scores\")\n", + " _, idx = scores.sort(0, descending=True)\n", + " return predictions[idx]\n", + "\n", + " def compute_colors_for_labels(self, labels):\n", + " \"\"\"\n", + " Simple function that adds fixed colors depending on the class\n", + " \"\"\"\n", + " colors = labels[:, None] * self.palette\n", + " colors = (colors % 255).numpy().astype(\"uint8\")\n", + " return colors\n", + "\n", + " def overlay_boxes(self, image, predictions):\n", + " \"\"\"\n", + " Adds the predicted boxes on top of the image\n", + " Arguments:\n", + " image (np.ndarray): an image as returned by OpenCV\n", + " predictions (BoxList): the result of the computation by the model.\n", + " It should contain the field `labels`.\n", + " \"\"\"\n", + " labels = predictions.get_field(\"labels\")\n", + " boxes = predictions.bbox\n", + "\n", + " colors = self.compute_colors_for_labels(labels).tolist()\n", + "\n", + " for box, color in zip(boxes, colors):\n", + " box = box.to(torch.int64)\n", + " top_left, bottom_right = box[:2].tolist(), box[2:].tolist()\n", + " image = cv2.rectangle(\n", + " image, tuple(top_left), tuple(bottom_right), tuple(color), 1\n", + " )\n", + "\n", + " return image\n", + "\n", + " def overlay_mask(self, image, predictions):\n", + " \"\"\"\n", + " Adds the instances contours for each predicted object.\n", + " Each label has a different color.\n", + " Arguments:\n", + " image (np.ndarray): an image as returned by OpenCV\n", + " predictions (BoxList): the result of the computation by the model.\n", + " It should contain the field `mask` and `labels`.\n", + " \"\"\"\n", + " masks = predictions.get_field(\"mask\").numpy()\n", + " labels = predictions.get_field(\"labels\")\n", + "\n", + " colors = self.compute_colors_for_labels(labels).tolist()\n", + "\n", + " for mask, color in zip(masks, colors):\n", + " thresh = mask[0, :, :, None]\n", + " contours, hierarchy = cv2_util.findContours(\n", + " thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE\n", + " )\n", + " image = cv2.drawContours(image, contours, -1, color, 3)\n", + "\n", + " composite = image\n", + "\n", + " return composite\n", + "\n", + " def overlay_keypoints(self, image, predictions):\n", + " keypoints = predictions.get_field(\"keypoints\")\n", + " kps = keypoints.keypoints\n", + " scores = keypoints.get_field(\"logits\")\n", + " kps = torch.cat((kps[:, :, 0:2], scores[:, :, None]), dim=2).numpy()\n", + " for region in kps:\n", + " image = vis_keypoints(image, region.transpose((1, 0)))\n", + " return image\n", + "\n", + " def create_mask_montage(self, image, predictions):\n", + " \"\"\"\n", + " Create a montage showing the probability heatmaps for each one one of the\n", + " detected objects\n", + " Arguments:\n", + " image (np.ndarray): an image as returned by OpenCV\n", + " predictions (BoxList): the result of the computation by the model.\n", + " It should contain the field `mask`.\n", + " \"\"\"\n", + " masks = predictions.get_field(\"mask\")\n", + " masks_per_dim = self.masks_per_dim\n", + " masks = L.interpolate(\n", + " masks.float(), scale_factor=1 / masks_per_dim\n", + " ).byte()\n", + " height, width = masks.shape[-2:]\n", + " max_masks = masks_per_dim ** 2\n", + " masks = masks[:max_masks]\n", + " # handle case where we have less detections than max_masks\n", + " if len(masks) < max_masks:\n", + " masks_padded = torch.zeros(max_masks, 1, height, width, dtype=torch.uint8)\n", + " masks_padded[: len(masks)] = masks\n", + " masks = masks_padded\n", + " masks = masks.reshape(masks_per_dim, masks_per_dim, height, width)\n", + " result = torch.zeros(\n", + " (masks_per_dim * height, masks_per_dim * width), dtype=torch.uint8\n", + " )\n", + " for y in range(masks_per_dim):\n", + " start_y = y * height\n", + " end_y = (y + 1) * height\n", + " for x in range(masks_per_dim):\n", + " start_x = x * width\n", + " end_x = (x + 1) * width\n", + " result[start_y:end_y, start_x:end_x] = masks[y, x]\n", + " return cv2.applyColorMap(result.numpy(), cv2.COLORMAP_JET)\n", + "\n", + " def overlay_class_names(self, image, predictions):\n", + " \"\"\"\n", + " Adds detected class names and scores in the positions defined by the\n", + " top-left corner of the predicted bounding box\n", + " Arguments:\n", + " image (np.ndarray): an image as returned by OpenCV\n", + " predictions (BoxList): the result of the computation by the model.\n", + " It should contain the field `scores` and `labels`.\n", + " \"\"\"\n", + " scores = predictions.get_field(\"scores\").tolist()\n", + " labels = predictions.get_field(\"labels\").tolist()\n", + " labels = [self.CATEGORIES[i] for i in labels]\n", + " boxes = predictions.bbox\n", + "\n", + " template = \"{}: {:.2f}\"\n", + " for box, score, label in zip(boxes, scores, labels):\n", + " x, y = box[:2]\n", + " s = template.format(label, score)\n", + " cv2.putText(\n", + " image, s, (x, y), cv2.FONT_HERSHEY_SIMPLEX, .5, (255, 255, 255), 1\n", + " )\n", + "\n", + " return image\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from maskrcnn_benchmark.structures.keypoint import PersonKeypoints\n", + "\n", + "def vis_keypoints(img, kps, kp_thresh=2, alpha=0.7):\n", + " \"\"\"Visualizes keypoints (adapted from vis_one_image).\n", + " kps has shape (4, #keypoints) where 4 rows are (x, y, logit, prob).\n", + " \"\"\"\n", + " dataset_keypoints = PersonKeypoints.NAMES\n", + " kp_lines = PersonKeypoints.CONNECTIONS\n", + "\n", + " # Convert from plt 0-1 RGBA colors to 0-255 BGR colors for opencv.\n", + " cmap = plt.get_cmap('rainbow')\n", + " colors = [cmap(i) for i in np.linspace(0, 1, len(kp_lines) + 2)]\n", + " colors = [(c[2] * 255, c[1] * 255, c[0] * 255) for c in colors]\n", + "\n", + " # Perform the drawing on a copy of the image, to allow for blending.\n", + " kp_mask = np.copy(img)\n", + "\n", + " # Draw mid shoulder / mid hip first for better visualization.\n", + " mid_shoulder = (\n", + " kps[:2, dataset_keypoints.index('right_shoulder')] +\n", + " kps[:2, dataset_keypoints.index('left_shoulder')]) / 2.0\n", + " sc_mid_shoulder = np.minimum(\n", + " kps[2, dataset_keypoints.index('right_shoulder')],\n", + " kps[2, dataset_keypoints.index('left_shoulder')])\n", + " mid_hip = (\n", + " kps[:2, dataset_keypoints.index('right_hip')] +\n", + " kps[:2, dataset_keypoints.index('left_hip')]) / 2.0\n", + " sc_mid_hip = np.minimum(\n", + " kps[2, dataset_keypoints.index('right_hip')],\n", + " kps[2, dataset_keypoints.index('left_hip')])\n", + " nose_idx = dataset_keypoints.index('nose')\n", + " if sc_mid_shoulder > kp_thresh and kps[2, nose_idx] > kp_thresh:\n", + " cv2.line(\n", + " kp_mask, tuple(mid_shoulder), tuple(kps[:2, nose_idx]),\n", + " color=colors[len(kp_lines)], thickness=2, lineType=cv2.LINE_AA)\n", + " if sc_mid_shoulder > kp_thresh and sc_mid_hip > kp_thresh:\n", + " cv2.line(\n", + " kp_mask, tuple(mid_shoulder), tuple(mid_hip),\n", + " color=colors[len(kp_lines) + 1], thickness=2, lineType=cv2.LINE_AA)\n", + "\n", + " # Draw the keypoints.\n", + " for l in range(len(kp_lines)):\n", + " i1 = kp_lines[l][0]\n", + " i2 = kp_lines[l][1]\n", + " p1 = kps[0, i1], kps[1, i1]\n", + " p2 = kps[0, i2], kps[1, i2]\n", + " if kps[2, i1] > kp_thresh and kps[2, i2] > kp_thresh:\n", + " cv2.line(\n", + " kp_mask, p1, p2,\n", + " color=colors[l], thickness=2, lineType=cv2.LINE_AA)\n", + " if kps[2, i1] > kp_thresh:\n", + " cv2.circle(\n", + " kp_mask, p1,\n", + " radius=3, color=colors[l], thickness=-1, lineType=cv2.LINE_AA)\n", + " if kps[2, i2] > kp_thresh:\n", + " cv2.circle(\n", + " kp_mask, p2,\n", + " radius=3, color=colors[l], thickness=-1, lineType=cv2.LINE_AA)\n", + "\n", + " # Blend the keypoints.\n", + " return cv2.addWeighted(img, 1.0 - alpha, kp_mask, alpha, 0)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "If8z4OZfDHmC", + "colab_type": "text" + }, + "source": [ + "### Base Model Config\n", + "\n", + "This is the base model that we will finetune from. First we need to replace the bounding box heads and mask heads to make it compatible with our Shapes Dataset." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "wM0coO44ClbV", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "9a9c1f2d-0f6d-420e-b737-30b295b15935" + }, + "source": [ + "%%writefile base_config.yaml\n", + "MODEL:\n", + " META_ARCHITECTURE: \"GeneralizedRCNN\"\n", + " WEIGHT: \"catalog://Caffe2Detectron/COCO/35858933/e2e_mask_rcnn_R-50-FPN_1x\"\n", + " BACKBONE:\n", + " CONV_BODY: \"R-50-FPN\"\n", + " RESNETS:\n", + " BACKBONE_OUT_CHANNELS: 256\n", + " RPN:\n", + " USE_FPN: True\n", + " ANCHOR_STRIDE: (4, 8, 16, 32, 64)\n", + " PRE_NMS_TOP_N_TRAIN: 2000\n", + " PRE_NMS_TOP_N_TEST: 1000\n", + " POST_NMS_TOP_N_TEST: 1000\n", + " FPN_POST_NMS_TOP_N_TEST: 1000\n", + " ROI_HEADS:\n", + " USE_FPN: True\n", + " ROI_BOX_HEAD:\n", + " POOLER_RESOLUTION: 7\n", + " POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125)\n", + " POOLER_SAMPLING_RATIO: 2\n", + " FEATURE_EXTRACTOR: \"FPN2MLPFeatureExtractor\"\n", + " PREDICTOR: \"FPNPredictor\"\n", + " ROI_MASK_HEAD:\n", + " POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125)\n", + " FEATURE_EXTRACTOR: \"MaskRCNNFPNFeatureExtractor\"\n", + " PREDICTOR: \"MaskRCNNC4Predictor\"\n", + " POOLER_RESOLUTION: 14\n", + " POOLER_SAMPLING_RATIO: 2\n", + " RESOLUTION: 28\n", + " SHARE_BOX_FEATURE_EXTRACTOR: False\n", + " MASK_ON: True\n", + "DATALOADER:\n", + " SIZE_DIVISIBILITY: 32" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Writing base_config.yaml\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mOo-0LGFEAmc", + "colab_type": "text" + }, + "source": [ + "### Pretrained weight removal\n", + "\n", + "Here, the pretrained weights of bbox, mask and class predictions are removed. This is done so that we can make the model shapes dataset compatible i.e predict 3 classes instead of Coco's 81 classes." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ISFsxBxBDZcQ", + "colab_type": "code", + "colab": {} + }, + "source": [ + "\n", + "def removekey(d, listofkeys):\n", + " r = dict(d)\n", + " for key in listofkeys:\n", + " print('key: {} is removed'.format(key))\n", + " r.pop(key)\n", + " return r\n", + " \n", + "logger_dir = 'log'\n", + "\n", + "if logger_dir:\n", + " mkdir(logger_dir)\n", + "\n", + "logger = setup_logger(\"maskrcnn_benchmark\", logger_dir, get_rank())\n", + "logger.info(\"Using {} GPUs\".format(1))\n", + "\n", + "config_file = \"base_config.yaml\"\n", + "\n", + "# update the config options with the config file\n", + "cfg.merge_from_file(config_file)\n", + "\n", + "\n", + "# Add these for printing class names over your predictions.\n", + "COCODemo.CATEGORIES = [\n", + " \"__background\",\n", + " \"square\",\n", + " \"circle\",\n", + " \"triangle\"\n", + "]\n", + "\n", + "demo = COCODemo(\n", + " cfg, \n", + " min_image_size=800,\n", + " confidence_threshold=0.7)\n", + "\n", + "base_model = demo.model\n", + "\n", + "# Removes pretrained weights from state dict\n", + "new_state_dict = removekey(base_model.state_dict(), [ \n", + " \"roi_heads.box.predictor.cls_score.weight\", \"roi_heads.box.predictor.cls_score.bias\", \n", + " \"roi_heads.box.predictor.bbox_pred.weight\", \"roi_heads.box.predictor.bbox_pred.bias\",\n", + " \"roi_heads.mask.predictor.mask_fcn_logits.weight\", \"roi_heads.mask.predictor.mask_fcn_logits.bias\"\n", + " ])\n", + "\n", + "# Save new state dict, we will use this as our starting weights for our finwe tuned model\n", + "torch.save(new_state_dict, \"base_model.pth\")\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bbCBInqHFUg7", + "colab_type": "text" + }, + "source": [ + "### Fine Tuned Model Config\n", + "\n", + "Here we define our shape Dataset config. The important fields are \n", + "\n", + "1. WEIGHT: which point to our base_model.pth saved in the previous step\n", + "2. NUM_CLASSES: Which define how many classes we will rpedict . note that the number includes the background, hence our shapes dataset has 4 classes. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "5AhIiTgmFXyi", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "outputId": "94f21f80-5baa-4a67-945a-0bfd726f5d11" + }, + "source": [ + "%%writefile shapes_config.yaml\n", + "MODEL:\n", + " META_ARCHITECTURE: \"GeneralizedRCNN\"\n", + " WEIGHT: \"base_model.pth\"\n", + " BACKBONE:\n", + " CONV_BODY: \"R-50-FPN\"\n", + " RESNETS:\n", + " BACKBONE_OUT_CHANNELS: 256\n", + " RPN:\n", + " USE_FPN: True\n", + " ANCHOR_STRIDE: (4, 8, 16, 32, 64)\n", + " PRE_NMS_TOP_N_TRAIN: 2000\n", + " PRE_NMS_TOP_N_TEST: 1000\n", + " POST_NMS_TOP_N_TEST: 1000\n", + " FPN_POST_NMS_TOP_N_TEST: 1000\n", + " ROI_HEADS:\n", + " USE_FPN: True\n", + " ROI_BOX_HEAD:\n", + " POOLER_RESOLUTION: 7\n", + " POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125)\n", + " POOLER_SAMPLING_RATIO: 2\n", + " FEATURE_EXTRACTOR: \"FPN2MLPFeatureExtractor\"\n", + " PREDICTOR: \"FPNPredictor\"\n", + " NUM_CLASSES: 4 # background + num_classes : IMPORTANT dont forget to add this\n", + " ROI_MASK_HEAD:\n", + " POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125)\n", + " FEATURE_EXTRACTOR: \"MaskRCNNFPNFeatureExtractor\"\n", + " PREDICTOR: \"MaskRCNNC4Predictor\"\n", + " POOLER_RESOLUTION: 14\n", + " POOLER_SAMPLING_RATIO: 2\n", + " RESOLUTION: 28\n", + " SHARE_BOX_FEATURE_EXTRACTOR: False\n", + " MASK_ON: True\n", + "DATALOADER:\n", + " SIZE_DIVISIBILITY: 32" + ], + "execution_count": 10, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Writing shapes_config.yaml\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tAn3omCjTFGI", + "colab_type": "text" + }, + "source": [ + "### Data Loader\n", + "\n", + "This function creates a data loader with our shapes dataset. This data loader is used internally in the repo to train the model." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "oODu2UpVTHXz", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def build_data_loader(cfg, dataset, is_train=True, is_distributed=False, start_iter=0):\n", + " num_gpus = get_world_size()\n", + " if is_train:\n", + " images_per_batch = cfg.SOLVER.IMS_PER_BATCH\n", + " assert (\n", + " images_per_batch % num_gpus == 0\n", + " ), \"SOLVER.IMS_PER_BATCH ({}) must be divisible by the number of GPUs ({}) used.\".format(\n", + " images_per_batch, num_gpus)\n", + " images_per_gpu = images_per_batch // num_gpus\n", + " shuffle = True\n", + " num_iters = cfg.SOLVER.MAX_ITER\n", + " else:\n", + " images_per_batch = cfg.TEST.IMS_PER_BATCH\n", + " assert (\n", + " images_per_batch % num_gpus == 0\n", + " ), \"TEST.IMS_PER_BATCH ({}) must be divisible by the number of GPUs ({}) used.\".format(\n", + " images_per_batch, num_gpus)\n", + " images_per_gpu = images_per_batch // num_gpus\n", + " shuffle = False if not is_distributed else True\n", + " num_iters = None\n", + " start_iter = 0\n", + "\n", + " if images_per_gpu > 1:\n", + " logger = logging.getLogger(__name__)\n", + " logger.warning(\n", + " \"When using more than one image per GPU you may encounter \"\n", + " \"an out-of-memory (OOM) error if your GPU does not have \"\n", + " \"sufficient memory. If this happens, you can reduce \"\n", + " \"SOLVER.IMS_PER_BATCH (for training) or \"\n", + " \"TEST.IMS_PER_BATCH (for inference). For training, you must \"\n", + " \"also adjust the learning rate and schedule length according \"\n", + " \"to the linear scaling rule. See for example: \"\n", + " \"https://github.com/facebookresearch/Detectron/blob/master/configs/getting_started/tutorial_1gpu_e2e_faster_rcnn_R-50-FPN.yaml#L14\"\n", + " )\n", + "\n", + " # group images which have similar aspect ratio. In this case, we only\n", + " # group in two cases: those with width / height > 1, and the other way around,\n", + " # but the code supports more general grouping strategy\n", + " aspect_grouping = [1] if cfg.DATALOADER.ASPECT_RATIO_GROUPING else []\n", + "\n", + " paths_catalog = import_file(\n", + " \"maskrcnn_benchmark.config.paths_catalog\", cfg.PATHS_CATALOG, True\n", + " )\n", + " DatasetCatalog = paths_catalog.DatasetCatalog\n", + " dataset_list = cfg.DATASETS.TRAIN if is_train else cfg.DATASETS.TEST\n", + "\n", + " # If bbox aug is enabled in testing, simply set transforms to None and we will apply transforms later\n", + " transforms = None if not is_train and cfg.TEST.BBOX_AUG.ENABLED else build_transforms(cfg, is_train)\n", + " \n", + " dataset.transforms = transforms\n", + " datasets = [ dataset ]\n", + " \n", + " data_loaders = []\n", + " for dataset in datasets:\n", + " sampler = make_data_sampler(dataset, shuffle, is_distributed)\n", + " batch_sampler = make_batch_data_sampler(\n", + " dataset, sampler, aspect_grouping, images_per_gpu, num_iters, start_iter\n", + " )\n", + " collator = BBoxAugCollator() if not is_train and cfg.TEST.BBOX_AUG.ENABLED else \\\n", + " BatchCollator(cfg.DATALOADER.SIZE_DIVISIBILITY)\n", + " num_workers = cfg.DATALOADER.NUM_WORKERS\n", + " data_loader = torch.utils.data.DataLoader(\n", + " dataset,\n", + " num_workers=num_workers,\n", + " batch_sampler=batch_sampler,\n", + " collate_fn=collator,\n", + " )\n", + " data_loaders.append(data_loader)\n", + " if is_train:\n", + " # during training, a single (possibly concatenated) data_loader is returned\n", + " assert len(data_loaders) == 1\n", + " return data_loaders[0]\n", + " return data_loaders" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BTKsrHa-TkGr", + "colab_type": "text" + }, + "source": [ + "### Train Function\n", + "\n", + "The train function is the entry point into the training process. It creates data loaders, optimisers, loads from checkpoint. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LYTguCvrTnHW", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# See if we can use apex.DistributedDataParallel instead of the torch default,\n", + "# and enable mixed-precision via apex.amp\n", + "try:\n", + " from apex import amp\n", + "except ImportError:\n", + " raise ImportError('Use APEX for multi-precision via apex.amp')\n", + "\n", + "def train(cfg, local_rank, distributed, dataset):\n", + " model = build_detection_model(cfg)\n", + " device = torch.device('cuda')\n", + " model.to(device)\n", + "\n", + " optimizer = make_optimizer(cfg, model)\n", + " scheduler = make_lr_scheduler(cfg, optimizer)\n", + "\n", + " # Initialize mixed-precision training\n", + " use_mixed_precision = cfg.DTYPE == \"float16\"\n", + " amp_opt_level = 'O1' if use_mixed_precision else 'O0'\n", + " model, optimizer = amp.initialize(model, optimizer, opt_level=amp_opt_level)\n", + "\n", + " if distributed:\n", + " model = torch.nn.parallel.DistributedDataParallel(\n", + " model, device_ids=[local_rank], output_device=local_rank,\n", + " # this should be removed if we update BatchNorm stats\n", + " broadcast_buffers=False,\n", + " )\n", + "\n", + " arguments = {}\n", + " arguments[\"iteration\"] = 0\n", + "\n", + " output_dir = cfg.OUTPUT_DIR\n", + " save_to_disk = get_rank() == 0\n", + " checkpointer = DetectronCheckpointer(\n", + " cfg, model, optimizer, scheduler, output_dir, save_to_disk\n", + " )\n", + " extra_checkpoint_data = checkpointer.load(cfg.MODEL.WEIGHT)\n", + " arguments.update(extra_checkpoint_data)\n", + "\n", + "\n", + " data_loader = build_data_loader(cfg, dataset)\n", + "\n", + " checkpoint_period = cfg.SOLVER.CHECKPOINT_PERIOD\n", + "\n", + " do_train(\n", + " model,\n", + " data_loader,\n", + " optimizer,\n", + " scheduler,\n", + " checkpointer,\n", + " device,\n", + " checkpoint_period,\n", + " arguments,\n", + " )\n", + "\n", + " return model" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r-SfVh-qCmhe", + "colab_type": "text" + }, + "source": [ + "## Set training config and train\n", + "\n", + "here we fire off training by calling the above function. before that we set some important config for our training. We make our dataset and update our config. Then we fire off training !" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ad0VIyZqVDXy", + "colab_type": "code", + "outputId": "3b5a93ff-9203-4530-fc86-de557fe4428c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 6735 + } + }, + "source": [ + "config_file = \"shapes_config.yaml\"\n", + "\n", + "# update the config options with the config file\n", + "cfg.merge_from_file(config_file)\n", + "\n", + "cfg.merge_from_list(['OUTPUT_DIR', 'shapeDir']) # The output folder where all our model checkpoints will be saved during training.\n", + "cfg.merge_from_list(['SOLVER.IMS_PER_BATCH', 4]) # Number of images to take insiade a single batch. This number depends on the size of your GPU\n", + "cfg.merge_from_list(['SOLVER.BASE_LR', 0.0050]) # The Learning Rate when training starts. Please check Detectron scaling rules to determine your learning for your GPU setup. \n", + "cfg.merge_from_list(['SOLVER.MAX_ITER', 1000]) # The number of training iterations that will be executed during training. One iteration is given as one forward and backward pass of a mini batch of the network\n", + "cfg.merge_from_list(['SOLVER.STEPS', \"(700, 800)\"]) # These two numberes represent after how many iterations is the learning rate divided by 10. \n", + "cfg.merge_from_list(['TEST.IMS_PER_BATCH', 1]) # Batch size during testing/evaluation\n", + "cfg.merge_from_list(['MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN', 4000]) # This determines how many region proposals to take in for processing into the stage after the RPN. The rule is 1000*batch_size = 4*1000 \n", + "cfg.merge_from_list(['SOLVER.CHECKPOINT_PERIOD', 100]) # After how many iterations does one want to save the model.\n", + "\n", + "# Make the Output dir if one doesnt exist.\n", + "output_dir = cfg.OUTPUT_DIR\n", + "if output_dir:\n", + " mkdir(output_dir)\n", + "\n", + "# Start training.\n", + "model = train(cfg, local_rank=1, distributed=False, dataset=ShapeDataset(500))" + ], + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "text": [ + "loading annotations into memory...\n", + "Done (t=0.00s)\n", + "creating index...\n", + "index created!\n", + "Selected optimization level O0: Pure FP32 training.\n", + "\n", + "Defaults for this optimization level are:\n", + "enabled : True\n", + "opt_level : O0\n", + "cast_model_type : torch.float32\n", + "patch_torch_functions : False\n", + "keep_batchnorm_fp32 : None\n", + "master_weights : False\n", + "loss_scale : 1.0\n", + "Processing user overrides (additional kwargs that are not None)...\n", + "After processing overrides, optimization options are:\n", + "enabled : True\n", + "opt_level : O0\n", + "cast_model_type : torch.float32\n", + "patch_torch_functions : False\n", + "keep_batchnorm_fp32 : None\n", + "master_weights : False\n", + "loss_scale : 1.0\n", + "2019-06-02 18:25:29,979 maskrcnn_benchmark.utils.checkpoint INFO: Loading checkpoint from base_model.pth\n", + "2019-06-02 18:25:30,069 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn1.bias loaded from backbone.body.layer1.0.bn1.bias of shape (64,)\n", + "2019-06-02 18:25:30,070 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn1.running_mean loaded from backbone.body.layer1.0.bn1.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,074 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn1.running_var loaded from backbone.body.layer1.0.bn1.running_var of shape (64,)\n", + "2019-06-02 18:25:30,076 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn1.weight loaded from backbone.body.layer1.0.bn1.weight of shape (64,)\n", + "2019-06-02 18:25:30,077 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn2.bias loaded from backbone.body.layer1.0.bn2.bias of shape (64,)\n", + "2019-06-02 18:25:30,079 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn2.running_mean loaded from backbone.body.layer1.0.bn2.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,080 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn2.running_var loaded from backbone.body.layer1.0.bn2.running_var of shape (64,)\n", + "2019-06-02 18:25:30,082 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn2.weight loaded from backbone.body.layer1.0.bn2.weight of shape (64,)\n", + "2019-06-02 18:25:30,084 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn3.bias loaded from backbone.body.layer1.0.bn3.bias of shape (256,)\n", + "2019-06-02 18:25:30,085 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn3.running_mean loaded from backbone.body.layer1.0.bn3.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,087 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn3.running_var loaded from backbone.body.layer1.0.bn3.running_var of shape (256,)\n", + "2019-06-02 18:25:30,088 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.bn3.weight loaded from backbone.body.layer1.0.bn3.weight of shape (256,)\n", + "2019-06-02 18:25:30,089 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.conv1.weight loaded from backbone.body.layer1.0.conv1.weight of shape (64, 64, 1, 1)\n", + "2019-06-02 18:25:30,091 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.conv2.weight loaded from backbone.body.layer1.0.conv2.weight of shape (64, 64, 3, 3)\n", + "2019-06-02 18:25:30,092 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.conv3.weight loaded from backbone.body.layer1.0.conv3.weight of shape (256, 64, 1, 1)\n", + "2019-06-02 18:25:30,094 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.downsample.0.weight loaded from backbone.body.layer1.0.downsample.0.weight of shape (256, 64, 1, 1)\n", + "2019-06-02 18:25:30,095 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.downsample.1.bias loaded from backbone.body.layer1.0.downsample.1.bias of shape (256,)\n", + "2019-06-02 18:25:30,097 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.downsample.1.running_mean loaded from backbone.body.layer1.0.downsample.1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,098 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.downsample.1.running_var loaded from backbone.body.layer1.0.downsample.1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,100 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.0.downsample.1.weight loaded from backbone.body.layer1.0.downsample.1.weight of shape (256,)\n", + "2019-06-02 18:25:30,101 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn1.bias loaded from backbone.body.layer1.1.bn1.bias of shape (64,)\n", + "2019-06-02 18:25:30,103 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn1.running_mean loaded from backbone.body.layer1.1.bn1.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,105 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn1.running_var loaded from backbone.body.layer1.1.bn1.running_var of shape (64,)\n", + "2019-06-02 18:25:30,107 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn1.weight loaded from backbone.body.layer1.1.bn1.weight of shape (64,)\n", + "2019-06-02 18:25:30,108 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn2.bias loaded from backbone.body.layer1.1.bn2.bias of shape (64,)\n", + "2019-06-02 18:25:30,110 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn2.running_mean loaded from backbone.body.layer1.1.bn2.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,111 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn2.running_var loaded from backbone.body.layer1.1.bn2.running_var of shape (64,)\n", + "2019-06-02 18:25:30,112 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn2.weight loaded from backbone.body.layer1.1.bn2.weight of shape (64,)\n", + "2019-06-02 18:25:30,113 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn3.bias loaded from backbone.body.layer1.1.bn3.bias of shape (256,)\n", + "2019-06-02 18:25:30,115 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn3.running_mean loaded from backbone.body.layer1.1.bn3.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,116 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn3.running_var loaded from backbone.body.layer1.1.bn3.running_var of shape (256,)\n", + "2019-06-02 18:25:30,117 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.bn3.weight loaded from backbone.body.layer1.1.bn3.weight of shape (256,)\n", + "2019-06-02 18:25:30,118 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.conv1.weight loaded from backbone.body.layer1.1.conv1.weight of shape (64, 256, 1, 1)\n", + "2019-06-02 18:25:30,119 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.conv2.weight loaded from backbone.body.layer1.1.conv2.weight of shape (64, 64, 3, 3)\n", + "2019-06-02 18:25:30,121 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.1.conv3.weight loaded from backbone.body.layer1.1.conv3.weight of shape (256, 64, 1, 1)\n", + "2019-06-02 18:25:30,122 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn1.bias loaded from backbone.body.layer1.2.bn1.bias of shape (64,)\n", + "2019-06-02 18:25:30,123 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn1.running_mean loaded from backbone.body.layer1.2.bn1.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,124 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn1.running_var loaded from backbone.body.layer1.2.bn1.running_var of shape (64,)\n", + "2019-06-02 18:25:30,126 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn1.weight loaded from backbone.body.layer1.2.bn1.weight of shape (64,)\n", + "2019-06-02 18:25:30,127 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn2.bias loaded from backbone.body.layer1.2.bn2.bias of shape (64,)\n", + "2019-06-02 18:25:30,128 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn2.running_mean loaded from backbone.body.layer1.2.bn2.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,129 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn2.running_var loaded from backbone.body.layer1.2.bn2.running_var of shape (64,)\n", + "2019-06-02 18:25:30,130 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn2.weight loaded from backbone.body.layer1.2.bn2.weight of shape (64,)\n", + "2019-06-02 18:25:30,131 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn3.bias loaded from backbone.body.layer1.2.bn3.bias of shape (256,)\n", + "2019-06-02 18:25:30,133 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn3.running_mean loaded from backbone.body.layer1.2.bn3.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,134 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn3.running_var loaded from backbone.body.layer1.2.bn3.running_var of shape (256,)\n", + "2019-06-02 18:25:30,135 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.bn3.weight loaded from backbone.body.layer1.2.bn3.weight of shape (256,)\n", + "2019-06-02 18:25:30,136 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.conv1.weight loaded from backbone.body.layer1.2.conv1.weight of shape (64, 256, 1, 1)\n", + "2019-06-02 18:25:30,137 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.conv2.weight loaded from backbone.body.layer1.2.conv2.weight of shape (64, 64, 3, 3)\n", + "2019-06-02 18:25:30,138 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer1.2.conv3.weight loaded from backbone.body.layer1.2.conv3.weight of shape (256, 64, 1, 1)\n", + "2019-06-02 18:25:30,140 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn1.bias loaded from backbone.body.layer2.0.bn1.bias of shape (128,)\n", + "2019-06-02 18:25:30,141 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn1.running_mean loaded from backbone.body.layer2.0.bn1.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,142 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn1.running_var loaded from backbone.body.layer2.0.bn1.running_var of shape (128,)\n", + "2019-06-02 18:25:30,143 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn1.weight loaded from backbone.body.layer2.0.bn1.weight of shape (128,)\n", + "2019-06-02 18:25:30,144 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn2.bias loaded from backbone.body.layer2.0.bn2.bias of shape (128,)\n", + "2019-06-02 18:25:30,145 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn2.running_mean loaded from backbone.body.layer2.0.bn2.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,146 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn2.running_var loaded from backbone.body.layer2.0.bn2.running_var of shape (128,)\n", + "2019-06-02 18:25:30,147 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn2.weight loaded from backbone.body.layer2.0.bn2.weight of shape (128,)\n", + "2019-06-02 18:25:30,148 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn3.bias loaded from backbone.body.layer2.0.bn3.bias of shape (512,)\n", + "2019-06-02 18:25:30,149 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn3.running_mean loaded from backbone.body.layer2.0.bn3.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,150 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn3.running_var loaded from backbone.body.layer2.0.bn3.running_var of shape (512,)\n", + "2019-06-02 18:25:30,151 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.bn3.weight loaded from backbone.body.layer2.0.bn3.weight of shape (512,)\n", + "2019-06-02 18:25:30,152 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.conv1.weight loaded from backbone.body.layer2.0.conv1.weight of shape (128, 256, 1, 1)\n", + "2019-06-02 18:25:30,154 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.conv2.weight loaded from backbone.body.layer2.0.conv2.weight of shape (128, 128, 3, 3)\n", + "2019-06-02 18:25:30,155 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.conv3.weight loaded from backbone.body.layer2.0.conv3.weight of shape (512, 128, 1, 1)\n", + "2019-06-02 18:25:30,157 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.downsample.0.weight loaded from backbone.body.layer2.0.downsample.0.weight of shape (512, 256, 1, 1)\n", + "2019-06-02 18:25:30,158 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.downsample.1.bias loaded from backbone.body.layer2.0.downsample.1.bias of shape (512,)\n", + "2019-06-02 18:25:30,159 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.downsample.1.running_mean loaded from backbone.body.layer2.0.downsample.1.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,160 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.downsample.1.running_var loaded from backbone.body.layer2.0.downsample.1.running_var of shape (512,)\n", + "2019-06-02 18:25:30,161 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.0.downsample.1.weight loaded from backbone.body.layer2.0.downsample.1.weight of shape (512,)\n", + "2019-06-02 18:25:30,162 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn1.bias loaded from backbone.body.layer2.1.bn1.bias of shape (128,)\n", + "2019-06-02 18:25:30,163 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn1.running_mean loaded from backbone.body.layer2.1.bn1.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,164 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn1.running_var loaded from backbone.body.layer2.1.bn1.running_var of shape (128,)\n", + "2019-06-02 18:25:30,166 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn1.weight loaded from backbone.body.layer2.1.bn1.weight of shape (128,)\n", + "2019-06-02 18:25:30,167 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn2.bias loaded from backbone.body.layer2.1.bn2.bias of shape (128,)\n", + "2019-06-02 18:25:30,167 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn2.running_mean loaded from backbone.body.layer2.1.bn2.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,168 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn2.running_var loaded from backbone.body.layer2.1.bn2.running_var of shape (128,)\n", + "2019-06-02 18:25:30,169 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn2.weight loaded from backbone.body.layer2.1.bn2.weight of shape (128,)\n", + "2019-06-02 18:25:30,170 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn3.bias loaded from backbone.body.layer2.1.bn3.bias of shape (512,)\n", + "2019-06-02 18:25:30,171 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn3.running_mean loaded from backbone.body.layer2.1.bn3.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,172 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn3.running_var loaded from backbone.body.layer2.1.bn3.running_var of shape (512,)\n", + "2019-06-02 18:25:30,173 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.bn3.weight loaded from backbone.body.layer2.1.bn3.weight of shape (512,)\n", + "2019-06-02 18:25:30,174 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.conv1.weight loaded from backbone.body.layer2.1.conv1.weight of shape (128, 512, 1, 1)\n", + "2019-06-02 18:25:30,174 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.conv2.weight loaded from backbone.body.layer2.1.conv2.weight of shape (128, 128, 3, 3)\n", + "2019-06-02 18:25:30,175 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.1.conv3.weight loaded from backbone.body.layer2.1.conv3.weight of shape (512, 128, 1, 1)\n", + "2019-06-02 18:25:30,176 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn1.bias loaded from backbone.body.layer2.2.bn1.bias of shape (128,)\n", + "2019-06-02 18:25:30,177 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn1.running_mean loaded from backbone.body.layer2.2.bn1.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,178 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn1.running_var loaded from backbone.body.layer2.2.bn1.running_var of shape (128,)\n", + "2019-06-02 18:25:30,178 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn1.weight loaded from backbone.body.layer2.2.bn1.weight of shape (128,)\n", + "2019-06-02 18:25:30,179 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn2.bias loaded from backbone.body.layer2.2.bn2.bias of shape (128,)\n", + "2019-06-02 18:25:30,180 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn2.running_mean loaded from backbone.body.layer2.2.bn2.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,181 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn2.running_var loaded from backbone.body.layer2.2.bn2.running_var of shape (128,)\n", + "2019-06-02 18:25:30,182 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn2.weight loaded from backbone.body.layer2.2.bn2.weight of shape (128,)\n", + "2019-06-02 18:25:30,183 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn3.bias loaded from backbone.body.layer2.2.bn3.bias of shape (512,)\n", + "2019-06-02 18:25:30,184 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn3.running_mean loaded from backbone.body.layer2.2.bn3.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,184 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn3.running_var loaded from backbone.body.layer2.2.bn3.running_var of shape (512,)\n", + "2019-06-02 18:25:30,186 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.bn3.weight loaded from backbone.body.layer2.2.bn3.weight of shape (512,)\n", + "2019-06-02 18:25:30,186 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.conv1.weight loaded from backbone.body.layer2.2.conv1.weight of shape (128, 512, 1, 1)\n", + "2019-06-02 18:25:30,187 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.conv2.weight loaded from backbone.body.layer2.2.conv2.weight of shape (128, 128, 3, 3)\n", + "2019-06-02 18:25:30,188 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.2.conv3.weight loaded from backbone.body.layer2.2.conv3.weight of shape (512, 128, 1, 1)\n", + "2019-06-02 18:25:30,189 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn1.bias loaded from backbone.body.layer2.3.bn1.bias of shape (128,)\n", + "2019-06-02 18:25:30,190 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn1.running_mean loaded from backbone.body.layer2.3.bn1.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,190 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn1.running_var loaded from backbone.body.layer2.3.bn1.running_var of shape (128,)\n", + "2019-06-02 18:25:30,191 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn1.weight loaded from backbone.body.layer2.3.bn1.weight of shape (128,)\n", + "2019-06-02 18:25:30,192 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn2.bias loaded from backbone.body.layer2.3.bn2.bias of shape (128,)\n", + "2019-06-02 18:25:30,193 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn2.running_mean loaded from backbone.body.layer2.3.bn2.running_mean of shape (128,)\n", + "2019-06-02 18:25:30,194 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn2.running_var loaded from backbone.body.layer2.3.bn2.running_var of shape (128,)\n", + "2019-06-02 18:25:30,195 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn2.weight loaded from backbone.body.layer2.3.bn2.weight of shape (128,)\n", + "2019-06-02 18:25:30,196 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn3.bias loaded from backbone.body.layer2.3.bn3.bias of shape (512,)\n", + "2019-06-02 18:25:30,197 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn3.running_mean loaded from backbone.body.layer2.3.bn3.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,200 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn3.running_var loaded from backbone.body.layer2.3.bn3.running_var of shape (512,)\n", + "2019-06-02 18:25:30,201 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.bn3.weight loaded from backbone.body.layer2.3.bn3.weight of shape (512,)\n", + "2019-06-02 18:25:30,202 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.conv1.weight loaded from backbone.body.layer2.3.conv1.weight of shape (128, 512, 1, 1)\n", + "2019-06-02 18:25:30,203 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.conv2.weight loaded from backbone.body.layer2.3.conv2.weight of shape (128, 128, 3, 3)\n", + "2019-06-02 18:25:30,204 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer2.3.conv3.weight loaded from backbone.body.layer2.3.conv3.weight of shape (512, 128, 1, 1)\n", + "2019-06-02 18:25:30,205 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn1.bias loaded from backbone.body.layer3.0.bn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,206 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn1.running_mean loaded from backbone.body.layer3.0.bn1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,207 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn1.running_var loaded from backbone.body.layer3.0.bn1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,208 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn1.weight loaded from backbone.body.layer3.0.bn1.weight of shape (256,)\n", + "2019-06-02 18:25:30,211 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn2.bias loaded from backbone.body.layer3.0.bn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,212 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn2.running_mean loaded from backbone.body.layer3.0.bn2.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,214 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn2.running_var loaded from backbone.body.layer3.0.bn2.running_var of shape (256,)\n", + "2019-06-02 18:25:30,216 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn2.weight loaded from backbone.body.layer3.0.bn2.weight of shape (256,)\n", + "2019-06-02 18:25:30,217 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn3.bias loaded from backbone.body.layer3.0.bn3.bias of shape (1024,)\n", + "2019-06-02 18:25:30,218 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn3.running_mean loaded from backbone.body.layer3.0.bn3.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,220 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn3.running_var loaded from backbone.body.layer3.0.bn3.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,221 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.bn3.weight loaded from backbone.body.layer3.0.bn3.weight of shape (1024,)\n", + "2019-06-02 18:25:30,222 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.conv1.weight loaded from backbone.body.layer3.0.conv1.weight of shape (256, 512, 1, 1)\n", + "2019-06-02 18:25:30,224 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.conv2.weight loaded from backbone.body.layer3.0.conv2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,226 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.conv3.weight loaded from backbone.body.layer3.0.conv3.weight of shape (1024, 256, 1, 1)\n", + "2019-06-02 18:25:30,227 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.downsample.0.weight loaded from backbone.body.layer3.0.downsample.0.weight of shape (1024, 512, 1, 1)\n", + "2019-06-02 18:25:30,228 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.downsample.1.bias loaded from backbone.body.layer3.0.downsample.1.bias of shape (1024,)\n", + "2019-06-02 18:25:30,230 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.downsample.1.running_mean loaded from backbone.body.layer3.0.downsample.1.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,231 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.downsample.1.running_var loaded from backbone.body.layer3.0.downsample.1.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,232 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.0.downsample.1.weight loaded from backbone.body.layer3.0.downsample.1.weight of shape (1024,)\n", + "2019-06-02 18:25:30,234 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn1.bias loaded from backbone.body.layer3.1.bn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,236 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn1.running_mean loaded from backbone.body.layer3.1.bn1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,237 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn1.running_var loaded from backbone.body.layer3.1.bn1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,238 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn1.weight loaded from backbone.body.layer3.1.bn1.weight of shape (256,)\n", + "2019-06-02 18:25:30,240 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn2.bias loaded from backbone.body.layer3.1.bn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,241 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn2.running_mean loaded from backbone.body.layer3.1.bn2.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,243 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn2.running_var loaded from backbone.body.layer3.1.bn2.running_var of shape (256,)\n", + "2019-06-02 18:25:30,245 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn2.weight loaded from backbone.body.layer3.1.bn2.weight of shape (256,)\n", + "2019-06-02 18:25:30,246 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn3.bias loaded from backbone.body.layer3.1.bn3.bias of shape (1024,)\n", + "2019-06-02 18:25:30,248 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn3.running_mean loaded from backbone.body.layer3.1.bn3.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,250 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn3.running_var loaded from backbone.body.layer3.1.bn3.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,252 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.bn3.weight loaded from backbone.body.layer3.1.bn3.weight of shape (1024,)\n", + "2019-06-02 18:25:30,253 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.conv1.weight loaded from backbone.body.layer3.1.conv1.weight of shape (256, 1024, 1, 1)\n", + "2019-06-02 18:25:30,255 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.conv2.weight loaded from backbone.body.layer3.1.conv2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,256 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.1.conv3.weight loaded from backbone.body.layer3.1.conv3.weight of shape (1024, 256, 1, 1)\n", + "2019-06-02 18:25:30,257 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn1.bias loaded from backbone.body.layer3.2.bn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,260 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn1.running_mean loaded from backbone.body.layer3.2.bn1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,265 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn1.running_var loaded from backbone.body.layer3.2.bn1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,266 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn1.weight loaded from backbone.body.layer3.2.bn1.weight of shape (256,)\n", + "2019-06-02 18:25:30,267 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn2.bias loaded from backbone.body.layer3.2.bn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,268 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn2.running_mean loaded from backbone.body.layer3.2.bn2.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,270 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn2.running_var loaded from backbone.body.layer3.2.bn2.running_var of shape (256,)\n", + "2019-06-02 18:25:30,271 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn2.weight loaded from backbone.body.layer3.2.bn2.weight of shape (256,)\n", + "2019-06-02 18:25:30,272 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn3.bias loaded from backbone.body.layer3.2.bn3.bias of shape (1024,)\n", + "2019-06-02 18:25:30,275 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn3.running_mean loaded from backbone.body.layer3.2.bn3.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,276 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn3.running_var loaded from backbone.body.layer3.2.bn3.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,277 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.bn3.weight loaded from backbone.body.layer3.2.bn3.weight of shape (1024,)\n", + "2019-06-02 18:25:30,278 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.conv1.weight loaded from backbone.body.layer3.2.conv1.weight of shape (256, 1024, 1, 1)\n", + "2019-06-02 18:25:30,281 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.conv2.weight loaded from backbone.body.layer3.2.conv2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,282 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.2.conv3.weight loaded from backbone.body.layer3.2.conv3.weight of shape (1024, 256, 1, 1)\n", + "2019-06-02 18:25:30,284 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn1.bias loaded from backbone.body.layer3.3.bn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,285 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn1.running_mean loaded from backbone.body.layer3.3.bn1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,286 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn1.running_var loaded from backbone.body.layer3.3.bn1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,288 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn1.weight loaded from backbone.body.layer3.3.bn1.weight of shape (256,)\n", + "2019-06-02 18:25:30,290 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn2.bias loaded from backbone.body.layer3.3.bn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,291 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn2.running_mean loaded from backbone.body.layer3.3.bn2.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,293 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn2.running_var loaded from backbone.body.layer3.3.bn2.running_var of shape (256,)\n", + "2019-06-02 18:25:30,295 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn2.weight loaded from backbone.body.layer3.3.bn2.weight of shape (256,)\n", + "2019-06-02 18:25:30,296 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn3.bias loaded from backbone.body.layer3.3.bn3.bias of shape (1024,)\n", + "2019-06-02 18:25:30,297 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn3.running_mean loaded from backbone.body.layer3.3.bn3.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,299 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn3.running_var loaded from backbone.body.layer3.3.bn3.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,300 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.bn3.weight loaded from backbone.body.layer3.3.bn3.weight of shape (1024,)\n", + "2019-06-02 18:25:30,301 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.conv1.weight loaded from backbone.body.layer3.3.conv1.weight of shape (256, 1024, 1, 1)\n", + "2019-06-02 18:25:30,303 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.conv2.weight loaded from backbone.body.layer3.3.conv2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,304 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.3.conv3.weight loaded from backbone.body.layer3.3.conv3.weight of shape (1024, 256, 1, 1)\n", + "2019-06-02 18:25:30,305 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn1.bias loaded from backbone.body.layer3.4.bn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,306 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn1.running_mean loaded from backbone.body.layer3.4.bn1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,307 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn1.running_var loaded from backbone.body.layer3.4.bn1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,310 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn1.weight loaded from backbone.body.layer3.4.bn1.weight of shape (256,)\n", + "2019-06-02 18:25:30,311 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn2.bias loaded from backbone.body.layer3.4.bn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,313 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn2.running_mean loaded from backbone.body.layer3.4.bn2.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,314 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn2.running_var loaded from backbone.body.layer3.4.bn2.running_var of shape (256,)\n", + "2019-06-02 18:25:30,316 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn2.weight loaded from backbone.body.layer3.4.bn2.weight of shape (256,)\n", + "2019-06-02 18:25:30,317 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn3.bias loaded from backbone.body.layer3.4.bn3.bias of shape (1024,)\n", + "2019-06-02 18:25:30,319 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn3.running_mean loaded from backbone.body.layer3.4.bn3.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,320 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn3.running_var loaded from backbone.body.layer3.4.bn3.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,322 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.bn3.weight loaded from backbone.body.layer3.4.bn3.weight of shape (1024,)\n", + "2019-06-02 18:25:30,324 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.conv1.weight loaded from backbone.body.layer3.4.conv1.weight of shape (256, 1024, 1, 1)\n", + "2019-06-02 18:25:30,325 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.conv2.weight loaded from backbone.body.layer3.4.conv2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,326 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.4.conv3.weight loaded from backbone.body.layer3.4.conv3.weight of shape (1024, 256, 1, 1)\n", + "2019-06-02 18:25:30,327 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn1.bias loaded from backbone.body.layer3.5.bn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,329 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn1.running_mean loaded from backbone.body.layer3.5.bn1.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,331 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn1.running_var loaded from backbone.body.layer3.5.bn1.running_var of shape (256,)\n", + "2019-06-02 18:25:30,332 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn1.weight loaded from backbone.body.layer3.5.bn1.weight of shape (256,)\n", + "2019-06-02 18:25:30,334 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn2.bias loaded from backbone.body.layer3.5.bn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,335 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn2.running_mean loaded from backbone.body.layer3.5.bn2.running_mean of shape (256,)\n", + "2019-06-02 18:25:30,336 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn2.running_var loaded from backbone.body.layer3.5.bn2.running_var of shape (256,)\n", + "2019-06-02 18:25:30,338 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn2.weight loaded from backbone.body.layer3.5.bn2.weight of shape (256,)\n", + "2019-06-02 18:25:30,339 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn3.bias loaded from backbone.body.layer3.5.bn3.bias of shape (1024,)\n", + "2019-06-02 18:25:30,341 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn3.running_mean loaded from backbone.body.layer3.5.bn3.running_mean of shape (1024,)\n", + "2019-06-02 18:25:30,342 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn3.running_var loaded from backbone.body.layer3.5.bn3.running_var of shape (1024,)\n", + "2019-06-02 18:25:30,343 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.bn3.weight loaded from backbone.body.layer3.5.bn3.weight of shape (1024,)\n", + "2019-06-02 18:25:30,344 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.conv1.weight loaded from backbone.body.layer3.5.conv1.weight of shape (256, 1024, 1, 1)\n", + "2019-06-02 18:25:30,346 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.conv2.weight loaded from backbone.body.layer3.5.conv2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,348 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer3.5.conv3.weight loaded from backbone.body.layer3.5.conv3.weight of shape (1024, 256, 1, 1)\n", + "2019-06-02 18:25:30,350 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn1.bias loaded from backbone.body.layer4.0.bn1.bias of shape (512,)\n", + "2019-06-02 18:25:30,351 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn1.running_mean loaded from backbone.body.layer4.0.bn1.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,352 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn1.running_var loaded from backbone.body.layer4.0.bn1.running_var of shape (512,)\n", + "2019-06-02 18:25:30,354 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn1.weight loaded from backbone.body.layer4.0.bn1.weight of shape (512,)\n", + "2019-06-02 18:25:30,354 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn2.bias loaded from backbone.body.layer4.0.bn2.bias of shape (512,)\n", + "2019-06-02 18:25:30,356 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn2.running_mean loaded from backbone.body.layer4.0.bn2.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,359 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn2.running_var loaded from backbone.body.layer4.0.bn2.running_var of shape (512,)\n", + "2019-06-02 18:25:30,360 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn2.weight loaded from backbone.body.layer4.0.bn2.weight of shape (512,)\n", + "2019-06-02 18:25:30,362 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn3.bias loaded from backbone.body.layer4.0.bn3.bias of shape (2048,)\n", + "2019-06-02 18:25:30,363 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn3.running_mean loaded from backbone.body.layer4.0.bn3.running_mean of shape (2048,)\n", + "2019-06-02 18:25:30,364 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn3.running_var loaded from backbone.body.layer4.0.bn3.running_var of shape (2048,)\n", + "2019-06-02 18:25:30,366 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.bn3.weight loaded from backbone.body.layer4.0.bn3.weight of shape (2048,)\n", + "2019-06-02 18:25:30,368 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.conv1.weight loaded from backbone.body.layer4.0.conv1.weight of shape (512, 1024, 1, 1)\n", + "2019-06-02 18:25:30,369 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.conv2.weight loaded from backbone.body.layer4.0.conv2.weight of shape (512, 512, 3, 3)\n", + "2019-06-02 18:25:30,371 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.conv3.weight loaded from backbone.body.layer4.0.conv3.weight of shape (2048, 512, 1, 1)\n", + "2019-06-02 18:25:30,372 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.downsample.0.weight loaded from backbone.body.layer4.0.downsample.0.weight of shape (2048, 1024, 1, 1)\n", + "2019-06-02 18:25:30,373 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.downsample.1.bias loaded from backbone.body.layer4.0.downsample.1.bias of shape (2048,)\n", + "2019-06-02 18:25:30,374 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.downsample.1.running_mean loaded from backbone.body.layer4.0.downsample.1.running_mean of shape (2048,)\n", + "2019-06-02 18:25:30,376 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.downsample.1.running_var loaded from backbone.body.layer4.0.downsample.1.running_var of shape (2048,)\n", + "2019-06-02 18:25:30,378 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.0.downsample.1.weight loaded from backbone.body.layer4.0.downsample.1.weight of shape (2048,)\n", + "2019-06-02 18:25:30,380 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn1.bias loaded from backbone.body.layer4.1.bn1.bias of shape (512,)\n", + "2019-06-02 18:25:30,381 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn1.running_mean loaded from backbone.body.layer4.1.bn1.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,383 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn1.running_var loaded from backbone.body.layer4.1.bn1.running_var of shape (512,)\n", + "2019-06-02 18:25:30,384 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn1.weight loaded from backbone.body.layer4.1.bn1.weight of shape (512,)\n", + "2019-06-02 18:25:30,385 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn2.bias loaded from backbone.body.layer4.1.bn2.bias of shape (512,)\n", + "2019-06-02 18:25:30,387 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn2.running_mean loaded from backbone.body.layer4.1.bn2.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,388 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn2.running_var loaded from backbone.body.layer4.1.bn2.running_var of shape (512,)\n", + "2019-06-02 18:25:30,389 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn2.weight loaded from backbone.body.layer4.1.bn2.weight of shape (512,)\n", + "2019-06-02 18:25:30,390 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn3.bias loaded from backbone.body.layer4.1.bn3.bias of shape (2048,)\n", + "2019-06-02 18:25:30,392 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn3.running_mean loaded from backbone.body.layer4.1.bn3.running_mean of shape (2048,)\n", + "2019-06-02 18:25:30,393 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn3.running_var loaded from backbone.body.layer4.1.bn3.running_var of shape (2048,)\n", + "2019-06-02 18:25:30,394 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.bn3.weight loaded from backbone.body.layer4.1.bn3.weight of shape (2048,)\n", + "2019-06-02 18:25:30,397 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.conv1.weight loaded from backbone.body.layer4.1.conv1.weight of shape (512, 2048, 1, 1)\n", + "2019-06-02 18:25:30,398 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.conv2.weight loaded from backbone.body.layer4.1.conv2.weight of shape (512, 512, 3, 3)\n", + "2019-06-02 18:25:30,399 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.1.conv3.weight loaded from backbone.body.layer4.1.conv3.weight of shape (2048, 512, 1, 1)\n", + "2019-06-02 18:25:30,401 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn1.bias loaded from backbone.body.layer4.2.bn1.bias of shape (512,)\n", + "2019-06-02 18:25:30,402 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn1.running_mean loaded from backbone.body.layer4.2.bn1.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,403 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn1.running_var loaded from backbone.body.layer4.2.bn1.running_var of shape (512,)\n", + "2019-06-02 18:25:30,405 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn1.weight loaded from backbone.body.layer4.2.bn1.weight of shape (512,)\n", + "2019-06-02 18:25:30,406 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn2.bias loaded from backbone.body.layer4.2.bn2.bias of shape (512,)\n", + "2019-06-02 18:25:30,408 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn2.running_mean loaded from backbone.body.layer4.2.bn2.running_mean of shape (512,)\n", + "2019-06-02 18:25:30,410 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn2.running_var loaded from backbone.body.layer4.2.bn2.running_var of shape (512,)\n", + "2019-06-02 18:25:30,411 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn2.weight loaded from backbone.body.layer4.2.bn2.weight of shape (512,)\n", + "2019-06-02 18:25:30,413 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn3.bias loaded from backbone.body.layer4.2.bn3.bias of shape (2048,)\n", + "2019-06-02 18:25:30,414 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn3.running_mean loaded from backbone.body.layer4.2.bn3.running_mean of shape (2048,)\n", + "2019-06-02 18:25:30,415 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn3.running_var loaded from backbone.body.layer4.2.bn3.running_var of shape (2048,)\n", + "2019-06-02 18:25:30,417 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.bn3.weight loaded from backbone.body.layer4.2.bn3.weight of shape (2048,)\n", + "2019-06-02 18:25:30,421 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.conv1.weight loaded from backbone.body.layer4.2.conv1.weight of shape (512, 2048, 1, 1)\n", + "2019-06-02 18:25:30,423 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.conv2.weight loaded from backbone.body.layer4.2.conv2.weight of shape (512, 512, 3, 3)\n", + "2019-06-02 18:25:30,424 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.layer4.2.conv3.weight loaded from backbone.body.layer4.2.conv3.weight of shape (2048, 512, 1, 1)\n", + "2019-06-02 18:25:30,425 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.stem.bn1.bias loaded from backbone.body.stem.bn1.bias of shape (64,)\n", + "2019-06-02 18:25:30,426 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.stem.bn1.running_mean loaded from backbone.body.stem.bn1.running_mean of shape (64,)\n", + "2019-06-02 18:25:30,428 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.stem.bn1.running_var loaded from backbone.body.stem.bn1.running_var of shape (64,)\n", + "2019-06-02 18:25:30,429 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.stem.bn1.weight loaded from backbone.body.stem.bn1.weight of shape (64,)\n", + "2019-06-02 18:25:30,431 maskrcnn_benchmark.utils.model_serialization INFO: backbone.body.stem.conv1.weight loaded from backbone.body.stem.conv1.weight of shape (64, 3, 7, 7)\n", + "2019-06-02 18:25:30,432 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner1.bias loaded from backbone.fpn.fpn_inner1.bias of shape (256,)\n", + "2019-06-02 18:25:30,434 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner1.weight loaded from backbone.fpn.fpn_inner1.weight of shape (256, 256, 1, 1)\n", + "2019-06-02 18:25:30,435 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner2.bias loaded from backbone.fpn.fpn_inner2.bias of shape (256,)\n", + "2019-06-02 18:25:30,437 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner2.weight loaded from backbone.fpn.fpn_inner2.weight of shape (256, 512, 1, 1)\n", + "2019-06-02 18:25:30,438 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner3.bias loaded from backbone.fpn.fpn_inner3.bias of shape (256,)\n", + "2019-06-02 18:25:30,439 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner3.weight loaded from backbone.fpn.fpn_inner3.weight of shape (256, 1024, 1, 1)\n", + "2019-06-02 18:25:30,442 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner4.bias loaded from backbone.fpn.fpn_inner4.bias of shape (256,)\n", + "2019-06-02 18:25:30,443 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_inner4.weight loaded from backbone.fpn.fpn_inner4.weight of shape (256, 2048, 1, 1)\n", + "2019-06-02 18:25:30,445 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer1.bias loaded from backbone.fpn.fpn_layer1.bias of shape (256,)\n", + "2019-06-02 18:25:30,446 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer1.weight loaded from backbone.fpn.fpn_layer1.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,448 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer2.bias loaded from backbone.fpn.fpn_layer2.bias of shape (256,)\n", + "2019-06-02 18:25:30,449 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer2.weight loaded from backbone.fpn.fpn_layer2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,451 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer3.bias loaded from backbone.fpn.fpn_layer3.bias of shape (256,)\n", + "2019-06-02 18:25:30,452 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer3.weight loaded from backbone.fpn.fpn_layer3.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,454 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer4.bias loaded from backbone.fpn.fpn_layer4.bias of shape (256,)\n", + "2019-06-02 18:25:30,455 maskrcnn_benchmark.utils.model_serialization INFO: backbone.fpn.fpn_layer4.weight loaded from backbone.fpn.fpn_layer4.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,456 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.box.feature_extractor.fc6.bias loaded from roi_heads.box.feature_extractor.fc6.bias of shape (1024,)\n", + "2019-06-02 18:25:30,458 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.box.feature_extractor.fc6.weight loaded from roi_heads.box.feature_extractor.fc6.weight of shape (1024, 12544)\n", + "2019-06-02 18:25:30,460 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.box.feature_extractor.fc7.bias loaded from roi_heads.box.feature_extractor.fc7.bias of shape (1024,)\n", + "2019-06-02 18:25:30,461 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.box.feature_extractor.fc7.weight loaded from roi_heads.box.feature_extractor.fc7.weight of shape (1024, 1024)\n", + "2019-06-02 18:25:30,462 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn1.bias loaded from roi_heads.mask.feature_extractor.mask_fcn1.bias of shape (256,)\n", + "2019-06-02 18:25:30,463 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn1.weight loaded from roi_heads.mask.feature_extractor.mask_fcn1.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,464 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn2.bias loaded from roi_heads.mask.feature_extractor.mask_fcn2.bias of shape (256,)\n", + "2019-06-02 18:25:30,466 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn2.weight loaded from roi_heads.mask.feature_extractor.mask_fcn2.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,467 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn3.bias loaded from roi_heads.mask.feature_extractor.mask_fcn3.bias of shape (256,)\n", + "2019-06-02 18:25:30,469 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn3.weight loaded from roi_heads.mask.feature_extractor.mask_fcn3.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,470 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn4.bias loaded from roi_heads.mask.feature_extractor.mask_fcn4.bias of shape (256,)\n", + "2019-06-02 18:25:30,471 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.feature_extractor.mask_fcn4.weight loaded from roi_heads.mask.feature_extractor.mask_fcn4.weight of shape (256, 256, 3, 3)\n", + "2019-06-02 18:25:30,473 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.predictor.conv5_mask.bias loaded from roi_heads.mask.predictor.conv5_mask.bias of shape (256,)\n", + "2019-06-02 18:25:30,474 maskrcnn_benchmark.utils.model_serialization INFO: roi_heads.mask.predictor.conv5_mask.weight loaded from roi_heads.mask.predictor.conv5_mask.weight of shape (256, 256, 2, 2)\n", + "2019-06-02 18:25:30,475 maskrcnn_benchmark.utils.model_serialization INFO: rpn.anchor_generator.cell_anchors.0 loaded from rpn.anchor_generator.cell_anchors.0 of shape (3, 4)\n", + "2019-06-02 18:25:30,477 maskrcnn_benchmark.utils.model_serialization INFO: rpn.anchor_generator.cell_anchors.1 loaded from rpn.anchor_generator.cell_anchors.1 of shape (3, 4)\n", + "2019-06-02 18:25:30,479 maskrcnn_benchmark.utils.model_serialization INFO: rpn.anchor_generator.cell_anchors.2 loaded from rpn.anchor_generator.cell_anchors.2 of shape (3, 4)\n", + "2019-06-02 18:25:30,480 maskrcnn_benchmark.utils.model_serialization INFO: rpn.anchor_generator.cell_anchors.3 loaded from rpn.anchor_generator.cell_anchors.3 of shape (3, 4)\n", + "2019-06-02 18:25:30,482 maskrcnn_benchmark.utils.model_serialization INFO: rpn.anchor_generator.cell_anchors.4 loaded from rpn.anchor_generator.cell_anchors.4 of shape (3, 4)\n", + "2019-06-02 18:25:30,484 maskrcnn_benchmark.utils.model_serialization INFO: rpn.head.bbox_pred.bias loaded from rpn.head.bbox_pred.bias of shape (12,)\n", + "2019-06-02 18:25:30,485 maskrcnn_benchmark.utils.model_serialization INFO: rpn.head.bbox_pred.weight loaded from rpn.head.bbox_pred.weight of shape (12, 256, 1, 1)\n", + "2019-06-02 18:25:30,486 maskrcnn_benchmark.utils.model_serialization INFO: rpn.head.cls_logits.bias loaded from rpn.head.cls_logits.bias of shape (3,)\n", + "2019-06-02 18:25:30,488 maskrcnn_benchmark.utils.model_serialization INFO: rpn.head.cls_logits.weight loaded from rpn.head.cls_logits.weight of shape (3, 256, 1, 1)\n", + "2019-06-02 18:25:30,489 maskrcnn_benchmark.utils.model_serialization INFO: rpn.head.conv.bias loaded from rpn.head.conv.bias of shape (256,)\n", + "2019-06-02 18:25:30,491 maskrcnn_benchmark.utils.model_serialization INFO: rpn.head.conv.weight loaded from rpn.head.conv.weight of shape (256, 256, 3, 3)\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "When using more than one image per GPU you may encounter an out-of-memory (OOM) error if your GPU does not have sufficient memory. If this happens, you can reduce SOLVER.IMS_PER_BATCH (for training) or TEST.IMS_PER_BATCH (for inference). For training, you must also adjust the learning rate and schedule length according to the linear scaling rule. See for example: https://github.com/facebookresearch/Detectron/blob/master/configs/getting_started/tutorial_1gpu_e2e_faster_rcnn_R-50-FPN.yaml#L14\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "2019-06-02 18:25:30,618 maskrcnn_benchmark.trainer INFO: Start training\n", + "2019-06-02 18:26:01,757 maskrcnn_benchmark.trainer INFO: eta: 0:25:25 iter: 20 loss: 1.1445 (1.5619) loss_classifier: 0.2888 (0.4150) loss_box_reg: 0.1517 (0.1493) loss_mask: 0.6485 (0.9632) loss_objectness: 0.0139 (0.0225) loss_rpn_box_reg: 0.0110 (0.0118) time: 1.5431 (1.5567) data: 0.0098 (0.0571) lr: 0.001800 max mem: 3682\n", + "2019-06-02 18:26:39,893 maskrcnn_benchmark.trainer INFO: eta: 0:27:42 iter: 40 loss: 1.1412 (1.3516) loss_classifier: 0.3149 (0.3693) loss_box_reg: 0.2243 (0.1846) loss_mask: 0.5719 (0.7712) loss_objectness: 0.0105 (0.0165) loss_rpn_box_reg: 0.0080 (0.0100) time: 1.8800 (1.7318) data: 0.0097 (0.0336) lr: 0.001933 max mem: 3964\n", + "2019-06-02 18:27:17,478 maskrcnn_benchmark.trainer INFO: eta: 0:27:54 iter: 60 loss: 1.0707 (1.2536) loss_classifier: 0.2933 (0.3452) loss_box_reg: 0.2103 (0.1916) loss_mask: 0.5354 (0.6944) loss_objectness: 0.0071 (0.0133) loss_rpn_box_reg: 0.0077 (0.0092) time: 1.8667 (1.7809) data: 0.0097 (0.0257) lr: 0.002067 max mem: 3964\n", + "2019-06-02 18:27:51,484 maskrcnn_benchmark.trainer INFO: eta: 0:26:59 iter: 80 loss: 0.8470 (1.1564) loss_classifier: 0.2146 (0.3149) loss_box_reg: 0.1536 (0.1821) loss_mask: 0.4718 (0.6395) loss_objectness: 0.0047 (0.0113) loss_rpn_box_reg: 0.0064 (0.0087) time: 1.6526 (1.7608) data: 0.0098 (0.0219) lr: 0.002200 max mem: 3964\n", + "2019-06-02 18:28:24,797 maskrcnn_benchmark.trainer INFO: eta: 0:26:07 iter: 100 loss: 0.6682 (1.0617) loss_classifier: 0.1826 (0.2883) loss_box_reg: 0.1375 (0.1720) loss_mask: 0.3474 (0.5834) loss_objectness: 0.0029 (0.0099) loss_rpn_box_reg: 0.0048 (0.0081) time: 1.7250 (1.7418) data: 0.0100 (0.0196) lr: 0.002333 max mem: 3964\n", + "2019-06-02 18:28:24,800 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000100.pth\n", + "2019-06-02 18:29:01,969 maskrcnn_benchmark.trainer INFO: eta: 0:25:49 iter: 120 loss: 0.6775 (1.0049) loss_classifier: 0.1876 (0.2719) loss_box_reg: 0.1401 (0.1673) loss_mask: 0.3231 (0.5485) loss_objectness: 0.0045 (0.0093) loss_rpn_box_reg: 0.0057 (0.0079) time: 1.8055 (1.7612) data: 0.0100 (0.0300) lr: 0.002467 max mem: 3964\n", + "2019-06-02 18:29:38,625 maskrcnn_benchmark.trainer INFO: eta: 0:25:23 iter: 140 loss: 0.5469 (0.9371) loss_classifier: 0.1636 (0.2570) loss_box_reg: 0.0984 (0.1575) loss_mask: 0.2496 (0.5061) loss_objectness: 0.0036 (0.0088) loss_rpn_box_reg: 0.0058 (0.0077) time: 1.8503 (1.7715) data: 0.0099 (0.0271) lr: 0.002600 max mem: 3984\n", + "2019-06-02 18:30:16,395 maskrcnn_benchmark.trainer INFO: eta: 0:25:00 iter: 160 loss: 0.4306 (0.8793) loss_classifier: 0.1549 (0.2447) loss_box_reg: 0.0649 (0.1465) loss_mask: 0.2048 (0.4718) loss_objectness: 0.0040 (0.0086) loss_rpn_box_reg: 0.0057 (0.0077) time: 1.9542 (1.7861) data: 0.0098 (0.0250) lr: 0.002733 max mem: 3984\n", + "2019-06-02 18:30:54,339 maskrcnn_benchmark.trainer INFO: eta: 0:24:34 iter: 180 loss: 0.4060 (0.8277) loss_classifier: 0.1222 (0.2322) loss_box_reg: 0.0425 (0.1354) loss_mask: 0.2194 (0.4441) loss_objectness: 0.0057 (0.0084) loss_rpn_box_reg: 0.0048 (0.0075) time: 1.8666 (1.7984) data: 0.0098 (0.0234) lr: 0.002867 max mem: 3984\n", + "2019-06-02 18:31:31,686 maskrcnn_benchmark.trainer INFO: eta: 0:24:04 iter: 200 loss: 0.3353 (0.7811) loss_classifier: 0.1224 (0.2217) loss_box_reg: 0.0405 (0.1266) loss_mask: 0.1575 (0.4177) loss_objectness: 0.0031 (0.0079) loss_rpn_box_reg: 0.0040 (0.0072) time: 1.8499 (1.8053) data: 0.0098 (0.0220) lr: 0.003000 max mem: 3984\n", + "2019-06-02 18:31:31,694 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000200.pth\n", + "2019-06-02 18:32:11,665 maskrcnn_benchmark.trainer INFO: eta: 0:23:41 iter: 220 loss: 0.2797 (0.7377) loss_classifier: 0.1134 (0.2118) loss_box_reg: 0.0337 (0.1184) loss_mask: 0.1281 (0.3930) loss_objectness: 0.0016 (0.0075) loss_rpn_box_reg: 0.0047 (0.0070) time: 1.9676 (1.8229) data: 0.0099 (0.0274) lr: 0.003133 max mem: 4045\n", + "2019-06-02 18:32:48,993 maskrcnn_benchmark.trainer INFO: eta: 0:23:08 iter: 240 loss: 0.2811 (0.7005) loss_classifier: 0.0951 (0.2025) loss_box_reg: 0.0390 (0.1120) loss_mask: 0.1331 (0.3720) loss_objectness: 0.0030 (0.0072) loss_rpn_box_reg: 0.0035 (0.0067) time: 1.8686 (1.8265) data: 0.0100 (0.0260) lr: 0.003267 max mem: 4045\n", + "2019-06-02 18:33:26,674 maskrcnn_benchmark.trainer INFO: eta: 0:22:34 iter: 260 loss: 0.2268 (0.6662) loss_classifier: 0.0845 (0.1937) loss_box_reg: 0.0259 (0.1057) loss_mask: 0.0996 (0.3535) loss_objectness: 0.0015 (0.0069) loss_rpn_box_reg: 0.0028 (0.0064) time: 1.7613 (1.8310) data: 0.0101 (0.0248) lr: 0.003400 max mem: 4045\n", + "2019-06-02 18:34:05,409 maskrcnn_benchmark.trainer INFO: eta: 0:22:03 iter: 280 loss: 0.2316 (0.6376) loss_classifier: 0.0758 (0.1859) loss_box_reg: 0.0310 (0.1005) loss_mask: 0.1212 (0.3385) loss_objectness: 0.0011 (0.0065) loss_rpn_box_reg: 0.0032 (0.0063) time: 1.9362 (1.8385) data: 0.0101 (0.0238) lr: 0.003533 max mem: 4045\n", + "2019-06-02 18:34:45,986 maskrcnn_benchmark.trainer INFO: eta: 0:21:35 iter: 300 loss: 0.1890 (0.6097) loss_classifier: 0.0672 (0.1786) loss_box_reg: 0.0266 (0.0958) loss_mask: 0.1022 (0.3229) loss_objectness: 0.0011 (0.0063) loss_rpn_box_reg: 0.0026 (0.0061) time: 1.9844 (1.8512) data: 0.0101 (0.0229) lr: 0.003667 max mem: 4045\n", + "2019-06-02 18:34:45,996 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000300.pth\n", + "2019-06-02 18:35:26,169 maskrcnn_benchmark.trainer INFO: eta: 0:21:05 iter: 320 loss: 0.1838 (0.5837) loss_classifier: 0.0716 (0.1717) loss_box_reg: 0.0230 (0.0915) loss_mask: 0.0802 (0.3086) loss_objectness: 0.0009 (0.0060) loss_rpn_box_reg: 0.0028 (0.0059) time: 2.0302 (1.8611) data: 0.0101 (0.0273) lr: 0.003800 max mem: 4045\n", + "2019-06-02 18:36:03,978 maskrcnn_benchmark.trainer INFO: eta: 0:20:29 iter: 340 loss: 0.1670 (0.5607) loss_classifier: 0.0494 (0.1650) loss_box_reg: 0.0220 (0.0876) loss_mask: 0.0916 (0.2968) loss_objectness: 0.0006 (0.0057) loss_rpn_box_reg: 0.0013 (0.0056) time: 1.7739 (1.8628) data: 0.0102 (0.0263) lr: 0.003933 max mem: 4045\n", + "2019-06-02 18:36:45,869 maskrcnn_benchmark.trainer INFO: eta: 0:20:00 iter: 360 loss: 0.2075 (0.5412) loss_classifier: 0.0643 (0.1597) loss_box_reg: 0.0332 (0.0845) loss_mask: 0.0991 (0.2861) loss_objectness: 0.0008 (0.0055) loss_rpn_box_reg: 0.0019 (0.0055) time: 2.0951 (1.8757) data: 0.0102 (0.0254) lr: 0.004067 max mem: 4069\n", + "2019-06-02 18:37:25,994 maskrcnn_benchmark.trainer INFO: eta: 0:19:27 iter: 380 loss: 0.1814 (0.5235) loss_classifier: 0.0546 (0.1546) loss_box_reg: 0.0247 (0.0816) loss_mask: 0.0975 (0.2768) loss_objectness: 0.0005 (0.0052) loss_rpn_box_reg: 0.0023 (0.0053) time: 2.0547 (1.8826) data: 0.0103 (0.0246) lr: 0.004200 max mem: 4069\n", + "2019-06-02 18:38:05,156 maskrcnn_benchmark.trainer INFO: eta: 0:18:51 iter: 400 loss: 0.1724 (0.5063) loss_classifier: 0.0503 (0.1496) loss_box_reg: 0.0215 (0.0787) loss_mask: 0.0902 (0.2679) loss_objectness: 0.0002 (0.0050) loss_rpn_box_reg: 0.0015 (0.0051) time: 1.9256 (1.8863) data: 0.0103 (0.0240) lr: 0.004333 max mem: 4069\n", + "2019-06-02 18:38:05,159 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000400.pth\n", + "2019-06-02 18:38:47,340 maskrcnn_benchmark.trainer INFO: eta: 0:18:20 iter: 420 loss: 0.1519 (0.4901) loss_classifier: 0.0508 (0.1448) loss_box_reg: 0.0172 (0.0760) loss_mask: 0.0781 (0.2595) loss_objectness: 0.0001 (0.0048) loss_rpn_box_reg: 0.0017 (0.0050) time: 2.1132 (1.8969) data: 0.0103 (0.0267) lr: 0.004467 max mem: 4069\n", + "2019-06-02 18:39:27,212 maskrcnn_benchmark.trainer INFO: eta: 0:17:44 iter: 440 loss: 0.1524 (0.4753) loss_classifier: 0.0423 (0.1405) loss_box_reg: 0.0227 (0.0737) loss_mask: 0.0650 (0.2516) loss_objectness: 0.0003 (0.0047) loss_rpn_box_reg: 0.0021 (0.0049) time: 1.9556 (1.9013) data: 0.0104 (0.0260) lr: 0.004600 max mem: 4085\n", + "2019-06-02 18:40:07,888 maskrcnn_benchmark.trainer INFO: eta: 0:17:09 iter: 460 loss: 0.1374 (0.4616) loss_classifier: 0.0361 (0.1365) loss_box_reg: 0.0201 (0.0715) loss_mask: 0.0701 (0.2443) loss_objectness: 0.0003 (0.0045) loss_rpn_box_reg: 0.0018 (0.0048) time: 1.9376 (1.9071) data: 0.0103 (0.0253) lr: 0.004733 max mem: 4085\n", + "2019-06-02 18:40:49,738 maskrcnn_benchmark.trainer INFO: eta: 0:16:35 iter: 480 loss: 0.1339 (0.4482) loss_classifier: 0.0375 (0.1325) loss_box_reg: 0.0204 (0.0694) loss_mask: 0.0691 (0.2373) loss_objectness: 0.0001 (0.0043) loss_rpn_box_reg: 0.0012 (0.0046) time: 2.0188 (1.9148) data: 0.0103 (0.0247) lr: 0.004867 max mem: 4085\n", + "2019-06-02 18:41:30,488 maskrcnn_benchmark.trainer INFO: eta: 0:15:59 iter: 500 loss: 0.1137 (0.4359) loss_classifier: 0.0400 (0.1293) loss_box_reg: 0.0183 (0.0675) loss_mask: 0.0510 (0.2304) loss_objectness: 0.0006 (0.0042) loss_rpn_box_reg: 0.0016 (0.0045) time: 2.0013 (1.9197) data: 0.0105 (0.0242) lr: 0.005000 max mem: 4085\n", + "2019-06-02 18:41:30,498 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000500.pth\n", + "2019-06-02 18:42:14,193 maskrcnn_benchmark.trainer INFO: eta: 0:15:26 iter: 520 loss: 0.1359 (0.4249) loss_classifier: 0.0486 (0.1262) loss_box_reg: 0.0196 (0.0658) loss_mask: 0.0663 (0.2243) loss_objectness: 0.0002 (0.0041) loss_rpn_box_reg: 0.0016 (0.0044) time: 2.0601 (1.9299) data: 0.0104 (0.0268) lr: 0.005000 max mem: 4085\n", + "2019-06-02 18:42:55,547 maskrcnn_benchmark.trainer INFO: eta: 0:14:50 iter: 540 loss: 0.1428 (0.4143) loss_classifier: 0.0347 (0.1230) loss_box_reg: 0.0164 (0.0641) loss_mask: 0.0742 (0.2189) loss_objectness: 0.0001 (0.0039) loss_rpn_box_reg: 0.0011 (0.0043) time: 2.0419 (1.9350) data: 0.0104 (0.0262) lr: 0.005000 max mem: 4085\n", + "2019-06-02 18:43:36,910 maskrcnn_benchmark.trainer INFO: eta: 0:14:13 iter: 560 loss: 0.1221 (0.4041) loss_classifier: 0.0331 (0.1200) loss_box_reg: 0.0159 (0.0625) loss_mask: 0.0595 (0.2136) loss_objectness: 0.0003 (0.0038) loss_rpn_box_reg: 0.0011 (0.0042) time: 2.0205 (1.9398) data: 0.0106 (0.0257) lr: 0.005000 max mem: 4085\n", + "2019-06-02 18:44:16,298 maskrcnn_benchmark.trainer INFO: eta: 0:13:35 iter: 580 loss: 0.0916 (0.3943) loss_classifier: 0.0323 (0.1171) loss_box_reg: 0.0120 (0.0609) loss_mask: 0.0499 (0.2084) loss_objectness: 0.0002 (0.0037) loss_rpn_box_reg: 0.0012 (0.0041) time: 1.9197 (1.9408) data: 0.0105 (0.0252) lr: 0.005000 max mem: 4093\n", + "2019-06-02 18:44:56,001 maskrcnn_benchmark.trainer INFO: eta: 0:12:56 iter: 600 loss: 0.1242 (0.3854) loss_classifier: 0.0348 (0.1145) loss_box_reg: 0.0146 (0.0594) loss_mask: 0.0651 (0.2037) loss_objectness: 0.0002 (0.0036) loss_rpn_box_reg: 0.0011 (0.0041) time: 1.9603 (1.9423) data: 0.0103 (0.0247) lr: 0.005000 max mem: 4093\n", + "2019-06-02 18:44:56,008 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000600.pth\n", + "2019-06-02 18:45:33,872 maskrcnn_benchmark.trainer INFO: eta: 0:12:17 iter: 620 loss: 0.0824 (0.3761) loss_classifier: 0.0214 (0.1118) loss_box_reg: 0.0079 (0.0579) loss_mask: 0.0436 (0.1989) loss_objectness: 0.0001 (0.0035) loss_rpn_box_reg: 0.0006 (0.0040) time: 1.7794 (1.9407) data: 0.0104 (0.0271) lr: 0.005000 max mem: 4093\n", + "2019-06-02 18:46:15,480 maskrcnn_benchmark.trainer INFO: eta: 0:11:40 iter: 640 loss: 0.1025 (0.3676) loss_classifier: 0.0313 (0.1093) loss_box_reg: 0.0117 (0.0565) loss_mask: 0.0525 (0.1945) loss_objectness: 0.0002 (0.0035) loss_rpn_box_reg: 0.0008 (0.0039) time: 1.9964 (1.9451) data: 0.0107 (0.0266) lr: 0.005000 max mem: 4093\n", + "2019-06-02 18:46:54,442 maskrcnn_benchmark.trainer INFO: eta: 0:11:01 iter: 660 loss: 0.0961 (0.3595) loss_classifier: 0.0282 (0.1068) loss_box_reg: 0.0091 (0.0551) loss_mask: 0.0556 (0.1904) loss_objectness: 0.0001 (0.0034) loss_rpn_box_reg: 0.0007 (0.0038) time: 1.9556 (1.9452) data: 0.0104 (0.0261) lr: 0.005000 max mem: 4093\n", + "2019-06-02 18:47:33,371 maskrcnn_benchmark.trainer INFO: eta: 0:10:22 iter: 680 loss: 0.0902 (0.3517) loss_classifier: 0.0275 (0.1045) loss_box_reg: 0.0073 (0.0538) loss_mask: 0.0446 (0.1863) loss_objectness: 0.0001 (0.0033) loss_rpn_box_reg: 0.0010 (0.0037) time: 1.9129 (1.9452) data: 0.0105 (0.0257) lr: 0.005000 max mem: 4093\n", + "2019-06-02 18:48:12,252 maskrcnn_benchmark.trainer INFO: eta: 0:09:43 iter: 700 loss: 0.0981 (0.3448) loss_classifier: 0.0275 (0.1024) loss_box_reg: 0.0103 (0.0527) loss_mask: 0.0611 (0.1829) loss_objectness: 0.0001 (0.0032) loss_rpn_box_reg: 0.0014 (0.0036) time: 1.9010 (1.9452) data: 0.0105 (0.0253) lr: 0.000500 max mem: 4093\n", + "2019-06-02 18:48:12,255 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000700.pth\n", + "2019-06-02 18:48:52,993 maskrcnn_benchmark.trainer INFO: eta: 0:09:05 iter: 720 loss: 0.0963 (0.3379) loss_classifier: 0.0256 (0.1003) loss_box_reg: 0.0104 (0.0516) loss_mask: 0.0569 (0.1793) loss_objectness: 0.0001 (0.0031) loss_rpn_box_reg: 0.0008 (0.0036) time: 2.0065 (1.9477) data: 0.0104 (0.0275) lr: 0.000500 max mem: 4093\n", + "2019-06-02 18:49:32,413 maskrcnn_benchmark.trainer INFO: eta: 0:08:26 iter: 740 loss: 0.0970 (0.3314) loss_classifier: 0.0244 (0.0984) loss_box_reg: 0.0099 (0.0505) loss_mask: 0.0453 (0.1759) loss_objectness: 0.0001 (0.0031) loss_rpn_box_reg: 0.0008 (0.0035) time: 1.9561 (1.9484) data: 0.0104 (0.0270) lr: 0.000500 max mem: 4093\n", + "2019-06-02 18:50:12,060 maskrcnn_benchmark.trainer INFO: eta: 0:07:47 iter: 760 loss: 0.0891 (0.3255) loss_classifier: 0.0305 (0.0967) loss_box_reg: 0.0089 (0.0496) loss_mask: 0.0521 (0.1728) loss_objectness: 0.0002 (0.0030) loss_rpn_box_reg: 0.0009 (0.0034) time: 1.9450 (1.9493) data: 0.0105 (0.0266) lr: 0.000500 max mem: 4093\n", + "2019-06-02 18:50:51,359 maskrcnn_benchmark.trainer INFO: eta: 0:07:08 iter: 780 loss: 0.0689 (0.3191) loss_classifier: 0.0214 (0.0948) loss_box_reg: 0.0076 (0.0485) loss_mask: 0.0436 (0.1695) loss_objectness: 0.0001 (0.0029) loss_rpn_box_reg: 0.0005 (0.0034) time: 1.9242 (1.9497) data: 0.0106 (0.0262) lr: 0.000500 max mem: 4093\n", + "2019-06-02 18:51:30,422 maskrcnn_benchmark.trainer INFO: eta: 0:06:29 iter: 800 loss: 0.0606 (0.3130) loss_classifier: 0.0171 (0.0929) loss_box_reg: 0.0068 (0.0475) loss_mask: 0.0367 (0.1664) loss_objectness: 0.0001 (0.0029) loss_rpn_box_reg: 0.0005 (0.0033) time: 1.9414 (1.9497) data: 0.0103 (0.0258) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:51:30,430 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000800.pth\n", + "2019-06-02 18:52:11,778 maskrcnn_benchmark.trainer INFO: eta: 0:05:51 iter: 820 loss: 0.0856 (0.3077) loss_classifier: 0.0259 (0.0913) loss_box_reg: 0.0090 (0.0466) loss_mask: 0.0485 (0.1637) loss_objectness: 0.0000 (0.0028) loss_rpn_box_reg: 0.0006 (0.0032) time: 2.0000 (1.9526) data: 0.0104 (0.0274) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:52:50,917 maskrcnn_benchmark.trainer INFO: eta: 0:05:12 iter: 840 loss: 0.0698 (0.3024) loss_classifier: 0.0236 (0.0897) loss_box_reg: 0.0062 (0.0458) loss_mask: 0.0450 (0.1610) loss_objectness: 0.0001 (0.0027) loss_rpn_box_reg: 0.0004 (0.0032) time: 1.9542 (1.9527) data: 0.0104 (0.0270) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:53:30,312 maskrcnn_benchmark.trainer INFO: eta: 0:04:33 iter: 860 loss: 0.0664 (0.2972) loss_classifier: 0.0175 (0.0882) loss_box_reg: 0.0065 (0.0449) loss_mask: 0.0435 (0.1583) loss_objectness: 0.0000 (0.0027) loss_rpn_box_reg: 0.0003 (0.0031) time: 1.8427 (1.9531) data: 0.0104 (0.0267) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:54:12,617 maskrcnn_benchmark.trainer INFO: eta: 0:03:54 iter: 880 loss: 0.0959 (0.2928) loss_classifier: 0.0296 (0.0869) loss_box_reg: 0.0087 (0.0442) loss_mask: 0.0573 (0.1560) loss_objectness: 0.0001 (0.0027) loss_rpn_box_reg: 0.0010 (0.0031) time: 2.1273 (1.9568) data: 0.0104 (0.0263) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:54:52,317 maskrcnn_benchmark.trainer INFO: eta: 0:03:15 iter: 900 loss: 0.0774 (0.2881) loss_classifier: 0.0200 (0.0855) loss_box_reg: 0.0065 (0.0434) loss_mask: 0.0428 (0.1535) loss_objectness: 0.0000 (0.0026) loss_rpn_box_reg: 0.0004 (0.0030) time: 1.9623 (1.9574) data: 0.0104 (0.0260) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:54:52,320 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0000900.pth\n", + "2019-06-02 18:55:35,081 maskrcnn_benchmark.trainer INFO: eta: 0:02:36 iter: 920 loss: 0.0763 (0.2840) loss_classifier: 0.0269 (0.0843) loss_box_reg: 0.0079 (0.0428) loss_mask: 0.0498 (0.1514) loss_objectness: 0.0001 (0.0026) loss_rpn_box_reg: 0.0005 (0.0030) time: 2.1085 (1.9614) data: 0.0105 (0.0274) lr: 0.000050 max mem: 4093\n", + "2019-06-02 18:56:15,652 maskrcnn_benchmark.trainer INFO: eta: 0:01:57 iter: 940 loss: 0.0823 (0.2798) loss_classifier: 0.0235 (0.0831) loss_box_reg: 0.0078 (0.0421) loss_mask: 0.0511 (0.1493) loss_objectness: 0.0001 (0.0025) loss_rpn_box_reg: 0.0007 (0.0029) time: 2.0190 (1.9628) data: 0.0102 (0.0270) lr: 0.000050 max mem: 4102\n", + "2019-06-02 18:56:55,399 maskrcnn_benchmark.trainer INFO: eta: 0:01:18 iter: 960 loss: 0.0843 (0.2758) loss_classifier: 0.0246 (0.0819) loss_box_reg: 0.0096 (0.0414) loss_mask: 0.0429 (0.1472) loss_objectness: 0.0001 (0.0025) loss_rpn_box_reg: 0.0006 (0.0029) time: 2.0232 (1.9633) data: 0.0102 (0.0267) lr: 0.000050 max mem: 4102\n", + "2019-06-02 18:57:32,252 maskrcnn_benchmark.trainer INFO: eta: 0:00:39 iter: 980 loss: 0.0525 (0.2716) loss_classifier: 0.0154 (0.0806) loss_box_reg: 0.0049 (0.0407) loss_mask: 0.0351 (0.1450) loss_objectness: 0.0000 (0.0024) loss_rpn_box_reg: 0.0004 (0.0028) time: 1.7520 (1.9608) data: 0.0103 (0.0264) lr: 0.000050 max mem: 4102\n", + "2019-06-02 18:58:10,741 maskrcnn_benchmark.trainer INFO: eta: 0:00:00 iter: 1000 loss: 0.0669 (0.2677) loss_classifier: 0.0201 (0.0795) loss_box_reg: 0.0075 (0.0400) loss_mask: 0.0417 (0.1430) loss_objectness: 0.0000 (0.0024) loss_rpn_box_reg: 0.0003 (0.0028) time: 1.8464 (1.9601) data: 0.0104 (0.0261) lr: 0.000050 max mem: 4102\n", + "2019-06-02 18:58:10,753 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_0001000.pth\n", + "2019-06-02 18:58:12,164 maskrcnn_benchmark.utils.checkpoint INFO: Saving checkpoint to shapeDir/model_final.pth\n", + "2019-06-02 18:58:14,438 maskrcnn_benchmark.trainer INFO: Total training time: 0:32:43.816510 (1.9638 s / it)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ONldqRzHUAm0", + "colab_type": "text" + }, + "source": [ + "# Evaluation\n", + "\n", + "Now after our model is trainined, we would like to see how well it predicts objects in our sample images. One way to validate your model is through a standard metric called COCO mAP. This metric is used widely. Hence, we shall do this now. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s8YFliWAUG-E", + "colab_type": "text" + }, + "source": [ + "### Doing Inference\n", + "\n", + "Helper function to perform inference." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6VbvZXhWkQ-0", + "colab_type": "code", + "colab": {} + }, + "source": [ + "\n", + "def do_inference(\n", + " model,\n", + " data_loader,\n", + " dataset_name,\n", + " iou_types=(\"bbox\",),\n", + " box_only=False,\n", + " device=\"cuda\",\n", + " expected_results=(),\n", + " expected_results_sigma_tol=4,\n", + " output_folder=None,):\n", + " \n", + " # convert to a torch.device for efficiency\n", + " device = torch.device(device)\n", + " num_devices = get_world_size()\n", + " logger = logging.getLogger(\"maskrcnn_benchmark.inference\")\n", + " dataset = data_loader.dataset\n", + " logger.info(\"Start evaluation on {} dataset({} images).\".format(dataset_name, len(dataset)))\n", + " total_timer = Timer()\n", + " inference_timer = Timer()\n", + " total_timer.tic()\n", + " predictions = compute_on_dataset(model, data_loader, device, inference_timer)\n", + " \n", + " # wait for all processes to complete before measuring the time\n", + " synchronize()\n", + " total_time = total_timer.toc()\n", + " total_time_str = get_time_str(total_time)\n", + " logger.info(\n", + " \"Total run time: {} ({} s / img per device, on {} devices)\".format(\n", + " total_time_str, total_time * num_devices / len(dataset), num_devices\n", + " )\n", + " )\n", + " \n", + " total_infer_time = get_time_str(inference_timer.total_time)\n", + " logger.info(\n", + " \"Model inference time: {} ({} s / img per device, on {} devices)\".format(\n", + " total_infer_time,\n", + " inference_timer.total_time * num_devices / len(dataset),\n", + " num_devices,\n", + " )\n", + " )\n", + " \n", + " predictions = _accumulate_predictions_from_multiple_gpus(predictions)\n", + " if not is_main_process():\n", + " return\n", + "\n", + " if output_folder:\n", + " torch.save(predictions, os.path.join(output_folder, \"predictions.pth\"))\n", + "\n", + " extra_args = dict(\n", + " box_only=box_only,\n", + " iou_types=iou_types,\n", + " expected_results=expected_results,\n", + " expected_results_sigma_tol=expected_results_sigma_tol,\n", + " )\n", + "\n", + " return coco_evaluation(dataset=dataset,\n", + " predictions=predictions,\n", + " output_folder=output_folder,\n", + " **extra_args)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YcAidp5fUccv", + "colab_type": "text" + }, + "source": [ + "### Testing Function\n", + "\n", + "Driver function to run the model test" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "rIC4k6dUd4UB", + "colab_type": "code", + "colab": {} + }, + "source": [ + "def run_test(cfg, model, distributed, dataset):\n", + " if distributed:\n", + " model = model.module\n", + " torch.cuda.empty_cache() # TODO check if it helps\n", + " iou_types = (\"bbox\",)\n", + " \n", + " data_loaders_val = build_data_loader(cfg, dataset, is_train=False)\n", + " mkdir(\"shapeVal\")\n", + " for data_loader in data_loaders_val:\n", + " do_inference(\n", + " model,\n", + " data_loader, # For test we need this as zero\n", + " dataset_name=\"shape-val\",\n", + " iou_types=iou_types,\n", + " box_only=False if cfg.MODEL.RETINANET_ON else cfg.MODEL.RPN_ONLY,\n", + " device=cfg.MODEL.DEVICE,\n", + " expected_results=cfg.TEST.EXPECTED_RESULTS,\n", + " expected_results_sigma_tol=cfg.TEST.EXPECTED_RESULTS_SIGMA_TOL,\n", + " output_folder=\"shapeVal\",\n", + " )\n", + " synchronize()\n", + "\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LdVLSfrAd9Mi", + "colab_type": "text" + }, + "source": [ + "### Run Evaluation" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "0GAUbBC-hsUq", + "colab_type": "code", + "outputId": "6a51b2cc-d7c3-40b9-b8e7-15a96f30df8b", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 598 + } + }, + "source": [ + "cfg.merge_from_list(['TEST.IMS_PER_BATCH', 1])\n", + "\n", + "run_test(cfg, model=model, distributed=False, dataset=ShapeDataset(50))" + ], + "execution_count": 18, + "outputs": [ + { + "output_type": "stream", + "text": [ + "loading annotations into memory...\n", + "Done (t=0.00s)\n", + "creating index...\n", + "index created!\n", + "2019-06-02 18:58:27,806 maskrcnn_benchmark.inference INFO: Start evaluation on shape-val dataset(50 images).\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "100%|██████████| 50/50 [00:05<00:00, 10.60it/s]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "2019-06-02 18:58:33,089 maskrcnn_benchmark.inference INFO: Total run time: 0:00:05.282553 (0.10565105438232422 s / img per device, on 1 devices)\n", + "2019-06-02 18:58:33,090 maskrcnn_benchmark.inference INFO: Model inference time: 0:00:04.614043 (0.09228085994720459 s / img per device, on 1 devices)\n", + "2019-06-02 18:58:33,105 maskrcnn_benchmark.inference INFO: Preparing results for COCO format\n", + "2019-06-02 18:58:33,107 maskrcnn_benchmark.inference INFO: Preparing bbox results\n", + "2019-06-02 18:58:33,117 maskrcnn_benchmark.inference INFO: Evaluating predictions\n", + "Loading and preparing results...\n", + "DONE (t=0.00s)\n", + "creating index...\n", + "index created!\n", + "Running per image evaluation...\n", + "Evaluate annotation type *bbox*\n", + "DONE (t=0.04s).\n", + "Accumulating evaluation results...\n", + "DONE (t=0.02s).\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.885\n", + " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.976\n", + " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.957\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.885\n", + " Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.714\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.908\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.908\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.908\n", + " Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n", + "2019-06-02 18:58:33,193 maskrcnn_benchmark.inference INFO: OrderedDict([('bbox', OrderedDict([('AP', 0.8847684644516638), ('AP50', 0.9764087519863097), ('AP75', 0.956912357902457), ('APs', -1.0), ('APm', 0.8847684644516638), ('APl', -1.0)]))])\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ccHt8YMdKq6K", + "colab_type": "text" + }, + "source": [ + "# Visualise\n", + "\n", + "Another important part of validating your model is visualising the results. This is done below" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "kb9VchvVzRpu", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Load Trained Model\n", + "config_file = \"shapes_config.yaml\"\n", + "\n", + "cfg.merge_from_file(config_file)\n", + "# manual override some options\n", + "cfg.merge_from_list([\"MODEL.DEVICE\", \"cpu\"])\n", + "\n", + "vis_demo = COCODemo(\n", + " cfg, \n", + " min_image_size=800,\n", + " confidence_threshold=0.7)\n", + "\n", + "# Load Dataset\n", + "dataset = ShapeDataset(50)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c8b6wHAXjyE5", + "colab_type": "text" + }, + "source": [ + "## Visualise" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "StOBbFmujxIw", + "colab_type": "code", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 486 + }, + "outputId": "c52ae633-20d7-4aa8-ad8a-1e1caeaf3ba5" + }, + "source": [ + "# Visualise Results\n", + "rows = 2\n", + "cols = 2\n", + "fig = plt.figure(figsize=(8, 8))\n", + "for i in range(1, rows*cols+1):\n", + " img = dataset.load_image(i)\n", + " image = np.array(img)[:, :, [2, 1, 0]]\n", + " result = vis_demo.run_on_opencv_image(image)\n", + " \n", + " fig.add_subplot(rows, cols, i)\n", + " plt.imshow(result)\n", + "plt.show()\n" + ], + "execution_count": 27, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAHVCAYAAADLvzPyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X/QHPV15/vPiX4ktsAghAvLkkDg\nEN8I/8BCYBKz4DLxXlkhiM1SBG8uYCCrdV1IbNjYBrNVkqvWa3vZhXUqvmZlAxZcYiDEKbSE2CbY\nhqISZCSMAcFiBBEgRSDz0xj7WjwP5/4xPVJrND39u/vbM+9XleqZ6emePjOPvs+Zc/rbPebuAgAA\n4fi1tgMAAAB7IzkDABAYkjMAAIEhOQMAEBiSMwAAgSE5AwAQmNqSs5ktN7PHzGyLmV1S134A1Iux\nDDTP6jjP2cxmSPqJpA9L2ibpPkkfdfdHKt8ZgNowloF21FU5Hydpi7s/6e67JN0oaWVN+wJQH8Yy\n0IKZNT3vAknPxO5vk/T+xCD2n+mz5s2uKZRwHXXwkpGPb35+T3Fy1MFL9rk/bL1h6+bZ97DtRu17\n1HZl95n2vHmeL75NlliLyhJ31ufYtGnT8+7+1koCKy7XWJakA+cc7G8/6LBagwK65F9efEovv/a8\n5dmmruScysxWSVolSbPmzdIRq3+zrVBas/HcjSMfP+ra9+y17uD9YesNWzfPvodtN2rfo7Yru8+0\n583zfPFtssRaVJa4sz6HmT1VSVANiI/nt809VNdd9E8tRwSE4+wrfyf3NnUl5+2SFsXuL4yW7ebu\nayWtlaQ3LX7zRF7gO/4HfPO5D+6zLMnmcx8s/Md/2H76y7JuX2Tbottlfb605y27vwmWOpalvcfz\nkkXHTOR4BqpUV3K+T9KRZna4egP5TEn/rqZ9TYw8CTxp+2HbHnXte1KTV9q+05Ll4GP9+3mTZpb3\nYNg6dVbLY46xDLSgluTs7lNmdqGk70iaIekad99cx74A1IexDLSjtmPO7n67pNvrev5JVGUru6p9\nj6qAy7Tf09Cmbg5jGWheaxPCgDJoUwMYZ1y+EwCAwJCcAQAIDMl5Ahx17XsyzciuY7+bz32w8f2O\n2mdIx6qT4uz/vgBMLpIzAACBYUJYCx7benTiY/FKaubqszNvl3UfaZVj0nZZ9p31dZWR5fzoPBcn\naUto8QAISy3fSpXXmxa/2Sfp8p39JPbOxQ+0HsOgpmOq83SrrnvkvIc2ufuytuPIa8miY5zLdwJ7\nnH3l7+iRZzblurY2bW0AAAJDW3vCpLWnQ6jqAWDSkZwnSJ5j1lWLH2NtY+Y4AHQJbW0AAAJD5TwB\nilTMj209utLWdrxajk8C6+2nst0AwFigcgYAIDBUzkhU9eQwTpkCgGyonMfcqJb2intO14p7Tm8w\nGgBAFiRnAAACQ3IeU49tPTq1ah52O+m5AADNITkDABAYkjMAAIEhOQMAEBhOpRpDWY81D1t++wm3\njHxOrrkNAPWjcgYAIDCFk7OZLTKz75vZI2a22cw+ES0/yMzuMLPHo59zqwsXo+SZoT1qnVHrMXN7\nPDGegbCUqZynJP1Hd18i6XhJF5jZEkmXSLrT3Y+UdGd0H0DYGM9AQAonZ3ff4e73R7dflfSopAWS\nVkpaF622TtJpZYMEUC/GMxCWSiaEmdliSe+TtEHSIe6+I3roWUmHVLEPJEtrNRe5ROeKe05nctiE\nYjwD7Ss9IczM9pP0N5I+6e4/iz/m7i7JE7ZbZWYbzWzj9M+nyoYxsepIzPFtOf48WaoYzy+99nwD\nkQLjrVRyNrNZ6g3kG9z9W9Hi58xsfvT4fEk7h23r7mvdfZm7L5uxH2d0AW2rajzPnXNwMwEDY6zM\nbG2TdLWkR939ithD6yWdE90+R9KtxcNDkiwzs6v6xqm06pkKuvsYz0BYypSsH5B0lqSHzKx/8PGz\nkr4o6WYzO1/SU5LOKBcigAYwnoGAFE7O7n6PJEt4+OSizwugeYxnICxcIQwAgMCQnAEACAzJuYPq\nPH1q1HNyWhUANINzmDqkjaQ8bB9cnAQA6kXlDABAYEjOAAAEZuzb2pvPfVCSdNS172k5knLKfhVk\nlfr7o70NAPWgcgYAIDDBVs6bz32wdLXbr5rHwdTnrtPM1We3HUZuVXYuRv0+054/7f9C1zsryG/Z\nxbO18YpdbYcBDBVscq7COPzBDWGG9qh9D2ttT33uukr3kyXBj1onywe9Kj4MonuWXTxbkhpP0l38\nYFB1zP33fpi0/YzaNsv2XUBbGwCAwIx15dx1IU0CSxKfHNavmOPt96qraKAq41BdNSGtSi36fEnv\nf9r+RlXwVcfapqCS8+Bxwfj9eMtxWAszz7pp+40b3C5t30nb5dlv0fbq+jtPHbr81JPXj1w/6fG8\nunhMHMBwg0m0qcS38YpdhQ83xGPt+ocv2toAAAQmqMp5sBrNU30WrTaH7SfrLO+sVfSgfqs3abu8\nk8BGVcBJ1XTS8qJGXdZT6r2mJs57LjOxa1zOiZ9kSdXdsCoqqboatjz+vHlaqmWqtzzPV0elWEfl\nWSbOtibvtYXKGQCAwARVOedVV5V01LXvSa2A0/ad9Pjmcx/cfWz2nYv33mc8rqzW33lq6jHjYZV1\nVceZQ1H0/YtvP07nxU+aUVVVmWOlWaq1YetUvc/+siyToSalshx3nU7OVbayq9p3UpLY+8NAsa9X\nTJqhXXWLuigu64k2ZGmV5k1cWddP2nfRiUlZnm8YEvL4oa0NAEBgOl05j5O0SWCjhNaizlJBV109\n1zExEOOhSFVZVSWet1IfvI3JRXIGgEDQnkYfbW0AAAJTOjmb2Qwz+5GZ3RbdP9zMNpjZFjO7ycyC\n69H0Z+Y2OTv3qGvf09qM4PV3njp00ljS8raMumLaqPeu35IeNUkv6ffdX05Lu6eL4xndsuzi2ZXM\nZh93VbS1PyHpUUlvie5/SdKV7n6jmV0l6XxJXx31BP/frjcNPeYa/2M67NKQeY7TJq2bliyTtsuy\n76R1il5vOn4Mt39c99ST16cm2DwXJ6nSqOt/x9+bPB9Yyn644XSpVKXHc4jyztZ+/0UfHr2C935M\nX3yXZvhJiatN212Z9pl1dneeC6e0ZdR7Hb8056jti8xYT9t315SqnM1soaTfl/T16L5J+pCkfhZZ\nJ+m0MvsA0AzGMxAOc/fiG5vdIukLkvaX9OeSPibpXnf/zejxRZL+3t3fNWTbVZJWSZIOmHPMjE/8\nYevnv6ZVw1XFN2o/U5+7bneXIKnyvP2EW4L5Vqo0oy7p2df27z1Uj5z30CZ3X9bU/qoaz2+be+gx\n/+s/PV5rrIOV7bTdNXL9YdXttA2vepOWD1Nkv2n7zvp8G668Y/ftLJcXLStrdZ41lkn6Puezr/wd\nPfLMJsuzTeG2tpmdImmnu28ysw/m3d7d10paK0n29nnFPyFUIGt7vKnrQ4+TtNOqEIYqx/OSRcc0\nMp7jyWmjRv8x3qA79lm2Ubuki/ZdN2tizrvu4Hbx+PeKaYQNumNoy72JZJR1H1WvV/W2XVHmmPMH\nJJ1qZisk/YZ6x6i+LOlAM5vp7lOSFkraXj5MADVjPAMBKZyc3f1SSZdKUvRJ+8/d/Y/N7K8lnS7p\nRknnSLq1gjhrU+biH0XFJ4T1W9jxZV1pWecx6luruKxn+8ZlPKdJnejVoH4swypooI7znD8j6WIz\n2yJpnqSra9gHgGYwnoEWVHKFMHf/gaQfRLeflHRcFc8borJV3mNbj96rWu5XzGmTwMZBG5f1RH7j\nOJ5DqpgHvf+iD1M9Yx9cvrNFw87dBlCtvIl509tvq3T/x/zLKanrxGMkUUPi8p0AAARnYivntIlg\nv/3590qSHr3sx4nb04IFwtV2xTz4vFkqaImJYuiZyOScNTG3Iev5wJw3jEmXdKnGPEm5roScZV9p\nyZokPdloawMAEJiJrJzz+O3Pv3dka1vKNnP7Hf/vf+5tQ8WLCVfFlzQkXeYyrWpuslJO04/l6B37\nj7zSWKizuav8kolJupRnViRnAJ0zePnLLiXlIkJpcVf9dY1ZEvyodbJ80OvqN1XR1gYAIDATVTkX\nnQhWduZ2v6Utjb6M5aRbcc/pUnShkif+r//UcjToinGomvPO6G7DYJVadRWNvU1Mch6VmLPOzi56\n/Hkw0bwzQ0yTpP+ePcGpaWNr8A950tcKDmtBpq2rlO/AOnrH/kOXPzD/1aHrxZcP23ZwuzL7HLTp\n7bcNTdC7j7FfLE1rz+2kVm0drdyutYW7jrY2AACBmZjKOURlrs/NBVDQJYMVb9YqrMzM7n61mVYJ\nD5O1iq5qu1Gm7a6hs7mTZqyPa7u5bDegirMEmjT2ybnqC45kOf4s8fWHQBll/hAPS8p9D8x/NTVZ\nHr1j/5Ht56THk5b3l2VN0vFTrCSNPM1q2PvUpQSURf/1FP3QUXb7ttDWBgAgMGNfOQPonjKt7CwT\ntkZJ2j6pAh7Wyq5SUvsa443KGQCAwIx15VzF6VOjtk867tzfN8edgWrl/aapcTDqmHPbVwxrQhUT\nwcps35axTM4hf+sUAABpaGsDABCYsaycm8JpVUBY4qdK1TVBa9g+pfRTsOqQ1LINrZWbFE/8NKci\nX2yx8YpdnXkP8iI5A6jN8a/8yT7LpnTdXueczlx99l6PJW2XxbJVF+x+jrTzipetumCfuIYtHzSl\n64auM6Xrcu9z1PNpde/H9OeSZ2v3j0dv0J5jzyGdz5snlrJxh/S6q0BbGwCAwJh7ylXjR21sdqCk\nr0t6l3qXnz9P0mOSbpK0WNJWSWe4+0sjn+ft83zGv//9Stq/dc7QHmXUzG2p2tY2M8EnwyPnPbTJ\n3Zc1tb+qxvOSRcf4dRf9k6Q9FfC9B3y9khi7/A1Uedve/Wp649qvpH5b1STM2u6ys6/8HT3yzCbL\ns03ZyvnLkr7t7v+HpPdKelTSJZLudPcjJd0Z3a/dY1uPbi0x959/1D7S4gMCEMx4BiZd4eRsZgdI\nOlHS1ZLk7rvc/WVJKyWti1ZbJ+m0skECqBfjGQhLmQlhh0v6qaRrzey9kjZJ+oSkQ9x9R7TOs5IO\nGbaxma2StEqSdMCcEmEAqEBl4/ltcw+tP9pADU4Iy/JFG8AwZZLzTElLJf2pu28wsy9roOXl7m5m\nQw9qu/taSWul3jHnokGEdsGRLKdXcbwYAapsPC9ZdEzxiSwdN3j97TZOr8J4KHPMeZukbe6+Ibp/\ni3qD+zkzmy9J0c+d5UIE0ADGMxCQwsnZ3Z+V9IyZvTNadLKkRyStl3ROtOwcSbeWirCjmByGLmE8\nV+uB+a/u/gcUUfYiJH8q6QYzmy3pSUnnqpfwbzaz8yU9JemMkvtIFFpLG+i4VsczgD1KJWd3f0DS\nsPMwTy7zvACax3gGwtHJy3dSMQMAxhmX7wQAIDAk5xpluWoYAIRs2cWzx+5LJbqA5NwAZm4D6Lo2\nEnQXPxRUFTPJGQCAwHRuQhiTwQBMklHfSNXEt1FtvGJX7fsYB1VX+Z1Jzl1Pylku6ylV+9WSwLjp\nJ6Okr4485l9OCfprIzGell08e/eHGNraAACMqc5UzgCAaiRVd8Na2P11Bx8btjz+vEnt8GH7LtM6\nz/N8Sa+lrDpa/yTnhv3259+b2NqW+NYqoKz+MVra28PFW7CDy8s8p5SepAb3XbYFnPR8Sa8xabsQ\n0dYGACAwnaicR00GC30iGAAU4WueGvl43pnaWarbIm3fLFVo0vPGJ1Hl3Wfa8w0TerUcF3Ry7voM\n7STM3AbKyzJzW+pee3vUqVN1Kpq4qkj2RRLz4O1xQ1sbAIDABF05jzsmhwHVszWHSUpvC4corWre\ncOUdiZ2CSdKl9nRRwSbnSTnOTIsbKCeesPqJOX47nvBCbXG31cpGuGhrAwAQmOAq53GdBAagPvGK\nedhj/RZ3iJf3zFI1N3ENbanYbO2RbXbv/Zi++C5J0gw/aehq03bXXo+Ner0br9iVGmfaudxdaItT\nOQMAEJjgKmcAyOP4V/4kdZ34JLEQTrHKc4y5TNW8V1U7UMUOs7t6vWjPsmndte9zxZbnMW3Dtxms\nqPv7mtZdQyvz/r6TTqWa4Sft9RqGbRd69UxyDkSWiWFMCgP2yJKUB8Xb301NFCs62Wt3rAeUjyGe\n4DcqOSlt0L4fBGZcNLwVndSiLrtu0nZZX4M0/HW8/6IPN5KQq9oHbW0AAAJTqnI2s4sk/Yl6DZOH\nJJ0rab6kGyXNk7RJ0lnunumjxKScPjXKqHOf0ybLAWVUPZ7rNGoC2N9d8YeSpN+/+FupzxGfKNZX\nZRVdpGruV4jHK39noEqhnU8dj6epCXJtKpyczWyBpD+TtMTdf2lmN0s6U9IKSVe6+41mdpWk8yV9\ntUyQk5KYgbY0OZ7LSGtl9xNz/PaoJD0syS9bdcHu201dyCS0ZBNaYh7Ujy+0961KZdvaMyW9ycxm\nSnqzpB2SPiTplujxdZJOK7kPAM1gPAOBKFw5u/t2M/tvkp6W9EtJ31Wv7fWyu09Fq22TtGDY9ma2\nStIqSdIBc4qGAaACVY7nt809tP6Ac/i7K/4wtcWdZFT7XKqmsg6p+stTMc9627W1xfH6s+dmWm+c\nW91l2tpzJa2UdLiklyX9taTlWbd397WS1kqSvX2eD1tnUtvZaTO3gapVOZ6XLDpm6HhuU9bj0HkN\nJu+syTrERBJKYo4/f9YkLfXiD/F9LapMW/v3JP2zu//U3V+X9C1JH5B0YNQWk6SFkraXjBFA/RjP\nQEDKzNZ+WtLxZvZm9dpgJ0vaKOn7kk5Xb4bnOZJuLRskgNoFP55HTQaLTwQbJb5e1VW0NPwbsUKv\n5tIq5rqr5Kz7zlJFj9NEsTLHnDeY2S2S7pc0JelH6rW1/k7SjWb2n6NlV1cR6CTKcloVFyZBFUIe\nz2kztH3NU1qx5spCz512TLmoeJIuOvO5/xx1zpwe9dxtJuVh8iTqcWhxlzrP2d1XS1o9sPhJSceV\neV4AzWM8A+Hg8p2B47KemGRZLtF5+1uGXES5ImVb37bmMN17wNcLbdu/CEkdFWDIrewsskwY63qL\nm+QMIDhZknLW48xlpO0jS/Luv5aiSbpqXWplp8nS6g79gipJuLY2AACBoXIG0DlNVM1Z1D37u0pd\nb2VPGipnAAACQ3IGgAqkVfNFvn+6KmnHmcehah6H1xBHWxtAMNpMYFVIu0xoaJPDEC4qZwAAAkPl\nPAZ+7d+fI0l642vrWo4EKC6U06cmybi1gtPOf+7Suc9BJ2e+lQlAH4k5O2Zmdx9tbQAAAhN05Yx8\naG+ji9La2V2smLNMDGNSGEahcgaAmoz6YHH8K3/S+dnpqA/JGQCAwJCcO67/rVVx/fY2ELpxbGmj\nfWkT3rrwZRhBHHP+jdm/1BGLH9BjW49uO5TO6H9N5Btf4+sigZBx/BlFUDkDABCYICrnvn41CGC8\n0c4GRqNyBoAGMHMbeZCcAQAITFBtbQDjb1SFOO7tbCaHISsqZwAAApOanM3sGjPbaWYPx5YdZGZ3\nmNnj0c+50XIzs78wsy1m9qCZLa0zeAD5MJ6BbshSOX9D0vKBZZdIutPdj5R0Z3Rfkj4i6cjo3ypJ\nX60mTAAV+YZaGs9pk57GvaUdx+QwpElNzu5+t6QXBxavlNT/doV1kk6LLb/Oe+6VdKCZza8qWADl\nMJ6Bbig6IewQd98R3X5W0iHR7QWSnomtty1atkMDzGyVep/GNWverIJhAKhApeP5bXMP3WcHVMxA\nPqUnhLm7S/IC261192XuvmzGfkwaB0JQxXieO+fgGiIbP393xR/ywQSJiibn5/rtrejnzmj5dkmL\nYustjJYBCBfjGQhM0eS8XlL/q4/OkXRrbPnZ0SzP4yW9EmuXAQhTa+OZyhEYLsupVN+U9E+S3mlm\n28zsfElflPRhM3tc0u9F9yXpdklPStoi6WuS/u9aogZQCOM5PGkfUJi5nd/rz5478vENV97RUCTF\npR7sdfePJjx08pB1XdIFZYMCUA/GM9ANzMQCUAtmaGeX5bKekri05wTh8p0AKvXo9FYSM1ASyRkA\ngMCQnAFgzGy48o6Rk55ef/bc1ElTaBfJGQCAwDAhDAAwFsbhFKo+KmcA6IgXVt9e6fPR2g4XyRkA\ngMCQnAFgzE3C5LBxeA1xHHMG0KikC20gm35re97nVrQcCepE5QwAQGBIzgAwAbKc+9xFaW35tNcd\nKpIzAHRQ0Znb43T8eRyTch/JGQCAwDAhDEClfnvGYl13wNf5HuIK7f42qs81s794RTrrbdc2s9OM\nslT2Xa6Y+0jOAGpxLwm6EnV8TWQ/eb3/og+nrttPhm0l6Txt9nFIyn20tQEACAyVM4Da1FH1oTob\nrrwjU/UstdPqntSqWSI5A8BEiye1IolaqiZZF50lPm5JuY+2NgAAgaFyBgBIyjdRLK6tc6PHtWqW\nqJwBAAhOauVsZtdIOkXSTnd/V7Tsckl/IGmXpCcknevuL0ePXSrpfEnTkv7M3b9TU+wAcmI8I4si\nx6GbMs7VclyWtvY3JP2lpOtiy+6QdKm7T5nZlyRdKukzZrZE0pmSjpL0dkn/YGa/5e7T1YYNoKBv\niPGMHIq2uqve/6RJbWu7+92SXhxY9l13n4ru3itpYXR7paQb3f1X7v7PkrZIOq7CeAGUwHgGuqGK\nCWHnSbopur1AvcHdty1atg8zWyVplSTNmjergjAAVKD0eH7b3EPrjA8FhNaazqPLsZdRKjmb2WWS\npiTdkHdbd18raa0kvWnxm71MHADKq2o8L1l0DOM5EE21hKtIoJPavk5SODmb2cfUm1hysrv3B+N2\nSYtiqy2MlgEIGOMZCEuh5GxmyyV9WtJJ7v6L2EPrJf2VmV2h3gSSIyX9sHSUAGrDeEZZVL3Vy3Iq\n1TclfVDSwWa2TdJq9WZz/rqkO8xMku5194+7+2Yzu1nSI+q1xy5gZicQDsYz0A2pydndPzpk8dUj\n1v+8pM+XCQpAPRjPQDdwhTAAAAJDcgYAIDAkZwAAAkNyBgAgMCRnAAACQ3IGACAwtudiQC0GYfZT\nSa9Jer7tWBIcrDBjCzUuKdzYQo1L2je2w9z9rW0FU5SZvSrpsbbjSNCl338oQo1L6k5sucdyEMlZ\nksxso7svazuOYUKNLdS4pHBjCzUuKezY8gj5dRBbfqHGJY13bLS1AQAIDMkZAIDAhJSc17YdwAih\nxhZqXFK4sYUalxR2bHmE/DqILb9Q45LGOLZgjjkDAICekCpnAACgAJKzmS03s8fMbIuZXdJyLIvM\n7Ptm9oiZbTazT0TL15jZdjN7IPq3oqX4tprZQ1EMG6NlB5nZHWb2ePRzbsMxvTP2vjxgZj8zs0+2\n9Z6Z2TVmttPMHo4tG/oeWc9fRP/3HjSzpS3EdrmZ/e9o/39rZgdGyxeb2S9j799VdcZWlVDGM2O5\ncFyM5+JxVTuW3b21f5JmSHpC0hGSZkv6saQlLcYzX9LS6Pb+kn4iaYmkNZL+vM33Koppq6SDB5b9\nV0mXRLcvkfSlln+fz0o6rK33TNKJkpZKejjtPZK0QtLfSzJJx0va0EJs/1rSzOj2l2KxLY6v14V/\nIY1nxnJlv0/Gc/a4Kh3LbVfOx0na4u5PuvsuSTdKWtlWMO6+w93vj26/KulRSQvaiiejlZLWRbfX\nSTqtxVhOlvSEuz/VVgDufrekFwcWJ71HKyVd5z33SjrQzOY3GZu7f9fdp6K790paWNf+GxDMeGYs\nV4LxnCOuqsdy28l5gaRnYve3KZABZGaLJb1P0oZo0YVRu+KaNtpNEZf0XTPbZGaromWHuPuO6Paz\nkg5pJzRJ0pmSvhm7H8J7JiW/R6H9/ztPvU/+fYeb2Y/M7C4z+1dtBZVDaO+nJMZyCYzn4kqP5baT\nc5DMbD9JfyPpk+7+M0lflfQOSUdL2iHpv7cU2gnuvlTSRyRdYGYnxh/0Xg+llen3ZjZb0qmS/jpa\nFMp7tpc236NRzOwySVOSbogW7ZB0qLu/T9LFkv7KzN7SVnxdxVguhvFcXFVjue3kvF3Sotj9hdGy\n1pjZLPUG8w3u/i1Jcvfn3H3a3d+Q9DX12neNc/ft0c+dkv42iuO5fusm+rmzjdjU+yNzv7s/F8UY\nxHsWSXqPgvj/Z2Yfk3SKpD+O/tjI3X/l7i9Etzepdyz3t5qOLacg3s8+xnIpjOcCqhzLbSfn+yQd\naWaHR5/UzpS0vq1gzMwkXS3pUXe/IrY8ftzi30h6eHDbBmKbY2b792+rN/ngYfXer3Oi1c6RdGvT\nsUU+qlgLLIT3LCbpPVov6exolufxkl6JtcsaYWbLJX1a0qnu/ovY8rea2Yzo9hGSjpT0ZJOxFRDM\neGYsl8Z4zqnysVzXbLas/9SbYfcT9T5NXNZyLCeo1yJ5UNID0b8Vkq6X9FC0fL2k+S3EdoR6s19/\nLGlz/72SNE/SnZIel/QPkg5qIbY5kl6QdEBsWSvvmXp/UHZIel29Y07nJ71H6s3q/Er0f+8hScta\niG2LesfJ+v/frorW/bfR7/kBSfdL+oOmf68FX2MQ45mxXCo+xnOxuCody1whDACAwLTd1gYAAANI\nzgAABIbkDABAYEjOAAAEhuQMAEBgSM4AAASG5AwAQGBIzgAABIbkDABAYEjOAAAEhuQMAEBgSM4A\nAASG5AwAQGBIzgAABIbkDABAYEjOAAAEhuQMAEBgSM4AAASG5AwAQGBIzgAABIbkDABAYGpLzma2\n3MweM7MtZnZJXfsBUC/GMtA8c/fqn9RshqSfSPqwpG2S7pP0UXd/pPKdAagNYxlox8yanvc4SVvc\n/UlJMrMbJa2UNHRAHzRnP1904EE1hQJ004P/8szz7v7WlsPINZYlaabN9F/XrIbCA8L3K72uKZ+y\nPNvUlZwXSHomdn+bpPfHVzCzVZJWSdKCA+bq9gs+U1MoQDctvOzCp9qOQRnGsrT3eJ6tWVoy44hm\nogM64JHpJ3Nv09qEMHdf6+7L3H3ZvDn7tRUGgArEx/NMzWg7HKDz6krO2yUtit1fGC0D0C2MZaAF\ndSXn+yQdaWaHm9lsSWdKWl/TvgDUh7EMtKCWY87uPmVmF0r6jqQZkq5x98117AtAfboylu+b6oV0\n7MyjWo4EqEZdE8Lk7rdLur33kgEDAAAbUklEQVSu5wfQjC6N5fumNjeeoLv4waDqmPvPlyRtP6O2\n79L7WiWuEAYAQGBqq5wBoCmTWl0VkVblFnm+pPc/bV9pFXzVsXYJyRkAJkA8ETaV9PpJt2gbPR7r\npH0Ao60NAEBgqJwBBCupwhusokZVV4Nt1/hz5mmnlq3cBp8zbd9VV4pVP1/ZONuYvNclJGcAwUn6\nw1+2HZslIQxbp8x+h72W+6Y2Zz7eSgKbTLS1AQAIDJUzgGBkrRbztkSzPO+odYpOTEqKc9SkLCpl\nugYSyRlAhxT9Y11Fsi/yYWDwNvJp470L5YMBbW0AAAJD5QwANWm7+uqqpNn1k4TKGQCAwJCcAQCF\nlT3NbFIr4zS0tQF0TtUXsMgyGzvPRKH4ZSvzPl8oE5L6kuIZvDRnlm2zJuJRF4opclGZYftu8qIy\ny5Yty/38VM4AAASGyhlAq+5//eDdt2f4SZKk6ZQKa4afJMW2G/ZcWZYPk1Td9WMbtt9R+yj6fP3t\ndq8XWTrr+aHr163IOdvxbfvrDltetLWd9Ypvg/tu44pveZGcAQQhnnSO1eg/cEsH7vfXH1wuSTNe\nP2nI0iHrebb18mw36g/1sFilfV97fL08HzJGyZNA8rTxBxU5FJBlu65dVObR6Scz76OPtjYAAIGh\ncgYwNqqqLKsSj6etdnTIuOJbMpIzgM4LLSkPc//rB5OgO67JGfS0tQEACAzJGUCndaFq7rv/9YM7\nFS/aU7itbWaLJF0n6RBJLmmtu3/ZzA6SdJOkxZK2SjrD3V8qHyqAunRxPBdJcidc/6EaItnjnrO+\nl2k9Wtx7K3pRmaT/AzP8pN2n4yXNpp+2u0Y+Hlf2ojJNX4RkStJ/dPclko6XdIGZLZF0iaQ73f1I\nSXdG9wGEjfEMBKRw5ezuOyTtiG6/amaPSlogaaWkD0arrZP0A0mfKRUlgFp1bTznqZrrrpZH7WtU\nJT2uM7lDu6hMv0Ieus8Rqr6oTF6VzNY2s8WS3idpg6RDooEuSc+q1yYbts0qSaskacEBc6sIA0AF\nyo7n2ZpVW2xpf5SbTMRZxOPJkqjHKUlXdVGZwd951ovFhHJRmRmvn6Q39FjuOEpPCDOz/ST9jaRP\nuvvP4o+5u6t3/Gof7r7W3Ze5+7J5c/YrGwaAClQxnmdqRgORAuOtVHI2s1nqDeQb3P1b0eLnzGx+\n9Ph8STvLhQigCaGP51FV8wnXfyi4qnlQlviYyb230N6PJuMpnJzNzCRdLelRd78i9tB6SedEt8+R\ndGvx8AA0gfEMhKXMMecPSDpL0kNm9kC07LOSvijpZjM7X9JTks4oFyKABgQ7nrt2nHmUfqyTdvw5\nj9Cq5UFNTeQrM1v7HkmW8PDJRZ8XQPNCHc9preyuOuH6D6WeEz1pSbpIUl788bsrj2PrVSdmXrfO\n3xFXCAMAIDB88QWA4IxTKztJlhb3pMhTNddRLQ97/rwVdNXVM8kZQDAmISkPSmtxj/OlPrMk5bqT\ncZ79jkrYVbe4aWsDABAYkjMAtCztPO1x/DarkKvmJFniqer3RFsbQBAmsaU9idJ+z6El5EFZjklX\n0eKmcgYAIDAkZwBAI0ZVzYs/fnfwVXNc3bHS1gYQtElqZ2e5OEkXdb2VnSStxV3m+DOVMwAAgaFy\nBgDUYlwr5kGLP353rouWZEHlDAABmcTTqsZB1cfMSc4AAASG5AwAqNyktLTrQnIGACAwJGcAreGq\nYBg3VXUEmK0NAGgM7exsqJwBAAgMyRkAgMCQnAEACAzJGQCAwJROzmY2w8x+ZGa3RfcPN7MNZrbF\nzG4ys9nlwwTQBMYzEIYqKudPSHo0dv9Lkq5099+U9JKk8yvYB4BmMJ6BAJRKzma2UNLvS/p6dN8k\nfUjSLdEq6ySdVmYfAJrBeAbCUbZy/h+SPi3pjej+PEkvu/tUdH+bpAUl9wGgGYxnlMIlO6tTODmb\n2SmSdrr7poLbrzKzjWa28YXXfl40DAAVqHI8T2m64uiAyVPmCmEfkHSqma2Q9BuS3iLpy5IONLOZ\n0afthZK2D9vY3ddKWitJ711wqJeIA0B5lY3nOfYmxjNQUuHK2d0vdfeF7r5Y0pmSvufufyzp+5JO\nj1Y7R9KtpaMEUCvGMxCWOs5z/oyki81si3rHrK6uYR8AmsF4BlpQyRdfuPsPJP0guv2kpOOqeF4A\nzWtyPC+d9fzISUT3nPU9vpkKE4krhAEAEBiSMwAAgSE5AwAQmEqOOQMAqnHPWd8b+fjSWc83FAna\nROUMAEBgqJzRuAWfvWDo8u3/5SuVbttfd9TzZlknz36HPV9826T9DH3+yy4cGdOk6FeSVc3avuvs\n1ZKkk677XCXPhz3SZt9vvepELuGZEckZjUlLhAs+e0FqIk1KenVJS7ZZts+S+Jt+Xegl6aYTdBc/\nGNw3tVmSdOzMoyp5vif+5+gLyL3jP1jh7dO27RLa2gAABIbKGcFLqrj79+uqNJOq3u3/5Sup+yzT\nUs/S3h9HTV2QJOSqNW0yWJP6FXOVpu2uxOo2raLuP150+64hOaNxg4mtn4ySjhs3nayyHoceJc8x\n7MREzzHniRFSUpZ6ibnfxq4jSQ/TT7pP/E8v1J5+x3+w1ATeJbS1AQAIDJUzGpNlpvIktXQn6bVm\n1T+HN6m9nTRzuz/RapjBNnbSpKxhy+PPm9QOH7bvPK3zwap52u7aZ51+FTv4vlQ9WWtwf1Xpv6YZ\nfpIW59x2nKrhPEjOE6yK9m0Ri3ZdvveCNb37b6zZus9jb+iCfdcfkLTOG7pg+P5S1kna7pnZn9q9\nfNg6/cdRXpYk3U/QSbOuRyXsNFlnVQ/uu8w+pV4Sm+En7bOsn4QHH+uLt6ExHmhrAwAQGCpnNDLp\natGuy/XGmq29O2v23P61NYtHbvdraxZnXjev3fEM2Wf/8fg+d7ff1wyvxOMVdZosM76ZEDZ6BnfR\n6jZNlvWT9t2/nyW2eDs73vbt63cPjtVRiZOyulgtb73qREka24uR9F9fWVTOAAAEhsp5gjVxrHno\nseBYxZpUvQ6TZ11p7wo46fG05xz2eJY40o5FcxWw7PIcf44rej5zFZV42uPDqubB25oa+RTB41Ke\n5ZCcUZvBxBxvEedpVaetk5YsR22f9lg8ziwx52m9M1u7OvFkV9U1uMtIPm95+ISxfjs79RunRiQ7\ntKuqdnYfbW0AAAJD5TyGynyDUvyxst+s9IYuKD2Ja7AqTnq+uiaNVfF8/Q7CM7M/1drpa+MgrU3a\nV/W3WGVV5ipf4/odzbtfV9SiHzxVLM/ksJDPd666apZIzmMl6Q9/2eObRb5ZKet+XzrulKHLD1jx\nl5KkV26/cK9l/SQcXx73xpqtiY9VJSnmuT+8rdb9Iv34c1w/WU7bXYWPPw9LuDP8e5o+e9/Z1XHD\nZl8nOXbmUakXE0k6z7mui5AUdd/U5qGvOe1SoFuvOnH3ezaYfOOX9Ryl6GU/Q0VbGwCAwJSqnM3s\nQElfl/QuSS7pPEmPSbpJ0mJJWyWd4e4vlYoSI2VtleY9n7nMNytJxc5RHlYx971y+4W7Hx/2GMrp\nwniOV8z9Cm3Y5S7jZvhJ0lnxJb1JWftWxUnLkyXtO0vF3Bd/TUmVZfz5hnUNkirWNvTfk2O1bzV/\n7MyjNF3wizSyVtBNSmtnZz0UM0zZtvaXJX3b3U83s9mS3izps5LudPcvmtklki6R9JmS+0EJRY9v\nVpHs8xyzjSfepCQ8DIm5Mp0Zz/FjtMfqqFx/AJOSWJ7kVjQRVrHvwde+e3mhiJL139M8LfNjZ+75\nXSTFk3Sd8N1STrEKpXU9KjFXMYegcFvbzA6QdKKkqyXJ3Xe5+8uSVkpaF622TtJpZYMEUC/GMxCW\nMpXz4ZJ+KulaM3uvpE2SPiHpEHffEa3zrKRDyoWISUIV3JpOj+d4pVK0jdgV8dfX5Vneo1q+8ao0\npAuVZJmVXdXvpMyEsJnqdS6+6u7vk/Saei2v3dzd1Tt2tQ8zW2VmG81s4wuv/bxEGED3BHiFsMrG\n85Smaw92lKWznu900srj/tcP7vSHkSy/q61XnVjLqUp5ZImh6v93ZZLzNknb3H1DdP8W9Qb3c2Y2\nX5KinzuHbezua919mbsvmzdnvxJhAKhAZeN5pmY0EjAwzgq3td39WTN7xsze6e6PSTpZ0iPRv3Mk\nfTH6eWslkaK0vLO1D7305tEr+M2a/uyI8zk/l+/iIPHZ2Ent7aTH07ZrWtLr7r//oyrn3Y81+K1U\nIYznPBVgl6vFOnS91Z1lVvOwyrXqlnfRCr2O97zsbO0/lXRDNLPzSUnnqleN32xm50t6StIZJfeB\nIfZKnN673U+USWb4SdKle+5PR19vOCwJT2f86sPd6w87pWRNelKe+8PbZKsPi+23J2m29u4PAR/Z\ns+zFv3/37tsHrPjLYBK0tO/XTvZl+srI5rUynvP8Yds9E7jgH8OiSf13z7qp0HZV+cfr/yjzul1N\n1HkuMNPXdru7zve3VHJ29wckLRvy0MllnhdA8xjPQDi4fGfHPf2FPYXMdo1uWT89UPTsXv/SfdfN\net5l0fM949Wkf+4pSb3LYr6i0VXvi9pTJR/0kYd2326iWs6zjyxtfK6v3ZyuVsx9/TjyVNBS+U5D\nG8pcuKNJdb+nXL4TAIDAUDlPqNTJXhXpH0/uV8ejzP3hbYlfKgEUlbUKC6VKHmUwxqyV9P2vH9x4\n9dyF6reMul8fyXnCNJWUB9nqwzIn6LguJeuXjjtl5DdTLdp1uZ6Z/akGI5psWf54diEhj5Kn3d1U\ni7uu528z2Zd9TY9MT+XehrY2AACBoXKeIHmq5m+/+7D0lTJY/tCeajl+ylSWKlqi1Y38xqmNnVX8\ntaRV0W20uKswLOaqq+mQ3heS85jL28auKin39ZPwd245sXCi7reKSdJIk/bHepwScpIsibqLs7iH\n6Xr8o9DWBgAgMFTOY6qNFnaR/Sx/6KnMM7ppcSMJFfNwaRPGutringRUzmMoS2L+9rsP2/2vTfH9\nx1vdSeb+8LaRM6IxeUjMGEckZwAAAkNbe4xkrZhD048pT4sboGLO7nfPumnsJ4eNG5LzBAkxMcfF\n41seXTY7KUnP/eFtso+E/XqAkHD8uVtoawMAEBgq5zGQ1s4OvWIGAOyN5Nxx45qY9xyHTm5t715+\n3LuHPo7xNep4M8eaR+P4czfQ1gYAIDAkZwTPVh828hzoUM57TouDb6SqH1UzxgXJGQCAwJCcx1QI\nV/+qwji8BiA0v3vWTSO7DG1+dzJ6mBAGoBOYBIZJQuUMAEBgSiVnM7vIzDab2cNm9k0z+w0zO9zM\nNpjZFjO7ycxmVxUsJlvapLBQJoZ1FeN58tBxCFfh5GxmCyT9maRl7v4uSTMknSnpS5KudPfflPSS\npPOrCBSTa1yOn4eM8QyEpWxbe6akN5nZTElvlrRD0ock3RI9vk7SaSX3AaAZjGcgEIWTs7tvl/Tf\nJD2t3iB+RdImSS+7+1S02jZJC4Ztb2arzGyjmW184bWfFw1jIo3rVcHQnirH85SmmwgZGGtl2tpz\nJa2UdLikt0uaI2l51u3dfa27L3P3ZfPm7Fc0DAAVqHI8z9SMmqIEJkeZtvbvSfpnd/+pu78u6VuS\nPiDpwKgtJkkLJW0vGSOA+jGegYCUSc5PSzrezN5sZibpZEmPSPq+pNOjdc6RdGu5EAE0gPE8obJc\nkISLkjSvzDHnDepNFLlf0kPRc62V9BlJF5vZFknzJF1dQZwAahTyeE5LDpwOhHFU6gph7r5a0uqB\nxU9KOq7M8wJoHuMZCAdXCAMApHYgaG03i2trAwgW7WxMKipnAAACQ3IGACAwJGcAAAJDcgYAIDAk\n5w56+gtnjHx8+UNPNRRJM5Y/9NTYvSYAGIXkDABAYEjOAAAEhuQMAEBgSM4AAASGK4RhrMz94W2N\n7u+l405pfJ8Axh+VM1ACiRnj4h+v/6ORjy+d9XxDkUAiOQMAEByS85gal3ODx+E1oLhR1do/Xv9H\nqdUe0FUk546btAuS2OrDZKsPazuMXJ6Z/Sk9M/tTbYcBoENIzgAABIbZ2mjE/3n63bnW71p1DABV\nIjmPgX5r+9BLbx76eL+1/e13N5/wvnPLibnWT2vD727jz97zetNa+3VatOvyvVrWi3ZdLkm0sSvU\nP+58/+sHD338H6//I/3uWTc1GdJYYZZ2mGhrAwAQGCrnMZKlgm6jeh5ngxUyFTOAKlA5AwAQmNTk\nbGbXmNlOM3s4tuwgM7vDzB6Pfs6NlpuZ/YWZbTGzB81saZ3BY7hRx2BDPP+5H1Pm480ojPEMdEOW\nyvkbkpYPLLtE0p3ufqSkO6P7kvQRSUdG/1ZJ+mo1YSKvp79wRqYk3VaizrP/tNeCXL4hxjMQvNTk\n7O53S3pxYPFKSeui2+sknRZbfp333CvpQDObX1WwAMphPAPdUPSY8yHuviO6/aykQ6LbCyQ9E1tv\nW7RsH2a2ysw2mtnGF177ecEwUIWmq+c8+6NibkSl43lK07UEyaU8q8d7Fq7Ss7Xd3c3MC2y3VtJa\nSXrvgkNzb49snv7CGYmzt+NGJcyiM7zLJH2ScjuqGM9z7E2tjWfOec4my7nNSeeVoxlFK+fn+u2t\n6OfOaPl2SYti6y2MlgEIF+MZCEzRynm9pHMkfTH6eWts+YVmdqOk90t6JdYuQ0vSzn9O02Tbm4q5\nFYxnIDCpydnMvinpg5IONrNtklarN4hvNrPzJT0lqf8X9XZJKyRtkfQLSefWEDMKiie+oom6DiTk\n5jCeMaqlzaU6w5GanN39owkPnTxkXZd0QdmgANSD8Qx0A5fvnFBtV9FUyyhj1ISlfmXIxDB0Gcm5\n40JqT+fR1bgRjizfViWRpPv49qlu4draAAAEhsq5o5psCxetcmldowlp5+Ry7jOTwLqIyhkAgMBQ\nOSMVFTBCx/Hn4TjO3F1UzgDGRlqymZRrSadda3zprOdJzIEjOQMAEBja2gDGyqS2uLN2BaiYu4HK\nGcBYSmvdjlOLO8troZXdLSRnAAACQ1sbwFjLcqnPuK60u/NU/lTM3UNyBjD20o5Dx4V+0RKS8mSg\nrQ0AQGConAFMjLRLffaF1O4uMnGNirn7SM4AJkqeFndcPEnWmaiLziInIY8X2toAAASGyhnARMra\n4h4mtHOkqZrHD8kZwMQaltSKJuymkZDHG21tAAACQ+UMoHO6Ut3WifdgvFE5AwAQmNTK2cyukXSK\npJ3u/q5o2eWS/kDSLklPSDrX3V+OHrtU0vmSpiX9mbt/p6bYAeTU9fHc5nHWtipVji1PpiyV8zck\nLR9Ydoekd7n7eyT9RNKlkmRmSySdKemoaJv/x8xmVBYtgLK+IcZzIf1vdao7Wcb3Q2KeXKnJ2d3v\nlvTiwLLvuvtUdPdeSQuj2ysl3ejuv3L3f5a0RdJxFcYLoATGM9ANVUwIO09S/3I5C9Qb3H3bomX7\nMLNVklZJ0oID5lYQBoAKlB7PszWrzviCQEWLupWaEGZml0maknRD3m3dfa27L3P3ZfPm7FcmDAAV\nqGo8z9TEdr6ByhSunM3sY+pNLDnZ3T1avF3SothqC6NlAALGeAbCUqhyNrPlkj4t6VR3/0XsofWS\nzjSzXzezwyUdKemH5cMEUBfGMxCeLKdSfVPSByUdbGbbJK1Wbzbnr0u6w8wk6V53/7i7bzazmyU9\nol577AJ3n64reAD5MJ6BbkhNzu7+0SGLrx6x/uclfb5MUADqwXgGuoErhAEAEBiSMwAAgSE5AwAQ\nGJIzAACBITkDABAYkjMAAIGxPRcDajEIs59Kek1SqBesPVhhxhZqXFK4sYUal7RvbIe5+1vbCqYo\nM3tV0mNtx5GgS7//UIQal9Sd2HKP5SCSsySZ2UZ3X9Z2HMOEGluocUnhxhZqXFLYseUR8usgtvxC\njUsa79hoawMAEBiSMwAAgQkpOa9tO4ARQo0t1LikcGMLNS4p7NjyCPl1EFt+ocYljXFswRxzBgAA\nPSFVzgAAQCRnAACC03pyNrPlZvaYmW0xs0tajmWRmX3fzB4xs81m9olo+Roz225mD0T/VrQU31Yz\neyiKYWO07CAzu8PMHo9+zm04pnfG3pcHzOxnZvbJtt4zM7vGzHaa2cOxZUPfI+v5i+j/3oNmtrSF\n2C43s/8d7f9vzezAaPliM/tl7P27qs7YqhLKeGYsF46L8Vw8rmrHsru39k/SDElPSDpC0mxJP5a0\npMV45ktaGt3eX9JPJC2RtEbSn7f5XkUxbZV08MCy/yrpkuj2JZK+1PLv81lJh7X1nkk6UdJSSQ+n\nvUeSVkj6e0km6XhJG1qI7V9Lmhnd/lIstsXx9brwL6TxzFiu7PfJeM4eV6Vjue3K+ThJW9z9SXff\nJelGSSvbCsbdd7j7/dHtVyU9KmlBW/FktFLSuuj2OkmntRjLyZKecPen2grA3e+W9OLA4qT3aKWk\n67znXkkHmtn8JmNz9++6+1R0915JC+vafwOCGc+M5UownnPEVfVYbjs5L5D0TOz+NgUygMxssaT3\nSdoQLbowaldc00a7KeKSvmtmm8xsVbTsEHffEd1+VtIh7YQmSTpT0jdj90N4z6Tk9yi0/3/nqffJ\nv+9wM/uRmd1lZv+qraByCO39lMRYLoHxXFzpsdx2cg6Sme0n6W8kfdLdfybpq5LeIeloSTsk/feW\nQjvB3ZdK+oikC8zsxPiD3uuhtHJunJnNlnSqpL+OFoXynu2lzfdoFDO7TNKUpBuiRTskHeru75N0\nsaS/MrO3tBVfVzGWi2E8F1fVWG47OW+XtCh2f2G0rDVmNku9wXyDu39Lktz9OXefdvc3JH1NvfZd\n49x9e/Rzp6S/jeJ4rt+6iX7ubCM29f7I3O/uz0UxBvGeRZLeoyD+/5nZxySdIumPoz82cvdfufsL\n0e1N6h3L/a2mY8spiPezj7FcCuO5gCrHctvJ+T5JR5rZ4dEntTMlrW8rGDMzSVdLetTdr4gtjx+3\n+DeSHh7ctoHY5pjZ/v3b6k0+eFi99+ucaLVzJN3adGyRjyrWAgvhPYtJeo/WSzo7muV5vKRXYu2y\nRpjZckmflnSqu/8itvytZjYjun2EpCMlPdlkbAUEM54Zy6UxnnOqfCzXNZst6z/1Ztj9RL1PE5e1\nHMsJ6rVIHpT0QPRvhaTrJT0ULV8vaX4LsR2h3uzXH0va3H+vJM2TdKekxyX9g6SDWohtjqQXJB0Q\nW9bKe6beH5Qdkl5X75jT+UnvkXqzOr8S/d97SNKyFmLbot5xsv7/t6uidf9t9Ht+QNL9kv6g6d9r\nwdcYxHhmLJeKj/FcLK5KxzKX7wQAIDBtt7UBAMAAkjMAAIEhOQMAEBiSMwAAgSE5AwAQGJIzAACB\nITkDABCY/x+cDZZHF35ZBQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-Mc6KoQ2eL6I", + "colab_type": "text" + }, + "source": [ + "# Conclusion\n", + "\n", + "Looks good! Have fun training the models on your own dataset ! There are a lot of features that this demo doesn't use like:\n", + "\n", + "1. Mixed precision training\n", + "2. Lighter object detection architectures like RetinaNet\n", + "\n", + "and so much more !" + ] + } + ] +} \ No newline at end of file From b5b0a0a282171d99b0098f5e12e730a18a68a5ff Mon Sep 17 00:00:00 2001 From: Linus Date: Tue, 4 Jun 2019 20:44:04 +0900 Subject: [PATCH 41/52] out0 to out in backbone/resnet (#858) --- maskrcnn_benchmark/modeling/backbone/resnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maskrcnn_benchmark/modeling/backbone/resnet.py b/maskrcnn_benchmark/modeling/backbone/resnet.py index 147531091..3fd2d41e7 100644 --- a/maskrcnn_benchmark/modeling/backbone/resnet.py +++ b/maskrcnn_benchmark/modeling/backbone/resnet.py @@ -332,8 +332,8 @@ def forward(self, x): out = self.bn2(out) out = F.relu_(out) - out0 = self.conv3(out) - out = self.bn3(out0) + out = self.conv3(out) + out = self.bn3(out) if self.downsample is not None: identity = self.downsample(x) From c8eff2ca2d5c1c14393f1a1b21853895eaacfd51 Mon Sep 17 00:00:00 2001 From: Csaba Botos Date: Tue, 4 Jun 2019 16:09:51 +0200 Subject: [PATCH 42/52] Add RLE size check for SegmentationMask (#859) * support RLE and binary mask * do not convert to numpy * be consistent with Detectron * delete wrong comment * [WIP] add tests for segmentation_mask * update tests * minor change * Refactored segmentation_mask.py * Add unit test for segmentation_mask.py * Add RLE support for BinaryMaskList * PEP8 black formatting * Minor patch * Use internal that handles 0 channels * Fix polygon slicing * Revert "Pr150" * Add RLE size check for SegmentationMask --- .../structures/segmentation_mask.py | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 4547398a0..89b80aff2 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -48,31 +48,53 @@ def __init__(self, masks, size): initializing source data intact. """ + assert isinstance(size, (list, tuple)) + assert len(size) == 2 + if isinstance(masks, torch.Tensor): # The raw data representation is passed as argument masks = masks.clone() elif isinstance(masks, (list, tuple)): if len(masks) == 0: - masks = torch.empty([0, size[1], size[0]]) # num_instances = 0! + masks = torch.empty([0, size[1], size[0]]) # num_instances = 0! elif isinstance(masks[0], torch.Tensor): masks = torch.stack(masks, dim=2).clone() elif isinstance(masks[0], dict) and "counts" in masks[0]: # RLE interpretation - assert all( - [(size[1], size[0]) == tuple(inst["size"]) for inst in masks] - ) # in RLE, height come first in "size" + rle_sizes = [tuple(inst["size"]) for inst in masks] + masks = mask_utils.decode(masks) # [h, w, n] masks = torch.tensor(masks).permute(2, 0, 1) # [n, h, w] + + assert rle_sizes.count(rle_sizes[0]) == len(rle_sizes), ( + "All the sizes must be the same size: %s" % rle_sizes + ) + + # in RLE, height come first in "size" + rle_height, rle_width = rle_sizes[0] + assert masks.shape[1] == rle_height + assert masks.shape[2] == rle_width + + width, height = size + if width != rle_width or height != rle_height: + masks = interpolate( + input=masks[None].float(), + size=(height, width), + mode="bilinear", + align_corners=False, + )[0].type_as(masks) else: RuntimeError( - "Type of `masks[0]` could not be interpreted: %s" % type(masks) + "Type of `masks[0]` could not be interpreted: %s" + % type(masks) ) elif isinstance(masks, BinaryMaskList): # just hard copy the BinaryMaskList instance's underlying data masks = masks.masks.clone() else: RuntimeError( - "Type of `masks` argument could not be interpreted:%s" % type(masks) + "Type of `masks` argument could not be interpreted:%s" + % type(masks) ) if len(masks.shape) == 2: @@ -155,7 +177,9 @@ def _findContours(self): reshaped_contour = [] for entity in contour: assert len(entity.shape) == 3 - assert entity.shape[1] == 1, "Hierarchical contours are not allowed" + assert ( + entity.shape[1] == 1 + ), "Hierarchical contours are not allowed" reshaped_contour.append(entity.reshape(-1).tolist()) contours.append(reshaped_contour) return contours @@ -206,7 +230,8 @@ def __init__(self, polygons, size): else: RuntimeError( - "Type of argument `polygons` is not allowed:%s" % (type(polygons)) + "Type of argument `polygons` is not allowed:%s" + % (type(polygons)) ) """ This crashes the training way too many times... @@ -278,7 +303,9 @@ def resize(self, size): assert isinstance(size, (int, float)) size = size, size - ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(size, self.size)) + ratios = tuple( + float(s) / float(s_orig) for s, s_orig in zip(size, self.size) + ) if ratios[0] == ratios[1]: ratio = ratios[0] @@ -349,7 +376,9 @@ def __init__(self, polygons, size): type(polygons[0][0]) ) else: - assert isinstance(polygons[0], PolygonInstance), str(type(polygons[0])) + assert isinstance(polygons[0], PolygonInstance), str( + type(polygons[0]) + ) elif isinstance(polygons, PolygonList): size = polygons.size @@ -357,7 +386,8 @@ def __init__(self, polygons, size): else: RuntimeError( - "Type of argument `polygons` is not allowed:%s" % (type(polygons)) + "Type of argument `polygons` is not allowed:%s" + % (type(polygons)) ) assert isinstance(size, (list, tuple)), str(type(size)) @@ -404,7 +434,9 @@ def to(self, *args, **kwargs): def convert_to_binarymask(self): if len(self) > 0: - masks = torch.stack([p.convert_to_binarymask() for p in self.polygons]) + masks = torch.stack( + [p.convert_to_binarymask() for p in self.polygons] + ) else: size = self.size masks = torch.empty([0, size[1], size[0]], dtype=torch.uint8) From 378845c546efcc4c2ac7751366ed57311527ab7e Mon Sep 17 00:00:00 2001 From: keineahnung2345 Date: Wed, 12 Jun 2019 19:43:00 +0800 Subject: [PATCH 43/52] Fix typo for shapes_dataset_demo.ipynb (#885) Fix typo, highlight words in python language, add hyperlink --- demo/shapes_dataset_demo.ipynb | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/demo/shapes_dataset_demo.ipynb b/demo/shapes_dataset_demo.ipynb index e073ec997..977f123c0 100644 --- a/demo/shapes_dataset_demo.ipynb +++ b/demo/shapes_dataset_demo.ipynb @@ -928,7 +928,7 @@ "source": [ "### Checking our Installation\n", "\n", - "If a module not found error appears, restart the runtime. The libraries should be loaded after restarting" + "If a `Module not found` error appears, restart the runtime. The libraries should be loaded after restarting" ] }, { @@ -1035,21 +1035,21 @@ "source": [ "# Loading Our Dataset\n", "\n", - "To train a network using the MaskRCNN repo, we first need to define our dataset. The dataset needs to a class of type object and should extend 3 things. \n", + "To train a network using the `facebookresearch/maskrcnn-benchmark` repo, we first need to define our dataset. The dataset needs to be a subclass of `object` and should implement 6 things. \n", "\n", - "1. **__getitem__(self, idx)**: This function should return a PIL Image, a BoxList and the idx. The Boxlist is an abstraction for our bounding boxes, segmentation masks, class lables and also people keypoints. Please check ABSTRACTIONS.ms for more details on this. \n", + "1. `__getitem__(self, idx)`: This function should return a PIL Image, a BoxList and the idx. The Boxlist is an abstraction for our bounding boxes, segmentation masks, class labels and also people keypoints. Please check [ABSTRACTIONS.md](https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/ABSTRACTIONS.md) for more details on this. \n", "\n", - "2. **__len__()**: returns the length of the dataset. \n", + "2. `__len__()`: returns the length of the dataset. \n", "\n", - "3. **get_img_info(self, idx)**: Return a dict of img info with the fields \"height\" and \"width\" filled in with the idx's image's height and width.\n", + "3. `get_img_info(self, idx)`: Return a dict of img info with the fields \"height\" and \"width\" filled in with the idx's image's height and width.\n", "\n", - "4. **self.coco**: Should be a variable that holds the COCO object for your annotations so that you can perform evaluations of your dataset. \n", + "4. `self.coco`: Should be a variable that holds the COCO object for your annotations so that you can perform evaluations of your dataset. \n", "\n", - "5. **self.id_to_img_map**: Is a dictionary that maps the ids to coco image ids. Almost in all cases just map the idxs to idxs. This is simply a requirement for the coco evaluation. \n", + "5. `self.id_to_img_map`: Is a dictionary that maps the ids to coco image ids. Almost in all cases just map the idxs to idxs. This is simply a requirement for the coco evaluation. \n", "\n", - "6. **self.contiguous_category_id_to_json_id**: Another requirement for coco evaluation. It maps the categpry to json category id. Again, for almost all purposes category id and json id should be same. \n", + "6. `self.contiguous_category_id_to_json_id`: Another requirement for coco evaluation. It maps the categpry to json category id. Again, for almost all purposes category id and json id should be same. \n", "\n", - "Given below is a sample fo a dataset. It is the Shape Dataset taken from the Matterport Mask RCNN Repo. One important detail is that the constructor if the dataset should have the variable transforms that is set inside the constructor. It should thgen be used inside **__get__item(idx)** as shown below." + "Given below is a sample fo a dataset. It is the Shape Dataset taken from the [Matterport/Mask_RCNN](https://github.com/matterport/Mask_RCNN) Repo. One important detail is that the constructor of the dataset should have the variable `transforms`, which is set inside the constructor. It should then be used in `__getitem__(self, idx)` as shown below." ] }, { @@ -1160,7 +1160,7 @@ " self.image_info = []\n", " self.logger = logging.getLogger(__name__)\n", " \n", - " # Class Names: Note that the ids start fromm 1 not 0. This repo uses the 0 index for background\n", + " # Class Names: Note that the ids start from 1, not 0. This repo uses the index 0 for background\n", " self.class_names = {\"square\": 1, \"circle\": 2, \"triangle\": 3}\n", " \n", " # Add images\n", @@ -1188,7 +1188,7 @@ " def random_shape(self, height, width):\n", " \"\"\"Generates specifications of a random shape that lies within\n", " the given height and width boundaries.\n", - " Returns a tuple of three valus:\n", + " Returns a tuple of three values:\n", " * The shape name (square, circle, ...)\n", " * Shape color: a tuple of 3 values, RGB.\n", " * Shape dimensions: A tuple of values that define the shape size\n", @@ -1225,7 +1225,7 @@ " x, y, s = dims\n", " boxes.append([y-s, x-s, y+s, x+s])\n", "\n", - " # Apply non-max suppression wit 0.3 threshold to avoid\n", + " # Apply non-max suppression with 0.3 threshold to avoid\n", " # shapes covering each other\n", " keep_ixs = non_max_suppression(np.array(boxes), np.arange(N), 0.3)\n", " shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]\n", @@ -1310,7 +1310,7 @@ " masks = SegmentationMask(torch.tensor(masks), image.size, \"mask\")\n", " boxlist.add_field(\"masks\", masks)\n", " \n", - " # Important line! dont forget to add this\n", + " # Important line! don't forget to add this\n", " if self.transforms:\n", " image, boxlist = self.transforms(image, boxlist)\n", "\n", @@ -1362,7 +1362,7 @@ " bbox = [ x - s, y - s, x+s, y +s ] \n", " area = (bbox[0] - bbox[2]) * (bbox[1] - bbox[3])\n", " \n", - " # Format for COCOC\n", + " # Format for COCO\n", " annotations.append( {\n", " \"id\": int(ann_id),\n", " \"category_id\": category_id,\n", @@ -1492,7 +1492,7 @@ "source": [ "# Training a Model\n", "\n", - "Now we move on to training our very own model. Here we will be finetuning a Mask RCNN on this dataset. To do this we need\n", + "Now we move on to training our very own model. Here we will be fine-tuning a Mask RCNN on this dataset. To do this we need\n", "\n", "1. A base model that has the same amount of output classes as our dataset. In this case, we have need for only 3 classes instead of COCO's 80. Hence , we first need to do some model trimming. " ] @@ -1902,7 +1902,7 @@ "source": [ "### Base Model Config\n", "\n", - "This is the base model that we will finetune from. First we need to replace the bounding box heads and mask heads to make it compatible with our Shapes Dataset." + "This is the base model that we will fine-tune from. First we need to replace the bounding box heads and mask heads to make it compatible with our Shapes Dataset." ] }, { @@ -1970,9 +1970,9 @@ "colab_type": "text" }, "source": [ - "### Pretrained weight removal\n", + "### Pre-trained weight removal\n", "\n", - "Here, the pretrained weights of bbox, mask and class predictions are removed. This is done so that we can make the model shapes dataset compatible i.e predict 3 classes instead of Coco's 81 classes." + "Here, the pre-trained weights of bbox, mask and class predictions are removed. This is done so that we can make the model shapes dataset compatible i.e predict 3 classes instead of Coco's 81 classes." ] }, { @@ -2027,7 +2027,7 @@ " \"roi_heads.mask.predictor.mask_fcn_logits.weight\", \"roi_heads.mask.predictor.mask_fcn_logits.bias\"\n", " ])\n", "\n", - "# Save new state dict, we will use this as our starting weights for our finwe tuned model\n", + "# Save new state dict, we will use this as our starting weights for our fine-tuned model\n", "torch.save(new_state_dict, \"base_model.pth\")\n" ], "execution_count": 0, @@ -2044,8 +2044,8 @@ "\n", "Here we define our shape Dataset config. The important fields are \n", "\n", - "1. WEIGHT: which point to our base_model.pth saved in the previous step\n", - "2. NUM_CLASSES: Which define how many classes we will rpedict . note that the number includes the background, hence our shapes dataset has 4 classes. " + "1. WEIGHT: which point to our `base_model.pth` saved in the previous step\n", + "2. NUM_CLASSES: Which define how many classes we will predict . Note that the number includes the background, hence our shapes dataset has 4 classes. " ] }, { @@ -2313,10 +2313,10 @@ "cfg.merge_from_file(config_file)\n", "\n", "cfg.merge_from_list(['OUTPUT_DIR', 'shapeDir']) # The output folder where all our model checkpoints will be saved during training.\n", - "cfg.merge_from_list(['SOLVER.IMS_PER_BATCH', 4]) # Number of images to take insiade a single batch. This number depends on the size of your GPU\n", + "cfg.merge_from_list(['SOLVER.IMS_PER_BATCH', 4]) # Number of images to take inside a single batch. This number depends on the size of your GPU\n", "cfg.merge_from_list(['SOLVER.BASE_LR', 0.0050]) # The Learning Rate when training starts. Please check Detectron scaling rules to determine your learning for your GPU setup. \n", "cfg.merge_from_list(['SOLVER.MAX_ITER', 1000]) # The number of training iterations that will be executed during training. One iteration is given as one forward and backward pass of a mini batch of the network\n", - "cfg.merge_from_list(['SOLVER.STEPS', \"(700, 800)\"]) # These two numberes represent after how many iterations is the learning rate divided by 10. \n", + "cfg.merge_from_list(['SOLVER.STEPS', \"(700, 800)\"]) # These two numbers represent after how many iterations is the learning rate divided by 10. \n", "cfg.merge_from_list(['TEST.IMS_PER_BATCH', 1]) # Batch size during testing/evaluation\n", "cfg.merge_from_list(['MODEL.RPN.FPN_POST_NMS_TOP_N_TRAIN', 4000]) # This determines how many region proposals to take in for processing into the stage after the RPN. The rule is 1000*batch_size = 4*1000 \n", "cfg.merge_from_list(['SOLVER.CHECKPOINT_PERIOD', 100]) # After how many iterations does one want to save the model.\n", @@ -2754,7 +2754,7 @@ "source": [ "# Evaluation\n", "\n", - "Now after our model is trainined, we would like to see how well it predicts objects in our sample images. One way to validate your model is through a standard metric called COCO mAP. This metric is used widely. Hence, we shall do this now. " + "Now after our model is trained, we would like to see how well it predicts objects in our sample images. One way to validate your model is through a standard metric called COCO mAP. This metric is used widely. Hence, we shall do this now. " ] }, { @@ -3073,7 +3073,7 @@ "source": [ "# Conclusion\n", "\n", - "Looks good! Have fun training the models on your own dataset ! There are a lot of features that this demo doesn't use like:\n", + "Looks good! Have fun training the models on your own dataset ! There are a lot of features that this demo doesn't use, like:\n", "\n", "1. Mixed precision training\n", "2. Lighter object detection architectures like RetinaNet\n", @@ -3082,4 +3082,4 @@ ] } ] -} \ No newline at end of file +} From 5b20697aad3f014fdb2721f3ff40602054ba8114 Mon Sep 17 00:00:00 2001 From: SkeletonOne Date: Wed, 12 Jun 2019 19:43:32 +0800 Subject: [PATCH 44/52] Update build.py (#878) --- maskrcnn_benchmark/data/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/data/build.py b/maskrcnn_benchmark/data/build.py index 8cf610ade..26239155d 100644 --- a/maskrcnn_benchmark/data/build.py +++ b/maskrcnn_benchmark/data/build.py @@ -19,7 +19,7 @@ def build_dataset(dataset_list, transforms, dataset_catalog, is_train=True): """ Arguments: dataset_list (list[str]): Contains the names of the datasets, i.e., - coco_2014_trian, coco_2014_val, etc + coco_2014_train, coco_2014_val, etc transforms (callable): transforms to apply to each (image, target) sample dataset_catalog (DatasetCatalog): contains the information on how to construct a dataset. From 7ee52a489ea64adb42babc03262902620445d6d7 Mon Sep 17 00:00:00 2001 From: Gu Wang Date: Wed, 12 Jun 2019 19:45:29 +0800 Subject: [PATCH 45/52] allow test for any checkpoint (#874) --- maskrcnn_benchmark/utils/checkpoint.py | 4 ++-- tools/test_net.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/utils/checkpoint.py b/maskrcnn_benchmark/utils/checkpoint.py index dc403f5db..2af2565ed 100644 --- a/maskrcnn_benchmark/utils/checkpoint.py +++ b/maskrcnn_benchmark/utils/checkpoint.py @@ -49,8 +49,8 @@ def save(self, name, **kwargs): torch.save(data, save_file) self.tag_last_checkpoint(save_file) - def load(self, f=None): - if self.has_checkpoint(): + def load(self, f=None, use_latest=True): + if self.has_checkpoint() and use_latest: # override argument with existing checkpoint f = self.get_checkpoint_file() if not f: diff --git a/tools/test_net.py b/tools/test_net.py index c666a4655..7a2d3f7b9 100644 --- a/tools/test_net.py +++ b/tools/test_net.py @@ -33,6 +33,11 @@ def main(): help="path to config file", ) parser.add_argument("--local_rank", type=int, default=0) + parser.add_argument( + "--ckpt", + help="The path to the checkpoint for test, default is the latest checkpoint.", + default=None, + ) parser.add_argument( "opts", help="Modify config options using the command-line", @@ -73,7 +78,7 @@ def main(): output_dir = cfg.OUTPUT_DIR checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir) - _ = checkpointer.load(cfg.MODEL.WEIGHT) + _ = checkpointer.load(args.ckpt, use_latest=args.ckpt is None) iou_types = ("bbox",) if cfg.MODEL.MASK_ON: From d173a85b463d6a1ac47e00de7d9d8fd499cf47a8 Mon Sep 17 00:00:00 2001 From: keineahnung2345 Date: Wed, 12 Jun 2019 20:46:03 +0800 Subject: [PATCH 46/52] update dockerfile according to the new INSTALL.md (#883) From https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/INSTALL.md: 1. additionally install tqdm, cudatoolkit, apex 2. add FORCE_CUDA argument --- docker/docker-jupyter/Dockerfile | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/docker-jupyter/Dockerfile b/docker/docker-jupyter/Dockerfile index bf957b636..afc512679 100644 --- a/docker/docker-jupyter/Dockerfile +++ b/docker/docker-jupyter/Dockerfile @@ -28,11 +28,14 @@ ENV PATH=$CONDA_PREFIX/bin:$PATH ENV CONDA_AUTO_UPDATE_CONDA=false RUN conda install -y ipython -RUN pip install requests ninja yacs cython matplotlib jupyter +RUN pip install requests ninja yacs cython matplotlib jupyter tqdm -# Install PyTorch 1.0 Nightly and OpenCV -RUN conda install -y pytorch-nightly -c pytorch \ - && conda install -y opencv -c menpo \ +# Install PyTorch Nightly +ARG CUDA +RUN conda install -y pytorch-nightly cudatoolkit=${CUDA} -c pytorch + +# Install OpenCV +RUN conda install -y opencv -c menpo \ && conda clean -ya WORKDIR /root @@ -53,7 +56,14 @@ RUN git clone https://github.com/cocodataset/cocoapi.git \ && cd cocoapi/PythonAPI \ && python setup.py build_ext install +# install apex +RUN git clone https://github.com/NVIDIA/apex.git \ + && cd apex \ + && python setup.py install --cuda_ext --cpp_ext + # install PyTorch Detection +ARG FORCE_CUDA="1" +ENV FORCE_CUDA=${FORCE_CUDA} RUN git clone https://github.com/facebookresearch/maskrcnn-benchmark.git \ && cd maskrcnn-benchmark \ && python setup.py build develop From 5a431ef76c161986fea1d6890df427d1dacaf76b Mon Sep 17 00:00:00 2001 From: Johnqczhang Date: Fri, 14 Jun 2019 16:30:06 +0800 Subject: [PATCH 47/52] allow test for checkpoint from cfg.MODEL.WEIGHT (#894) --- tools/test_net.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/test_net.py b/tools/test_net.py index 7a2d3f7b9..ee3bf4cab 100644 --- a/tools/test_net.py +++ b/tools/test_net.py @@ -78,7 +78,8 @@ def main(): output_dir = cfg.OUTPUT_DIR checkpointer = DetectronCheckpointer(cfg, model, save_dir=output_dir) - _ = checkpointer.load(args.ckpt, use_latest=args.ckpt is None) + ckpt = cfg.MODEL.WEIGHT if args.ckpt is None else args.ckpt + _ = checkpointer.load(ckpt, use_latest=args.ckpt is None) iou_types = ("bbox",) if cfg.MODEL.MASK_ON: From eea7b82dbac960bd954595410e04d4e79ae6e629 Mon Sep 17 00:00:00 2001 From: keineahnung2345 Date: Mon, 17 Jun 2019 21:18:44 +0800 Subject: [PATCH 48/52] Fix typo in bounding_box.py (#902) --- maskrcnn_benchmark/structures/bounding_box.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/structures/bounding_box.py b/maskrcnn_benchmark/structures/bounding_box.py index 4084024fa..25791d578 100644 --- a/maskrcnn_benchmark/structures/bounding_box.py +++ b/maskrcnn_benchmark/structures/bounding_box.py @@ -166,7 +166,7 @@ def transpose(self, method): def crop(self, box): """ - Cropss a rectangular region from this bounding box. The box is a + Crops a rectangular region from this bounding box. The box is a 4-tuple defining the left, upper, right, and lower pixel coordinate. """ From 6dfb4db9d156b06a2fccbd8f8cfbd8385dd18123 Mon Sep 17 00:00:00 2001 From: Johnqczhang Date: Mon, 17 Jun 2019 21:38:00 +0800 Subject: [PATCH 49/52] print pretty evaluation results (#900) * print pretty evaluation results * remove `name` attribute from Datasets * add a newline before printing results --- .../data/datasets/evaluation/coco/coco_eval.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py b/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py index e4ca14baa..a8fdc280e 100644 --- a/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py +++ b/maskrcnn_benchmark/data/datasets/evaluation/coco/coco_eval.py @@ -364,8 +364,14 @@ def update(self, coco_eval): res[metric] = s[idx] def __repr__(self): - # TODO make it pretty - return repr(self.results) + results = '\n' + for task, metrics in self.results.items(): + results += 'Task: {}\n'.format(task) + metric_names = metrics.keys() + metric_vals = ['{:.4f}'.format(v) for v in metrics.values()] + results += (', '.join(metric_names) + '\n') + results += (', '.join(metric_vals) + '\n') + return results def check_expected_results(results, expected_results, sigma_tol): From 8783a3102426a986ac478ce992150519e6a992e6 Mon Sep 17 00:00:00 2001 From: keineahnung2345 Date: Thu, 20 Jun 2019 03:05:16 +0800 Subject: [PATCH 50/52] Fix typo: percentace -> percentage (#914) --- .../modeling/balanced_positive_negative_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/modeling/balanced_positive_negative_sampler.py b/maskrcnn_benchmark/modeling/balanced_positive_negative_sampler.py index c0bd00444..902a60eb2 100644 --- a/maskrcnn_benchmark/modeling/balanced_positive_negative_sampler.py +++ b/maskrcnn_benchmark/modeling/balanced_positive_negative_sampler.py @@ -11,7 +11,7 @@ def __init__(self, batch_size_per_image, positive_fraction): """ Arguments: batch_size_per_image (int): number of elements to be selected per image - positive_fraction (float): percentace of positive elements per batch + positive_fraction (float): percentage of positive elements per batch """ self.batch_size_per_image = batch_size_per_image self.positive_fraction = positive_fraction From 73ed87954c971b4f6f38d9da442bcac51055353e Mon Sep 17 00:00:00 2001 From: keineahnung2345 Date: Sun, 23 Jun 2019 17:21:47 +0800 Subject: [PATCH 51/52] Fix minor typo in rpn.py (#921) --- maskrcnn_benchmark/modeling/rpn/rpn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maskrcnn_benchmark/modeling/rpn/rpn.py b/maskrcnn_benchmark/modeling/rpn/rpn.py index 07997651c..c279a232f 100644 --- a/maskrcnn_benchmark/modeling/rpn/rpn.py +++ b/maskrcnn_benchmark/modeling/rpn/rpn.py @@ -108,8 +108,8 @@ def forward(self, x): class RPNModule(torch.nn.Module): """ - Module for RPN computation. Takes feature maps from the backbone and RPN - proposals and losses. Works for both FPN and non-FPN. + Module for RPN computation. Takes feature maps from the backbone and outputs + RPN proposals and losses. Works for both FPN and non-FPN. """ def __init__(self, cfg, in_channels): From 5018ca2da5a338748e04eb1b238408e1e223bea5 Mon Sep 17 00:00:00 2001 From: keineahnung2345 Date: Mon, 24 Jun 2019 17:01:14 +0800 Subject: [PATCH 52/52] Fix typos in ROIAlign_cpu.cpp (#923) --- maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp b/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp index d35aedf27..d531da623 100644 --- a/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp +++ b/maskrcnn_benchmark/csrc/cpu/ROIAlign_cpu.cpp @@ -91,7 +91,7 @@ void pre_calc_for_bilinear_interpolate( T hy = 1. - ly, hx = 1. - lx; T w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; - // save weights and indeces + // save weights and indices PreCalc pc; pc.pos1 = y_low * width + x_low; pc.pos2 = y_low * width + x_high; @@ -168,8 +168,8 @@ void ROIAlignForward_cpu_kernel( // We do average (integral) pooling inside a bin const T count = roi_bin_grid_h * roi_bin_grid_w; // e.g. = 4 - // we want to precalculate indeces and weights shared by all chanels, - // this is the key point of optimiation + // we want to precalculate indices and weights shared by all channels, + // this is the key point of optimization std::vector> pre_calc( roi_bin_grid_h * roi_bin_grid_w * pooled_width * pooled_height); pre_calc_for_bilinear_interpolate(