From 7df230153e1c4c1a288a52042ba0f547ce23eb9b Mon Sep 17 00:00:00 2001 From: wangg12 Date: Tue, 13 Nov 2018 22:46:59 +0800 Subject: [PATCH 01/14] support RLE and binary mask --- .../structures/segmentation_mask.py | 137 +++++++++++++----- 1 file changed, 97 insertions(+), 40 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index ba1290b91..a474040b3 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -1,51 +1,97 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. import torch - +import numpy as np +from torch.nn.functional import interpolate import pycocotools.mask as mask_utils # transpose FLIP_LEFT_RIGHT = 0 FLIP_TOP_BOTTOM = 1 - class Mask(object): """ This class is unfinished and not meant for use yet It is supposed to contain the mask for an object as a 2d tensor """ - - def __init__(self, masks, size, mode): - self.masks = masks + def __init__(self, segm, size, mode): + width, height = size + if isinstance(segm, Mask): + mask = segm.mask + else: + if type(segm) == list: + # polygons + rle = mask_utils.frPyObjects(segm, height, width) + mask = np.array(mask_utils.decode(rle), dtype=np.float32) + mask = np.sum(mask, axis=2) + mask = torch.from_numpy(np.array(mask > 0, dtype=np.float32)) + elif type(segm) == dict and 'counts' in segm: + if type(segm['counts']) == list: + # uncompressed RLE + h, w = segm['size'] + rle = mask_utils.frPyObjects(segm, h, w) + mask = mask_utils.decode(rle) + mask = torch.from_numpy(mask).to(dtype=torch.float32) + else: + # compressed RLE + mask = mask_utils.decode(segm) + mask = torch.from_numpy(mask).to(dtype=torch.float32) + else: + # binary mask + if type(segm) == np.ndarray: + mask = torch.from_numpy(segm).to(dtype=torch.float32) + else: # torch.Tensor + mask = segm + self.mask = mask self.size = size self.mode = mode def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): - raise NotImplementedError( - "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" - ) + raise NotImplementedError("Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented") width, height = self.size if method == FLIP_LEFT_RIGHT: - dim = width - idx = 2 + max_idx = width + dim = 1 elif method == FLIP_TOP_BOTTOM: - dim = height - idx = 1 + max_idx = height + dim = 0 - flip_idx = list(range(dim)[::-1]) - flipped_masks = self.masks.index_select(dim, flip_idx) - return Mask(flipped_masks, self.size, self.mode) + flip_idx = torch.tensor(list(range(max_idx)[::-1])) + flipped_mask = self.mask.index_select(dim, flip_idx) + return Mask(flipped_mask, self.size, self.mode) def crop(self, box): - w, h = box[2] - box[0], box[3] - box[1] - - cropped_masks = self.masks[:, box[1] : box[3], box[0] : box[2]] - return Mask(cropped_masks, size=(w, h), mode=self.mode) + box = [int(b) for b in box] + TO_REMOVE = 1 + w, h = box[2] - box[0] + TO_REMOVE, box[3] - box[1] + TO_REMOVE + cropped_mask = self.mask[box[1]: box[3]+1, box[0]: box[2]+1] + return Mask(cropped_mask, size=(w, h), mode=self.mode) + # torch.nn.functional.interpolate has a arg as dim, only tensor have dim, so turn array to tensor def resize(self, size, *args, **kwargs): - pass + width, height = size + scaled_mask = torch.squeeze(interpolate(torch.from_numpy(np.array(self.mask)[None, None, :, :]).float(), + (height, width), + mode='nearest')) + return Mask(scaled_mask, size=size, mode=self.mode) + + def convert(self, mode): + mask = self.mask + return mask + + def __iter__(self): + return iter(self.mask) + + def __repr__(self): + s = self.__class__.__name__ + "(" + # s += "num_mask={}, ".format(len(self.mask)) + s += "image_width={}, ".format(self.size[0]) + s += "image_height={}, ".format(self.size[1]) + s += "mode={})".format(self.mode) + return s + class Polygons(object): @@ -90,7 +136,8 @@ def transpose(self, method): return Polygons(flipped_polygons, size=self.size, mode=self.mode) def crop(self, box): - w, h = box[2] - box[0], box[3] - box[1] + TO_REMOVE = 1 + w, h = box[2] - box[0] + TO_REMOVE, box[3] - box[1] + TO_REMOVE # TODO chck if necessary w = max(w, 1) @@ -130,7 +177,8 @@ def convert(self, mode): ) rle = mask_utils.merge(rles) mask = mask_utils.decode(rle) - mask = torch.from_numpy(mask) + mask = np.sum(mask, axis=2) + mask = torch.from_numpy(np.array(mask > 0, dtype=np.float32)) # TODO add squeeze? return mask @@ -148,17 +196,26 @@ class SegmentationMask(object): This class stores the segmentations for all objects in the image """ - def __init__(self, polygons, size, mode=None): + def __init__(self, segms, size, mode=None): """ Arguments: - polygons: a list of list of lists of numbers. The first + segms: three types + (1) polygons: a list of list of lists of numbers. The first level of the list correspond to individual instances, the second level to all the polygons that compose the object, and the third level to the polygon coordinates. + (2) rles: COCO's run length encoding format, uncompressed or compressed + (3) binary masks + size: (width, height) + mode: 'polygon', 'mask'. if mode is 'mask', convert mask of any format to binary mask """ - assert isinstance(polygons, list) - - self.polygons = [Polygons(p, size, mode) for p in polygons] + assert isinstance(segms, list) + if type(segms[0]) != list: + mode = 'mask' + if mode == 'mask': + self.masks = [Mask(m, size, mode) for m in segms] + else: # polygons + self.masks = [Polygons(p, size, mode) for p in segms] self.size = size self.mode = mode @@ -169,21 +226,21 @@ def transpose(self, method): ) flipped = [] - for polygon in self.polygons: - flipped.append(polygon.transpose(method)) + for mask in self.masks: + flipped.append(mask.transpose(method)) return SegmentationMask(flipped, size=self.size, mode=self.mode) def crop(self, box): - w, h = box[2] - box[0], box[3] - box[1] + w, h = box[2] - box[0] + 1, box[3] - box[1] + 1 cropped = [] - for polygon in self.polygons: - cropped.append(polygon.crop(box)) + for mask in self.masks: + cropped.append(mask.crop(box)) return SegmentationMask(cropped, size=(w, h), mode=self.mode) def resize(self, size, *args, **kwargs): scaled = [] - for polygon in self.polygons: - scaled.append(polygon.resize(size, *args, **kwargs)) + for mask in self.masks: + scaled.append(mask.resize(size, *args, **kwargs)) return SegmentationMask(scaled, size=size, mode=self.mode) def to(self, *args, **kwargs): @@ -191,24 +248,24 @@ def to(self, *args, **kwargs): def __getitem__(self, item): if isinstance(item, (int, slice)): - selected_polygons = [self.polygons[item]] + selected_masks = [self.masks[item]] else: # advanced indexing on a single dimension - selected_polygons = [] + selected_masks = [] if isinstance(item, torch.Tensor) and item.dtype == torch.uint8: item = item.nonzero() item = item.squeeze(1) if item.numel() > 0 else item item = item.tolist() for i in item: - selected_polygons.append(self.polygons[i]) - return SegmentationMask(selected_polygons, size=self.size, mode=self.mode) + selected_masks.append(self.masks[i]) + return SegmentationMask(selected_masks, size=self.size, mode=self.mode) def __iter__(self): - return iter(self.polygons) + return iter(self.masks) def __repr__(self): s = self.__class__.__name__ + "(" - s += "num_instances={}, ".format(len(self.polygons)) + s += "num_instances={}, ".format(len(self.masks)) s += "image_width={}, ".format(self.size[0]) s += "image_height={})".format(self.size[1]) return s From d4226ad7764f0b9f0b9218a1560962f66618f882 Mon Sep 17 00:00:00 2001 From: wangg12 Date: Tue, 13 Nov 2018 23:36:52 +0800 Subject: [PATCH 02/14] do not convert to numpy --- maskrcnn_benchmark/structures/segmentation_mask.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index a474040b3..f073091e6 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -72,9 +72,7 @@ def crop(self, box): # torch.nn.functional.interpolate has a arg as dim, only tensor have dim, so turn array to tensor def resize(self, size, *args, **kwargs): width, height = size - scaled_mask = torch.squeeze(interpolate(torch.from_numpy(np.array(self.mask)[None, None, :, :]).float(), - (height, width), - mode='nearest')) + scaled_mask = interpolate(self.mask[None, None, :, :], (height, width), mode='nearest')[0, 0] return Mask(scaled_mask, size=size, mode=self.mode) def convert(self, mode): From fbadd1ceb0e1c1f194df3752e0d0855fa00faa93 Mon Sep 17 00:00:00 2001 From: wangg12 Date: Wed, 14 Nov 2018 00:18:09 +0800 Subject: [PATCH 03/14] be consistent with Detectron --- maskrcnn_benchmark/structures/segmentation_mask.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index f073091e6..109895271 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -64,8 +64,7 @@ def transpose(self, method): def crop(self, box): box = [int(b) for b in box] - TO_REMOVE = 1 - w, h = box[2] - box[0] + TO_REMOVE, box[3] - box[1] + TO_REMOVE + w, h = box[2] - box[0], box[3] - box[1] cropped_mask = self.mask[box[1]: box[3]+1, box[0]: box[2]+1] return Mask(cropped_mask, size=(w, h), mode=self.mode) @@ -134,8 +133,7 @@ def transpose(self, method): return Polygons(flipped_polygons, size=self.size, mode=self.mode) def crop(self, box): - TO_REMOVE = 1 - w, h = box[2] - box[0] + TO_REMOVE, box[3] - box[1] + TO_REMOVE + w, h = box[2] - box[0], box[3] - box[1] # TODO chck if necessary w = max(w, 1) @@ -175,8 +173,7 @@ def convert(self, mode): ) rle = mask_utils.merge(rles) mask = mask_utils.decode(rle) - mask = np.sum(mask, axis=2) - mask = torch.from_numpy(np.array(mask > 0, dtype=np.float32)) + mask = torch.from_numpy(mask) # TODO add squeeze? return mask @@ -229,7 +226,7 @@ def transpose(self, method): return SegmentationMask(flipped, size=self.size, mode=self.mode) def crop(self, box): - w, h = box[2] - box[0] + 1, box[3] - box[1] + 1 + w, h = box[2] - box[0], box[3] - box[1] cropped = [] for mask in self.masks: cropped.append(mask.crop(box)) From b91dd11fed5896604e9193decff1bcad0907504c Mon Sep 17 00:00:00 2001 From: wangg12 Date: Wed, 14 Nov 2018 00:23:44 +0800 Subject: [PATCH 04/14] delete wrong comment --- maskrcnn_benchmark/structures/segmentation_mask.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 109895271..2ec2905f7 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -68,7 +68,6 @@ def crop(self, box): cropped_mask = self.mask[box[1]: box[3]+1, box[0]: box[2]+1] return Mask(cropped_mask, size=(w, h), mode=self.mode) - # torch.nn.functional.interpolate has a arg as dim, only tensor have dim, so turn array to tensor def resize(self, size, *args, **kwargs): width, height = size scaled_mask = interpolate(self.mask[None, None, :, :], (height, width), mode='nearest')[0, 0] From febb5428bf02d785ae91a0d5c6392238497636d1 Mon Sep 17 00:00:00 2001 From: wangg12 Date: Wed, 14 Nov 2018 22:44:18 +0800 Subject: [PATCH 05/14] [WIP] add tests for segmentation_mask --- .../structures/segmentation_mask.py | 13 +- tests/common_utils.py | 839 ++++++++++++++++++ tests/expecttest.py | 202 +++++ tests/test_segmentation_mask.py | 54 ++ 4 files changed, 1101 insertions(+), 7 deletions(-) create mode 100644 tests/common_utils.py create mode 100644 tests/expecttest.py create mode 100644 tests/test_segmentation_mask.py diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 2ec2905f7..16f43a79a 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -21,10 +21,7 @@ def __init__(self, segm, size, mode): else: if type(segm) == list: # polygons - rle = mask_utils.frPyObjects(segm, height, width) - mask = np.array(mask_utils.decode(rle), dtype=np.float32) - mask = np.sum(mask, axis=2) - mask = torch.from_numpy(np.array(mask > 0, dtype=np.float32)) + mask = Polygons(segm, size, 'polygon').convert('mask').to(dtype=torch.float32) elif type(segm) == dict and 'counts' in segm: if type(segm['counts']) == list: # uncompressed RLE @@ -41,7 +38,7 @@ def __init__(self, segm, size, mode): if type(segm) == np.ndarray: mask = torch.from_numpy(segm).to(dtype=torch.float32) else: # torch.Tensor - mask = segm + mask = segm.to(dtype=torch.float32) self.mask = mask self.size = size self.mode = mode @@ -65,7 +62,9 @@ def transpose(self, method): def crop(self, box): box = [int(b) for b in box] w, h = box[2] - box[0], box[3] - box[1] - cropped_mask = self.mask[box[1]: box[3]+1, box[0]: box[2]+1] + w = max(w, 1) + h = max(h, 1) + cropped_mask = self.mask[box[1]: box[3], box[0]: box[2]] return Mask(cropped_mask, size=(w, h), mode=self.mode) def resize(self, size, *args, **kwargs): @@ -74,7 +73,7 @@ def resize(self, size, *args, **kwargs): return Mask(scaled_mask, size=size, mode=self.mode) def convert(self, mode): - mask = self.mask + mask = self.mask.to(dtype=torch.uint8) return mask def __iter__(self): diff --git a/tests/common_utils.py b/tests/common_utils.py new file mode 100644 index 000000000..2a3acc694 --- /dev/null +++ b/tests/common_utils.py @@ -0,0 +1,839 @@ +r"""Importing this file must **not** initialize CUDA context. test_distributed +relies on this assumption to properly run. This means that when this is imported +no CUDA calls shall be made, including torch.cuda.device_count(), etc. + +common_cuda.py can freely initialize CUDA context when imported. +""" + +import sys +import os +import platform +import re +import gc +import types +import inspect +import argparse +import unittest +import warnings +import random +import contextlib +import socket +from collections import OrderedDict +from functools import wraps +from itertools import product +from copy import deepcopy +from numbers import Number + +import __main__ +import errno + +import expecttest +import hashlib + +import torch +import torch.cuda +from torch._utils_internal import get_writable_path +from torch._six import string_classes, inf +import torch.backends.cudnn +import torch.backends.mkl + + +torch.set_default_tensor_type('torch.DoubleTensor') +torch.backends.cudnn.disable_global_flags() + + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument('--seed', type=int, default=1234) +parser.add_argument('--accept', action='store_true') +args, remaining = parser.parse_known_args() +SEED = args.seed +if not expecttest.ACCEPT: + expecttest.ACCEPT = args.accept +UNITTEST_ARGS = [sys.argv[0]] + remaining +torch.manual_seed(SEED) + + +def run_tests(argv=UNITTEST_ARGS): + unittest.main(argv=argv) + +PY3 = sys.version_info > (3, 0) +PY34 = sys.version_info >= (3, 4) + +IS_WINDOWS = sys.platform == "win32" +IS_PPC = platform.machine() == "ppc64le" + + +def _check_module_exists(name): + r"""Returns if a top-level module with :attr:`name` exists *without** + importing it. This is generally safer than try-catch block around a + `import X`. It avoids third party libraries breaking assumptions of some of + our tests, e.g., setting multiprocessing start method when imported + (see librosa/#747, torchvision/#544). + """ + if not PY3: # Python 2 + import imp + try: + imp.find_module(name) + return True + except ImportError: + return False + elif not PY34: # Python [3, 3.4) + import importlib + loader = importlib.find_loader(name) + return loader is not None + else: # Python >= 3.4 + import importlib + import importlib.util + spec = importlib.util.find_spec(name) + return spec is not None + +TEST_NUMPY = _check_module_exists('numpy') +TEST_SCIPY = _check_module_exists('scipy') +TEST_MKL = torch.backends.mkl.is_available() +TEST_NUMBA = _check_module_exists('numba') + +# On Py2, importing librosa 0.6.1 triggers a TypeError (if using newest joblib) +# see librosa/librosa#729. +# TODO: allow Py2 when librosa 0.6.2 releases +TEST_LIBROSA = _check_module_exists('librosa') and PY3 + +# Python 2.7 doesn't have spawn +NO_MULTIPROCESSING_SPAWN = os.environ.get('NO_MULTIPROCESSING_SPAWN', '0') == '1' or sys.version_info[0] == 2 +TEST_WITH_ASAN = os.getenv('PYTORCH_TEST_WITH_ASAN', '0') == '1' +TEST_WITH_UBSAN = os.getenv('PYTORCH_TEST_WITH_UBSAN', '0') == '1' +TEST_WITH_ROCM = os.getenv('PYTORCH_TEST_WITH_ROCM', '0') == '1' + +if TEST_NUMPY: + import numpy + + +def skipIfRocm(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if TEST_WITH_ROCM: + raise unittest.SkipTest("test doesn't currently work on the ROCm stack") + else: + fn(*args, **kwargs) + return wrapper + + +def skipIfNoLapack(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + if not torch._C.has_lapack: + raise unittest.SkipTest('PyTorch compiled without Lapack') + else: + fn(*args, **kwargs) + return wrapper + + +def skipCUDAMemoryLeakCheckIf(condition): + def dec(fn): + if getattr(fn, '_do_cuda_memory_leak_check', True): # if current True + fn._do_cuda_memory_leak_check = not condition + return fn + return dec + + +def suppress_warnings(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fn(*args, **kwargs) + return wrapper + + +def get_cpu_type(type_name): + module, name = type_name.rsplit('.', 1) + assert module == 'torch.cuda' + return getattr(torch, name) + + +def get_gpu_type(type_name): + if isinstance(type_name, type): + type_name = '{}.{}'.format(type_name.__module__, type_name.__name__) + module, name = type_name.rsplit('.', 1) + assert module == 'torch' + return getattr(torch.cuda, name) + + +def to_gpu(obj, type_map={}): + if isinstance(obj, torch.Tensor): + assert obj.is_leaf + t = type_map.get(obj.type(), get_gpu_type(obj.type())) + with torch.no_grad(): + res = obj.clone().type(t) + res.requires_grad = obj.requires_grad + return res + elif torch.is_storage(obj): + return obj.new().resize_(obj.size()).copy_(obj) + elif isinstance(obj, list): + return [to_gpu(o, type_map) for o in obj] + elif isinstance(obj, tuple): + return tuple(to_gpu(o, type_map) for o in obj) + else: + return deepcopy(obj) + + +def get_function_arglist(func): + return inspect.getargspec(func).args + + +def set_rng_seed(seed): + torch.manual_seed(seed) + random.seed(seed) + if TEST_NUMPY: + numpy.random.seed(seed) + + +@contextlib.contextmanager +def freeze_rng_state(): + rng_state = torch.get_rng_state() + if torch.cuda.is_available(): + cuda_rng_state = torch.cuda.get_rng_state() + yield + if torch.cuda.is_available(): + torch.cuda.set_rng_state(cuda_rng_state) + torch.set_rng_state(rng_state) + + +def iter_indices(tensor): + if tensor.dim() == 0: + return range(0) + if tensor.dim() == 1: + return range(tensor.size(0)) + return product(*(range(s) for s in tensor.size())) + + +def is_iterable(obj): + try: + iter(obj) + return True + except TypeError: + return False + + +class CudaMemoryLeakCheck(): + def __init__(self, testcase, name=None): + self.name = testcase.id() if name is None else name + self.testcase = testcase + + # initialize context & RNG to prevent false positive detections + # when the test is the first to initialize those + from common_cuda import initialize_cuda_context_rng + initialize_cuda_context_rng() + + @staticmethod + def get_cuda_memory_usage(): + # we don't need CUDA synchronize because the statistics are not tracked at + # actual freeing, but at when marking the block as free. + num_devices = torch.cuda.device_count() + gc.collect() + return tuple(torch.cuda.memory_allocated(i) for i in range(num_devices)) + + def __enter__(self): + self.befores = self.get_cuda_memory_usage() + + def __exit__(self, exec_type, exec_value, traceback): + # Don't check for leaks if an exception was thrown + if exec_type is not None: + return + afters = self.get_cuda_memory_usage() + for i, (before, after) in enumerate(zip(self.befores, afters)): + self.testcase.assertEqual( + before, after, '{} leaked {} bytes CUDA memory on device {}'.format( + self.name, after - before, i)) + + +class TestCase(expecttest.TestCase): + precision = 1e-5 + maxDiff = None + _do_cuda_memory_leak_check = False + + def __init__(self, method_name='runTest'): + super(TestCase, self).__init__(method_name) + # Wraps the tested method if we should do CUDA memory check. + test_method = getattr(self, method_name) + self._do_cuda_memory_leak_check &= getattr(test_method, '_do_cuda_memory_leak_check', True) + # FIXME: figure out the flaky -1024 anti-leaks on windows. See #8044 + if self._do_cuda_memory_leak_check and not IS_WINDOWS: + # the import below may initialize CUDA context, so we do it only if + # self._do_cuda_memory_leak_check is True. + from common_cuda import TEST_CUDA + fullname = self.id().lower() # class_name.method_name + if TEST_CUDA and ('gpu' in fullname or 'cuda' in fullname): + setattr(self, method_name, self.wrap_with_cuda_memory_check(test_method)) + + def assertLeaksNoCudaTensors(self, name=None): + name = self.id() if name is None else name + return CudaMemoryLeakCheck(self, name) + + def wrap_with_cuda_memory_check(self, method): + # Assumes that `method` is the tested function in `self`. + # NOTE: Python Exceptions (e.g., unittest.Skip) keeps objects in scope + # alive, so this cannot be done in setUp and tearDown because + # tearDown is run unconditionally no matter whether the test + # passes or not. For the same reason, we can't wrap the `method` + # call in try-finally and always do the check. + @wraps(method) + def wrapper(self, *args, **kwargs): + with self.assertLeaksNoCudaTensors(): + method(*args, **kwargs) + return types.MethodType(wrapper, self) + + def setUp(self): + set_rng_seed(SEED) + + def assertTensorsSlowEqual(self, x, y, prec=None, message=''): + max_err = 0 + self.assertEqual(x.size(), y.size()) + for index in iter_indices(x): + max_err = max(max_err, abs(x[index] - y[index])) + self.assertLessEqual(max_err, prec, message) + + def genSparseTensor(self, size, sparse_dim, nnz, is_uncoalesced, device='cpu'): + # Assert not given impossible combination, where the sparse dims have + # empty numel, but nnz > 0 makes the indices containing values. + assert all(size[d] > 0 for d in range(sparse_dim)) or nnz == 0, 'invalid arguments' + + v_size = [nnz] + list(size[sparse_dim:]) + v = torch.randn(*v_size, device=device) + i = torch.rand(sparse_dim, nnz, device=device) + i.mul_(torch.tensor(size[:sparse_dim]).unsqueeze(1).to(i)) + i = i.to(torch.long) + if is_uncoalesced: + v = torch.cat([v, torch.randn_like(v)], 0) + i = torch.cat([i, i], 1) + + x = torch.sparse_coo_tensor(i, v, torch.Size(size)) + + if not is_uncoalesced: + x = x.coalesce() + else: + # FIXME: `x` is a sparse view of `v`. Currently rebase_history for + # sparse views is not implemented, so this workaround is + # needed for inplace operations done on `x`, e.g., copy_(). + # Remove after implementing something equivalent to CopySlice + # for sparse views. + x = x.detach() + return x, x._indices().clone(), x._values().clone() + + def safeToDense(self, t): + r = self.safeCoalesce(t) + return r.to_dense() + + def safeCoalesce(self, t): + tc = t.coalesce() + self.assertEqual(tc.to_dense(), t.to_dense()) + self.assertTrue(tc.is_coalesced()) + + # Our code below doesn't work when nnz is 0, because + # then it's a 0D tensor, not a 2D tensor. + if t._nnz() == 0: + self.assertEqual(t._indices(), tc._indices()) + self.assertEqual(t._values(), tc._values()) + return tc + + value_map = {} + for idx, val in zip(t._indices().t(), t._values()): + idx_tup = tuple(idx.tolist()) + if idx_tup in value_map: + value_map[idx_tup] += val + else: + value_map[idx_tup] = val.clone() if isinstance(val, torch.Tensor) else val + + new_indices = sorted(list(value_map.keys())) + new_values = [value_map[idx] for idx in new_indices] + if t._values().ndimension() < 2: + new_values = t._values().new(new_values) + else: + new_values = torch.stack(new_values) + + new_indices = t._indices().new(new_indices).t() + tg = t.new(new_indices, new_values, t.size()) + + self.assertEqual(tc._indices(), tg._indices()) + self.assertEqual(tc._values(), tg._values()) + + if t.is_coalesced(): + self.assertEqual(tc._indices(), t._indices()) + self.assertEqual(tc._values(), t._values()) + + return tg + + def assertEqual(self, x, y, prec=None, message='', allow_inf=False): + if isinstance(prec, str) and message == '': + message = prec + prec = None + if prec is None: + prec = self.precision + + if isinstance(x, torch.Tensor) and isinstance(y, Number): + self.assertEqual(x.item(), y, prec, message, allow_inf) + elif isinstance(y, torch.Tensor) and isinstance(x, Number): + self.assertEqual(x, y.item(), prec, message, allow_inf) + elif isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor): + def assertTensorsEqual(a, b): + super(TestCase, self).assertEqual(a.size(), b.size(), message) + if a.numel() > 0: + b = b.type_as(a) + b = b.cuda(device=a.get_device()) if a.is_cuda else b.cpu() + # check that NaNs are in the same locations + nan_mask = a != a + self.assertTrue(torch.equal(nan_mask, b != b), message) + diff = a - b + diff[nan_mask] = 0 + # inf check if allow_inf=True + if allow_inf: + inf_mask = (a == float("inf")) | (a == float("-inf")) + self.assertTrue(torch.equal(inf_mask, + (b == float("inf")) | (b == float("-inf"))), + message) + diff[inf_mask] = 0 + # TODO: implement abs on CharTensor + if diff.is_signed() and 'CharTensor' not in diff.type(): + diff = diff.abs() + max_err = diff.max() + self.assertLessEqual(max_err, prec, message) + super(TestCase, self).assertEqual(x.is_sparse, y.is_sparse, message) + if x.is_sparse: + x = self.safeCoalesce(x) + y = self.safeCoalesce(y) + assertTensorsEqual(x._indices(), y._indices()) + assertTensorsEqual(x._values(), y._values()) + else: + assertTensorsEqual(x, y) + elif isinstance(x, string_classes) and isinstance(y, string_classes): + super(TestCase, self).assertEqual(x, y, message) + elif type(x) == set and type(y) == set: + super(TestCase, self).assertEqual(x, y, message) + elif isinstance(x, dict) and isinstance(y, dict): + if isinstance(x, OrderedDict) and isinstance(y, OrderedDict): + self.assertEqual(x.items(), y.items()) + else: + self.assertEqual(set(x.keys()), set(y.keys())) + key_list = list(x.keys()) + self.assertEqual([x[k] for k in key_list], [y[k] for k in key_list]) + elif is_iterable(x) and is_iterable(y): + super(TestCase, self).assertEqual(len(x), len(y), message) + for x_, y_ in zip(x, y): + self.assertEqual(x_, y_, prec, message) + elif isinstance(x, bool) and isinstance(y, bool): + super(TestCase, self).assertEqual(x, y, message) + elif isinstance(x, Number) and isinstance(y, Number): + if abs(x) == inf or abs(y) == inf: + if allow_inf: + super(TestCase, self).assertEqual(x, y, message) + else: + self.fail("Expected finite numeric values - x={}, y={}".format(x, y)) + return + super(TestCase, self).assertLessEqual(abs(x - y), prec, message) + else: + super(TestCase, self).assertEqual(x, y, message) + + def assertAlmostEqual(self, x, y, places=None, msg=None, delta=None, allow_inf=None): + prec = delta + if places: + prec = 10**(-places) + self.assertEqual(x, y, prec, msg, allow_inf) + + def assertNotEqual(self, x, y, prec=None, message=''): + if isinstance(prec, str) and message == '': + message = prec + prec = None + if prec is None: + prec = self.precision + + if isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor): + if x.size() != y.size(): + super(TestCase, self).assertNotEqual(x.size(), y.size()) + self.assertGreater(x.numel(), 0) + y = y.type_as(x) + y = y.cuda(device=x.get_device()) if x.is_cuda else y.cpu() + nan_mask = x != x + if torch.equal(nan_mask, y != y): + diff = x - y + if diff.is_signed(): + diff = diff.abs() + diff[nan_mask] = 0 + max_err = diff.max() + self.assertGreaterEqual(max_err, prec, message) + elif type(x) == str and type(y) == str: + super(TestCase, self).assertNotEqual(x, y) + elif is_iterable(x) and is_iterable(y): + super(TestCase, self).assertNotEqual(x, y) + else: + try: + self.assertGreaterEqual(abs(x - y), prec, message) + return + except (TypeError, AssertionError): + pass + super(TestCase, self).assertNotEqual(x, y, message) + + def assertObjectIn(self, obj, iterable): + for elem in iterable: + if id(obj) == id(elem): + return + raise AssertionError("object not found in iterable") + + # TODO: Support context manager interface + # NB: The kwargs forwarding to callable robs the 'subname' parameter. + # If you need it, manually apply your callable in a lambda instead. + def assertExpectedRaises(self, exc_type, callable, *args, **kwargs): + subname = None + if 'subname' in kwargs: + subname = kwargs['subname'] + del kwargs['subname'] + try: + callable(*args, **kwargs) + except exc_type as e: + self.assertExpected(str(e), subname) + return + # Don't put this in the try block; the AssertionError will catch it + self.fail(msg="Did not raise when expected to") + + def assertWarns(self, callable, msg=''): + r""" + Test if :attr:`callable` raises a warning. + """ + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") # allow any warning to be raised + callable() + self.assertTrue(len(ws) > 0, msg) + + def assertWarnsRegex(self, callable, regex, msg=''): + r""" + Test if :attr:`callable` raises any warning with message that contains + the regex pattern :attr:`regex`. + """ + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") # allow any warning to be raised + callable() + self.assertTrue(len(ws) > 0, msg) + found = any(re.search(regex, str(w.message)) is not None for w in ws) + self.assertTrue(found, msg) + + def assertExpected(self, s, subname=None): + r""" + Test that a string matches the recorded contents of a file + derived from the name of this test and subname. This file + is placed in the 'expect' directory in the same directory + as the test script. You can automatically update the recorded test + output using --accept. + + If you call this multiple times in a single function, you must + give a unique subname each time. + """ + if not (isinstance(s, str) or (sys.version_info[0] == 2 and isinstance(s, unicode))): + raise TypeError("assertExpected is strings only") + + def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix):] + return text + # NB: we take __file__ from the module that defined the test + # class, so we place the expect directory where the test script + # lives, NOT where test/common_utils.py lives. This doesn't matter in + # PyTorch where all test scripts are in the same directory as + # test/common_utils.py, but it matters in onnx-pytorch + module_id = self.__class__.__module__ + munged_id = remove_prefix(self.id(), module_id + ".") + test_file = os.path.realpath(sys.modules[module_id].__file__) + expected_file = os.path.join(os.path.dirname(test_file), + "expect", + munged_id) + + subname_output = "" + if subname: + expected_file += "-" + subname + subname_output = " ({})".format(subname) + expected_file += ".expect" + expected = None + + def accept_output(update_type): + print("Accepting {} for {}{}:\n\n{}".format(update_type, munged_id, subname_output, s)) + with open(expected_file, 'w') as f: + f.write(s) + + try: + with open(expected_file) as f: + expected = f.read() + except IOError as e: + if e.errno != errno.ENOENT: + raise + elif expecttest.ACCEPT: + return accept_output("output") + else: + raise RuntimeError( + ("I got this output for {}{}:\n\n{}\n\n" + "No expect file exists; to accept the current output, run:\n" + "python {} {} --accept").format(munged_id, subname_output, s, __main__.__file__, munged_id)) + + # a hack for JIT tests + if IS_WINDOWS: + expected = re.sub(r'CppOp\[(.+?)\]', 'CppOp[]', expected) + s = re.sub(r'CppOp\[(.+?)\]', 'CppOp[]', s) + + if expecttest.ACCEPT: + if expected != s: + return accept_output("updated output") + else: + if hasattr(self, "assertMultiLineEqual"): + # Python 2.7 only + # NB: Python considers lhs "old" and rhs "new". + self.assertMultiLineEqual(expected, s) + else: + self.assertEqual(s, expected) + + if sys.version_info < (3, 2): + # assertRegexpMatches renamed to assertRegex in 3.2 + assertRegex = unittest.TestCase.assertRegexpMatches + # assertRaisesRegexp renamed to assertRaisesRegex in 3.2 + assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + + +def download_file(url, binary=True): + if sys.version_info < (3,): + from urlparse import urlsplit + import urllib2 + request = urllib2 + error = urllib2 + else: + from urllib.parse import urlsplit + from urllib import request, error + + filename = os.path.basename(urlsplit(url)[2]) + data_dir = get_writable_path(os.path.join(os.path.dirname(__file__), 'data')) + path = os.path.join(data_dir, filename) + + if os.path.exists(path): + return path + try: + data = request.urlopen(url, timeout=15).read() + with open(path, 'wb' if binary else 'w') as f: + f.write(data) + return path + except error.URLError: + msg = "could not download test file '{}'".format(url) + warnings.warn(msg, RuntimeWarning) + raise unittest.SkipTest(msg) + + +def find_free_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('localhost', 0)) + sockname = sock.getsockname() + sock.close() + return sockname[1] + + +# Methods for matrix generation +# Used in test_autograd.py and test_torch.py +def prod_single_zero(dim_size): + result = torch.randn(dim_size, dim_size) + result[0, 1] = 0 + return result + + +def random_square_matrix_of_rank(l, rank): + assert rank <= l + A = torch.randn(l, l) + u, s, v = A.svd() + for i in range(l): + if i >= rank: + s[i] = 0 + elif s[i] == 0: + s[i] = 1 + return u.mm(torch.diag(s)).mm(v.transpose(0, 1)) + + +def random_symmetric_matrix(l): + A = torch.randn(l, l) + for i in range(l): + for j in range(i): + A[i, j] = A[j, i] + return A + + +def random_symmetric_psd_matrix(l): + A = torch.randn(l, l) + return A.mm(A.transpose(0, 1)) + + +def random_symmetric_pd_matrix(l, *batches): + A = torch.randn(*(batches + (l, l))) + return A.matmul(A.transpose(-2, -1)) + torch.eye(l) * 1e-5 + + +def make_nonzero_det(A, sign=None, min_singular_value=0.1): + u, s, v = A.svd() + s[s < min_singular_value] = min_singular_value + A = u.mm(torch.diag(s)).mm(v.t()) + det = A.det().item() + if sign is not None: + if (det < 0) ^ (sign < 0): + A[0, :].neg_() + return A + + +def random_fullrank_matrix_distinct_singular_value(l, *batches, **kwargs): + silent = kwargs.get("silent", False) + if silent and not torch._C.has_lapack: + return torch.ones(l, l) + + if len(batches) == 0: + A = torch.randn(l, l) + u, _, v = A.svd() + s = torch.arange(1., l + 1).mul_(1.0 / (l + 1)) + return u.mm(torch.diag(s)).mm(v.t()) + else: + all_matrices = [] + for _ in range(0, torch.prod(torch.as_tensor(batches)).item()): + A = torch.randn(l, l) + u, _, v = A.svd() + s = torch.arange(1., l + 1).mul_(1.0 / (l + 1)) + all_matrices.append(u.mm(torch.diag(s)).mm(v.t())) + return torch.stack(all_matrices).reshape(*(batches + (l, l))) + + +def do_test_dtypes(self, dtypes, layout, device): + for dtype in dtypes: + if dtype != torch.float16: + out = torch.zeros((2, 3), dtype=dtype, layout=layout, device=device) + self.assertIs(dtype, out.dtype) + self.assertIs(layout, out.layout) + self.assertEqual(device, out.device) + + +def do_test_empty_full(self, dtypes, layout, device): + shape = torch.Size([2, 3]) + + def check_value(tensor, dtype, layout, device, value, requires_grad): + self.assertEqual(shape, tensor.shape) + self.assertIs(dtype, tensor.dtype) + self.assertIs(layout, tensor.layout) + self.assertEqual(tensor.requires_grad, requires_grad) + if tensor.is_cuda and device is not None: + self.assertEqual(device, tensor.device) + if value is not None: + fill = tensor.new(shape).fill_(value) + self.assertEqual(tensor, fill) + + def get_int64_dtype(dtype): + module = '.'.join(str(dtype).split('.')[1:-1]) + if not module: + return torch.int64 + return operator.attrgetter(module)(torch).int64 + + default_dtype = torch.get_default_dtype() + check_value(torch.empty(shape), default_dtype, torch.strided, -1, None, False) + check_value(torch.full(shape, -5), default_dtype, torch.strided, -1, None, False) + for dtype in dtypes: + for rg in {dtype.is_floating_point, False}: + int64_dtype = get_int64_dtype(dtype) + v = torch.empty(shape, dtype=dtype, device=device, layout=layout, requires_grad=rg) + check_value(v, dtype, layout, device, None, rg) + out = v.new() + check_value(torch.empty(shape, out=out, device=device, layout=layout, requires_grad=rg), + dtype, layout, device, None, rg) + check_value(v.new_empty(shape), dtype, layout, device, None, False) + check_value(v.new_empty(shape, dtype=int64_dtype, device=device, requires_grad=False), + int64_dtype, layout, device, None, False) + check_value(torch.empty_like(v), dtype, layout, device, None, False) + check_value(torch.empty_like(v, dtype=int64_dtype, layout=layout, device=device, requires_grad=False), + int64_dtype, layout, device, None, False) + + if dtype is not torch.float16 and layout != torch.sparse_coo: + fv = 3 + v = torch.full(shape, fv, dtype=dtype, layout=layout, device=device, requires_grad=rg) + check_value(v, dtype, layout, device, fv, rg) + check_value(v.new_full(shape, fv + 1), dtype, layout, device, fv + 1, False) + out = v.new() + check_value(torch.full(shape, fv + 2, out=out, device=device, layout=layout, requires_grad=rg), + dtype, layout, device, fv + 2, rg) + check_value(v.new_full(shape, fv + 3, dtype=int64_dtype, device=device, requires_grad=False), + int64_dtype, layout, device, fv + 3, False) + check_value(torch.full_like(v, fv + 4), dtype, layout, device, fv + 4, False) + check_value(torch.full_like(v, fv + 5, + dtype=int64_dtype, layout=layout, device=device, requires_grad=False), + int64_dtype, layout, device, fv + 5, False) + + +IS_SANDCASTLE = os.getenv('SANDCASTLE') == '1' or os.getenv('TW_JOB_USER') == 'sandcastle' + +THESE_TAKE_WAY_TOO_LONG = { + 'test_Conv3d_groups', + 'test_conv_double_backward_groups', + 'test_Conv3d_dilated', + 'test_Conv3d_stride_padding', + 'test_Conv3d_dilated_strided', + 'test_Conv3d', + 'test_Conv2d_dilated', + 'test_ConvTranspose3d_dilated', + 'test_ConvTranspose2d_dilated', + 'test_snli', + 'test_Conv2d', + 'test_Conv2d_padding', + 'test_ConvTranspose2d_no_bias', + 'test_ConvTranspose2d', + 'test_ConvTranspose3d', + 'test_Conv2d_no_bias', + 'test_matmul_4d_4d', + 'test_multinomial_invalid_probs', +} + + +running_script_path = None + + +def set_running_script_path(): + global running_script_path + try: + running_file = os.path.abspath(os.path.realpath(sys.argv[0])) + if running_file.endswith('.py'): # skip if the running file is not a script + running_script_path = running_file + except Exception: + pass + + +def check_test_defined_in_running_script(test_case): + if running_script_path is None: + return + test_case_class_file = os.path.abspath(os.path.realpath(inspect.getfile(test_case.__class__))) + assert test_case_class_file == running_script_path, "Class of loaded TestCase \"{}\" " \ + "is not defined in the running script \"{}\", but in \"{}\". Did you " \ + "accidentally import a unittest.TestCase from another file?".format( + test_case.id(), running_script_path, test_case_class_file) + + +num_shards = os.environ.get('TEST_NUM_SHARDS', None) +shard = os.environ.get('TEST_SHARD', None) +if num_shards is not None and shard is not None: + num_shards = int(num_shards) + shard = int(shard) + + def load_tests(loader, tests, pattern): + set_running_script_path() + test_suite = unittest.TestSuite() + for test_group in tests: + for test in test_group: + check_test_defined_in_running_script(test) + name = test.id().split('.')[-1] + if name in THESE_TAKE_WAY_TOO_LONG: + continue + hash_id = int(hashlib.sha256(str(test).encode('utf-8')).hexdigest(), 16) + if hash_id % num_shards == shard: + test_suite.addTest(test) + return test_suite +else: + + def load_tests(loader, tests, pattern): + set_running_script_path() + test_suite = unittest.TestSuite() + for test_group in tests: + for test in test_group: + check_test_defined_in_running_script(test) + test_suite.addTest(test) + return test_suite diff --git a/tests/expecttest.py b/tests/expecttest.py new file mode 100644 index 000000000..f6f2649b5 --- /dev/null +++ b/tests/expecttest.py @@ -0,0 +1,202 @@ +import re +import unittest +import traceback +import os +import string + + +ACCEPT = os.getenv('EXPECTTEST_ACCEPT') + + +def nth_line(src, lineno): + """ + Compute the starting index of the n-th line (where n is 1-indexed) + + >>> nth_line("aaa\\nbb\\nc", 2) + 4 + """ + assert lineno >= 1 + pos = 0 + for _ in range(lineno - 1): + pos = src.find('\n', pos) + 1 + return pos + + +def nth_eol(src, lineno): + """ + Compute the ending index of the n-th line (before the newline, + where n is 1-indexed) + + >>> nth_eol("aaa\\nbb\\nc", 2) + 6 + """ + assert lineno >= 1 + pos = -1 + for _ in range(lineno): + pos = src.find('\n', pos + 1) + if pos == -1: + return len(src) + return pos + + +def normalize_nl(t): + return t.replace('\r\n', '\n').replace('\r', '\n') + + +def escape_trailing_quote(s, quote): + if s and s[-1] == quote: + return s[:-1] + '\\' + quote + else: + return s + + +class EditHistory(object): + def __init__(self): + self.state = {} + + def adjust_lineno(self, fn, lineno): + if fn not in self.state: + return lineno + for edit_loc, edit_diff in self.state[fn]: + if lineno > edit_loc: + lineno += edit_diff + return lineno + + def seen_file(self, fn): + return fn in self.state + + def record_edit(self, fn, lineno, delta): + self.state.setdefault(fn, []).append((lineno, delta)) + + +EDIT_HISTORY = EditHistory() + + +def ok_for_raw_triple_quoted_string(s, quote): + """ + Is this string representable inside a raw triple-quoted string? + Due to the fact that backslashes are always treated literally, + some strings are not representable. + + >>> ok_for_raw_triple_quoted_string("blah", quote="'") + True + >>> ok_for_raw_triple_quoted_string("'", quote="'") + False + >>> ok_for_raw_triple_quoted_string("a ''' b", quote="'") + False + """ + return quote * 3 not in s and (not s or s[-1] not in [quote, '\\']) + + +# This operates on the REVERSED string (that's why suffix is first) +RE_EXPECT = re.compile(r"^(?P[^\n]*?)" + r"(?P'''|" r'""")' + r"(?P.*?)" + r"(?P=quote)" + r"(?Pr?)", re.DOTALL) + + +def replace_string_literal(src, lineno, new_string): + r""" + Replace a triple quoted string literal with new contents. + Only handles printable ASCII correctly at the moment. This + will preserve the quote style of the original string, and + makes a best effort to preserve raw-ness (unless it is impossible + to do so.) + + Returns a tuple of the replaced string, as well as a delta of + number of lines added/removed. + + >>> replace_string_literal("'''arf'''", 1, "barf") + ("'''barf'''", 0) + >>> r = replace_string_literal(" moo = '''arf'''", 1, "'a'\n\\b\n") + >>> print(r[0]) + moo = '''\ + 'a' + \\b + ''' + >>> r[1] + 3 + >>> replace_string_literal(" moo = '''\\\narf'''", 2, "'a'\n\\b\n")[1] + 2 + >>> print(replace_string_literal(" f('''\"\"\"''')", 1, "a ''' b")[0]) + f('''a \'\'\' b''') + """ + # Haven't implemented correct escaping for non-printable characters + assert all(c in string.printable for c in new_string) + i = nth_eol(src, lineno) + new_string = normalize_nl(new_string) + + delta = [new_string.count("\n")] + if delta[0] > 0: + delta[0] += 1 # handle the extra \\\n + + def replace(m): + s = new_string + raw = m.group('raw') == 'r' + if not raw or not ok_for_raw_triple_quoted_string(s, quote=m.group('quote')[0]): + raw = False + s = s.replace('\\', '\\\\') + if m.group('quote') == "'''": + s = escape_trailing_quote(s, "'").replace("'''", r"\'\'\'") + else: + s = escape_trailing_quote(s, '"').replace('"""', r'\"\"\"') + + new_body = "\\\n" + s if "\n" in s and not raw else s + delta[0] -= m.group('body').count("\n") + + return ''.join([m.group('suffix'), + m.group('quote'), + new_body[::-1], + m.group('quote'), + 'r' if raw else '', + ]) + + # Having to do this in reverse is very irritating, but it's the + # only way to make the non-greedy matches work correctly. + return (RE_EXPECT.sub(replace, src[:i][::-1], count=1)[::-1] + src[i:], delta[0]) + + +class TestCase(unittest.TestCase): + longMessage = True + + def assertExpectedInline(self, actual, expect, skip=0): + if ACCEPT: + if actual != expect: + # current frame and parent frame, plus any requested skip + tb = traceback.extract_stack(limit=2 + skip) + fn, lineno, _, _ = tb[0] + print("Accepting new output for {} at {}:{}".format(self.id(), fn, lineno)) + with open(fn, 'r+') as f: + old = f.read() + + # compute the change in lineno + lineno = EDIT_HISTORY.adjust_lineno(fn, lineno) + new, delta = replace_string_literal(old, lineno, actual) + + assert old != new, "Failed to substitute string at {}:{}".format(fn, lineno) + + # Only write the backup file the first time we hit the + # file + if not EDIT_HISTORY.seen_file(fn): + with open(fn + ".bak", 'w') as f_bak: + f_bak.write(old) + f.seek(0) + f.truncate(0) + + f.write(new) + + EDIT_HISTORY.record_edit(fn, lineno, delta) + else: + help_text = ("To accept the new output, re-run test with " + "envvar EXPECTTEST_ACCEPT=1 (we recommend " + "staging/committing your changes before doing this)") + if hasattr(self, "assertMultiLineEqual"): + self.assertMultiLineEqual(expect, actual, msg=help_text) + else: + self.assertEqual(expect, actual, msg=help_text) + + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/tests/test_segmentation_mask.py b/tests/test_segmentation_mask.py new file mode 100644 index 000000000..06e32bff4 --- /dev/null +++ b/tests/test_segmentation_mask.py @@ -0,0 +1,54 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import torch +import numpy as np +from common_utils import TestCase, run_tests +from maskrcnn_benchmark.structures.segmentation_mask import Mask, Polygons, SegmentationMask + + +class TestSegmentationMask(TestCase): + def __init__(self, method_name='runTest'): + super(TestSegmentationMask, self).__init__(method_name) + self.poly = [[423.0, 306.5, 406.5, 277.0, 400.0, 271.5, 389.5, 277.0, 387.5, 292.0, + 384.5, 295.0, 374.5, 220.0, 378.5, 210.0, 391.0, 200.5, 404.0, 199.5, + 414.0, 203.5, 425.5, 221.0, 438.5, 297.0, 423.0, 306.5], + [385.5, 240.0, 404.0, 234.5, 419.5, 234.0, 416.5, 219.0, 409.0, 209.5, + 394.0, 207.5, 385.5, 213.0, 382.5, 221.0, 385.5, 240.0]] + self.width = 640 + self.height = 480 + self.size = (self.width, self.height) + self.box = [35, 55, 540, 400] # xyxy + + self.polygon = Polygons(self.poly, self.size, 'polygon') + self.mask = Mask(self.poly, self.size, 'mask') + + def test_crop(self): + poly_crop = self.polygon.crop(self.box) + mask_from_poly_crop = poly_crop.convert('mask') + mask_crop = self.mask.crop(self.box).mask + + self.assertEqual(mask_from_poly_crop, mask_crop) + + def test_convert(self): + mask_from_poly_convert = self.polygon.convert('mask') + mask = self.mask.convert('mask') + self.assertEqual(mask_from_poly_convert, mask) + + def test_transpose(self): + FLIP_LEFT_RIGHT = 0 + FLIP_TOP_BOTTOM = 1 + methods = (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM) + for method in methods: + mask_from_poly_flip = self.polygon.transpose(method).convert('mask') + mask_flip = self.mask.transpose(method).convert('mask') + print(method, torch.abs(mask_flip.float() - mask_from_poly_flip.float()).sum()) + self.assertEqual(mask_flip, mask_from_poly_flip) + + def test_resize(self): + new_size = (600, 500) + mask_from_poly_resize = self.polygon.resize(new_size).convert('mask') + mask_resize = self.mask.resize(new_size).convert('mask') + print('diff resize: ', torch.abs(mask_from_poly_resize.float() - mask_resize.float()).sum()) + self.assertEqual(mask_from_poly_resize, mask_resize) + +if __name__ == "__main__": + run_tests() From 7f22baab128958e07977fd6e1dcb92a097722477 Mon Sep 17 00:00:00 2001 From: wangg12 Date: Wed, 14 Nov 2018 23:25:25 +0800 Subject: [PATCH 06/14] update tests --- tests/common_utils.py | 839 -------------------------------- tests/expecttest.py | 202 -------- tests/test_segmentation_mask.py | 16 +- 3 files changed, 8 insertions(+), 1049 deletions(-) delete mode 100644 tests/common_utils.py delete mode 100644 tests/expecttest.py diff --git a/tests/common_utils.py b/tests/common_utils.py deleted file mode 100644 index 2a3acc694..000000000 --- a/tests/common_utils.py +++ /dev/null @@ -1,839 +0,0 @@ -r"""Importing this file must **not** initialize CUDA context. test_distributed -relies on this assumption to properly run. This means that when this is imported -no CUDA calls shall be made, including torch.cuda.device_count(), etc. - -common_cuda.py can freely initialize CUDA context when imported. -""" - -import sys -import os -import platform -import re -import gc -import types -import inspect -import argparse -import unittest -import warnings -import random -import contextlib -import socket -from collections import OrderedDict -from functools import wraps -from itertools import product -from copy import deepcopy -from numbers import Number - -import __main__ -import errno - -import expecttest -import hashlib - -import torch -import torch.cuda -from torch._utils_internal import get_writable_path -from torch._six import string_classes, inf -import torch.backends.cudnn -import torch.backends.mkl - - -torch.set_default_tensor_type('torch.DoubleTensor') -torch.backends.cudnn.disable_global_flags() - - -parser = argparse.ArgumentParser(add_help=False) -parser.add_argument('--seed', type=int, default=1234) -parser.add_argument('--accept', action='store_true') -args, remaining = parser.parse_known_args() -SEED = args.seed -if not expecttest.ACCEPT: - expecttest.ACCEPT = args.accept -UNITTEST_ARGS = [sys.argv[0]] + remaining -torch.manual_seed(SEED) - - -def run_tests(argv=UNITTEST_ARGS): - unittest.main(argv=argv) - -PY3 = sys.version_info > (3, 0) -PY34 = sys.version_info >= (3, 4) - -IS_WINDOWS = sys.platform == "win32" -IS_PPC = platform.machine() == "ppc64le" - - -def _check_module_exists(name): - r"""Returns if a top-level module with :attr:`name` exists *without** - importing it. This is generally safer than try-catch block around a - `import X`. It avoids third party libraries breaking assumptions of some of - our tests, e.g., setting multiprocessing start method when imported - (see librosa/#747, torchvision/#544). - """ - if not PY3: # Python 2 - import imp - try: - imp.find_module(name) - return True - except ImportError: - return False - elif not PY34: # Python [3, 3.4) - import importlib - loader = importlib.find_loader(name) - return loader is not None - else: # Python >= 3.4 - import importlib - import importlib.util - spec = importlib.util.find_spec(name) - return spec is not None - -TEST_NUMPY = _check_module_exists('numpy') -TEST_SCIPY = _check_module_exists('scipy') -TEST_MKL = torch.backends.mkl.is_available() -TEST_NUMBA = _check_module_exists('numba') - -# On Py2, importing librosa 0.6.1 triggers a TypeError (if using newest joblib) -# see librosa/librosa#729. -# TODO: allow Py2 when librosa 0.6.2 releases -TEST_LIBROSA = _check_module_exists('librosa') and PY3 - -# Python 2.7 doesn't have spawn -NO_MULTIPROCESSING_SPAWN = os.environ.get('NO_MULTIPROCESSING_SPAWN', '0') == '1' or sys.version_info[0] == 2 -TEST_WITH_ASAN = os.getenv('PYTORCH_TEST_WITH_ASAN', '0') == '1' -TEST_WITH_UBSAN = os.getenv('PYTORCH_TEST_WITH_UBSAN', '0') == '1' -TEST_WITH_ROCM = os.getenv('PYTORCH_TEST_WITH_ROCM', '0') == '1' - -if TEST_NUMPY: - import numpy - - -def skipIfRocm(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - if TEST_WITH_ROCM: - raise unittest.SkipTest("test doesn't currently work on the ROCm stack") - else: - fn(*args, **kwargs) - return wrapper - - -def skipIfNoLapack(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - if not torch._C.has_lapack: - raise unittest.SkipTest('PyTorch compiled without Lapack') - else: - fn(*args, **kwargs) - return wrapper - - -def skipCUDAMemoryLeakCheckIf(condition): - def dec(fn): - if getattr(fn, '_do_cuda_memory_leak_check', True): # if current True - fn._do_cuda_memory_leak_check = not condition - return fn - return dec - - -def suppress_warnings(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fn(*args, **kwargs) - return wrapper - - -def get_cpu_type(type_name): - module, name = type_name.rsplit('.', 1) - assert module == 'torch.cuda' - return getattr(torch, name) - - -def get_gpu_type(type_name): - if isinstance(type_name, type): - type_name = '{}.{}'.format(type_name.__module__, type_name.__name__) - module, name = type_name.rsplit('.', 1) - assert module == 'torch' - return getattr(torch.cuda, name) - - -def to_gpu(obj, type_map={}): - if isinstance(obj, torch.Tensor): - assert obj.is_leaf - t = type_map.get(obj.type(), get_gpu_type(obj.type())) - with torch.no_grad(): - res = obj.clone().type(t) - res.requires_grad = obj.requires_grad - return res - elif torch.is_storage(obj): - return obj.new().resize_(obj.size()).copy_(obj) - elif isinstance(obj, list): - return [to_gpu(o, type_map) for o in obj] - elif isinstance(obj, tuple): - return tuple(to_gpu(o, type_map) for o in obj) - else: - return deepcopy(obj) - - -def get_function_arglist(func): - return inspect.getargspec(func).args - - -def set_rng_seed(seed): - torch.manual_seed(seed) - random.seed(seed) - if TEST_NUMPY: - numpy.random.seed(seed) - - -@contextlib.contextmanager -def freeze_rng_state(): - rng_state = torch.get_rng_state() - if torch.cuda.is_available(): - cuda_rng_state = torch.cuda.get_rng_state() - yield - if torch.cuda.is_available(): - torch.cuda.set_rng_state(cuda_rng_state) - torch.set_rng_state(rng_state) - - -def iter_indices(tensor): - if tensor.dim() == 0: - return range(0) - if tensor.dim() == 1: - return range(tensor.size(0)) - return product(*(range(s) for s in tensor.size())) - - -def is_iterable(obj): - try: - iter(obj) - return True - except TypeError: - return False - - -class CudaMemoryLeakCheck(): - def __init__(self, testcase, name=None): - self.name = testcase.id() if name is None else name - self.testcase = testcase - - # initialize context & RNG to prevent false positive detections - # when the test is the first to initialize those - from common_cuda import initialize_cuda_context_rng - initialize_cuda_context_rng() - - @staticmethod - def get_cuda_memory_usage(): - # we don't need CUDA synchronize because the statistics are not tracked at - # actual freeing, but at when marking the block as free. - num_devices = torch.cuda.device_count() - gc.collect() - return tuple(torch.cuda.memory_allocated(i) for i in range(num_devices)) - - def __enter__(self): - self.befores = self.get_cuda_memory_usage() - - def __exit__(self, exec_type, exec_value, traceback): - # Don't check for leaks if an exception was thrown - if exec_type is not None: - return - afters = self.get_cuda_memory_usage() - for i, (before, after) in enumerate(zip(self.befores, afters)): - self.testcase.assertEqual( - before, after, '{} leaked {} bytes CUDA memory on device {}'.format( - self.name, after - before, i)) - - -class TestCase(expecttest.TestCase): - precision = 1e-5 - maxDiff = None - _do_cuda_memory_leak_check = False - - def __init__(self, method_name='runTest'): - super(TestCase, self).__init__(method_name) - # Wraps the tested method if we should do CUDA memory check. - test_method = getattr(self, method_name) - self._do_cuda_memory_leak_check &= getattr(test_method, '_do_cuda_memory_leak_check', True) - # FIXME: figure out the flaky -1024 anti-leaks on windows. See #8044 - if self._do_cuda_memory_leak_check and not IS_WINDOWS: - # the import below may initialize CUDA context, so we do it only if - # self._do_cuda_memory_leak_check is True. - from common_cuda import TEST_CUDA - fullname = self.id().lower() # class_name.method_name - if TEST_CUDA and ('gpu' in fullname or 'cuda' in fullname): - setattr(self, method_name, self.wrap_with_cuda_memory_check(test_method)) - - def assertLeaksNoCudaTensors(self, name=None): - name = self.id() if name is None else name - return CudaMemoryLeakCheck(self, name) - - def wrap_with_cuda_memory_check(self, method): - # Assumes that `method` is the tested function in `self`. - # NOTE: Python Exceptions (e.g., unittest.Skip) keeps objects in scope - # alive, so this cannot be done in setUp and tearDown because - # tearDown is run unconditionally no matter whether the test - # passes or not. For the same reason, we can't wrap the `method` - # call in try-finally and always do the check. - @wraps(method) - def wrapper(self, *args, **kwargs): - with self.assertLeaksNoCudaTensors(): - method(*args, **kwargs) - return types.MethodType(wrapper, self) - - def setUp(self): - set_rng_seed(SEED) - - def assertTensorsSlowEqual(self, x, y, prec=None, message=''): - max_err = 0 - self.assertEqual(x.size(), y.size()) - for index in iter_indices(x): - max_err = max(max_err, abs(x[index] - y[index])) - self.assertLessEqual(max_err, prec, message) - - def genSparseTensor(self, size, sparse_dim, nnz, is_uncoalesced, device='cpu'): - # Assert not given impossible combination, where the sparse dims have - # empty numel, but nnz > 0 makes the indices containing values. - assert all(size[d] > 0 for d in range(sparse_dim)) or nnz == 0, 'invalid arguments' - - v_size = [nnz] + list(size[sparse_dim:]) - v = torch.randn(*v_size, device=device) - i = torch.rand(sparse_dim, nnz, device=device) - i.mul_(torch.tensor(size[:sparse_dim]).unsqueeze(1).to(i)) - i = i.to(torch.long) - if is_uncoalesced: - v = torch.cat([v, torch.randn_like(v)], 0) - i = torch.cat([i, i], 1) - - x = torch.sparse_coo_tensor(i, v, torch.Size(size)) - - if not is_uncoalesced: - x = x.coalesce() - else: - # FIXME: `x` is a sparse view of `v`. Currently rebase_history for - # sparse views is not implemented, so this workaround is - # needed for inplace operations done on `x`, e.g., copy_(). - # Remove after implementing something equivalent to CopySlice - # for sparse views. - x = x.detach() - return x, x._indices().clone(), x._values().clone() - - def safeToDense(self, t): - r = self.safeCoalesce(t) - return r.to_dense() - - def safeCoalesce(self, t): - tc = t.coalesce() - self.assertEqual(tc.to_dense(), t.to_dense()) - self.assertTrue(tc.is_coalesced()) - - # Our code below doesn't work when nnz is 0, because - # then it's a 0D tensor, not a 2D tensor. - if t._nnz() == 0: - self.assertEqual(t._indices(), tc._indices()) - self.assertEqual(t._values(), tc._values()) - return tc - - value_map = {} - for idx, val in zip(t._indices().t(), t._values()): - idx_tup = tuple(idx.tolist()) - if idx_tup in value_map: - value_map[idx_tup] += val - else: - value_map[idx_tup] = val.clone() if isinstance(val, torch.Tensor) else val - - new_indices = sorted(list(value_map.keys())) - new_values = [value_map[idx] for idx in new_indices] - if t._values().ndimension() < 2: - new_values = t._values().new(new_values) - else: - new_values = torch.stack(new_values) - - new_indices = t._indices().new(new_indices).t() - tg = t.new(new_indices, new_values, t.size()) - - self.assertEqual(tc._indices(), tg._indices()) - self.assertEqual(tc._values(), tg._values()) - - if t.is_coalesced(): - self.assertEqual(tc._indices(), t._indices()) - self.assertEqual(tc._values(), t._values()) - - return tg - - def assertEqual(self, x, y, prec=None, message='', allow_inf=False): - if isinstance(prec, str) and message == '': - message = prec - prec = None - if prec is None: - prec = self.precision - - if isinstance(x, torch.Tensor) and isinstance(y, Number): - self.assertEqual(x.item(), y, prec, message, allow_inf) - elif isinstance(y, torch.Tensor) and isinstance(x, Number): - self.assertEqual(x, y.item(), prec, message, allow_inf) - elif isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor): - def assertTensorsEqual(a, b): - super(TestCase, self).assertEqual(a.size(), b.size(), message) - if a.numel() > 0: - b = b.type_as(a) - b = b.cuda(device=a.get_device()) if a.is_cuda else b.cpu() - # check that NaNs are in the same locations - nan_mask = a != a - self.assertTrue(torch.equal(nan_mask, b != b), message) - diff = a - b - diff[nan_mask] = 0 - # inf check if allow_inf=True - if allow_inf: - inf_mask = (a == float("inf")) | (a == float("-inf")) - self.assertTrue(torch.equal(inf_mask, - (b == float("inf")) | (b == float("-inf"))), - message) - diff[inf_mask] = 0 - # TODO: implement abs on CharTensor - if diff.is_signed() and 'CharTensor' not in diff.type(): - diff = diff.abs() - max_err = diff.max() - self.assertLessEqual(max_err, prec, message) - super(TestCase, self).assertEqual(x.is_sparse, y.is_sparse, message) - if x.is_sparse: - x = self.safeCoalesce(x) - y = self.safeCoalesce(y) - assertTensorsEqual(x._indices(), y._indices()) - assertTensorsEqual(x._values(), y._values()) - else: - assertTensorsEqual(x, y) - elif isinstance(x, string_classes) and isinstance(y, string_classes): - super(TestCase, self).assertEqual(x, y, message) - elif type(x) == set and type(y) == set: - super(TestCase, self).assertEqual(x, y, message) - elif isinstance(x, dict) and isinstance(y, dict): - if isinstance(x, OrderedDict) and isinstance(y, OrderedDict): - self.assertEqual(x.items(), y.items()) - else: - self.assertEqual(set(x.keys()), set(y.keys())) - key_list = list(x.keys()) - self.assertEqual([x[k] for k in key_list], [y[k] for k in key_list]) - elif is_iterable(x) and is_iterable(y): - super(TestCase, self).assertEqual(len(x), len(y), message) - for x_, y_ in zip(x, y): - self.assertEqual(x_, y_, prec, message) - elif isinstance(x, bool) and isinstance(y, bool): - super(TestCase, self).assertEqual(x, y, message) - elif isinstance(x, Number) and isinstance(y, Number): - if abs(x) == inf or abs(y) == inf: - if allow_inf: - super(TestCase, self).assertEqual(x, y, message) - else: - self.fail("Expected finite numeric values - x={}, y={}".format(x, y)) - return - super(TestCase, self).assertLessEqual(abs(x - y), prec, message) - else: - super(TestCase, self).assertEqual(x, y, message) - - def assertAlmostEqual(self, x, y, places=None, msg=None, delta=None, allow_inf=None): - prec = delta - if places: - prec = 10**(-places) - self.assertEqual(x, y, prec, msg, allow_inf) - - def assertNotEqual(self, x, y, prec=None, message=''): - if isinstance(prec, str) and message == '': - message = prec - prec = None - if prec is None: - prec = self.precision - - if isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor): - if x.size() != y.size(): - super(TestCase, self).assertNotEqual(x.size(), y.size()) - self.assertGreater(x.numel(), 0) - y = y.type_as(x) - y = y.cuda(device=x.get_device()) if x.is_cuda else y.cpu() - nan_mask = x != x - if torch.equal(nan_mask, y != y): - diff = x - y - if diff.is_signed(): - diff = diff.abs() - diff[nan_mask] = 0 - max_err = diff.max() - self.assertGreaterEqual(max_err, prec, message) - elif type(x) == str and type(y) == str: - super(TestCase, self).assertNotEqual(x, y) - elif is_iterable(x) and is_iterable(y): - super(TestCase, self).assertNotEqual(x, y) - else: - try: - self.assertGreaterEqual(abs(x - y), prec, message) - return - except (TypeError, AssertionError): - pass - super(TestCase, self).assertNotEqual(x, y, message) - - def assertObjectIn(self, obj, iterable): - for elem in iterable: - if id(obj) == id(elem): - return - raise AssertionError("object not found in iterable") - - # TODO: Support context manager interface - # NB: The kwargs forwarding to callable robs the 'subname' parameter. - # If you need it, manually apply your callable in a lambda instead. - def assertExpectedRaises(self, exc_type, callable, *args, **kwargs): - subname = None - if 'subname' in kwargs: - subname = kwargs['subname'] - del kwargs['subname'] - try: - callable(*args, **kwargs) - except exc_type as e: - self.assertExpected(str(e), subname) - return - # Don't put this in the try block; the AssertionError will catch it - self.fail(msg="Did not raise when expected to") - - def assertWarns(self, callable, msg=''): - r""" - Test if :attr:`callable` raises a warning. - """ - with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter("always") # allow any warning to be raised - callable() - self.assertTrue(len(ws) > 0, msg) - - def assertWarnsRegex(self, callable, regex, msg=''): - r""" - Test if :attr:`callable` raises any warning with message that contains - the regex pattern :attr:`regex`. - """ - with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter("always") # allow any warning to be raised - callable() - self.assertTrue(len(ws) > 0, msg) - found = any(re.search(regex, str(w.message)) is not None for w in ws) - self.assertTrue(found, msg) - - def assertExpected(self, s, subname=None): - r""" - Test that a string matches the recorded contents of a file - derived from the name of this test and subname. This file - is placed in the 'expect' directory in the same directory - as the test script. You can automatically update the recorded test - output using --accept. - - If you call this multiple times in a single function, you must - give a unique subname each time. - """ - if not (isinstance(s, str) or (sys.version_info[0] == 2 and isinstance(s, unicode))): - raise TypeError("assertExpected is strings only") - - def remove_prefix(text, prefix): - if text.startswith(prefix): - return text[len(prefix):] - return text - # NB: we take __file__ from the module that defined the test - # class, so we place the expect directory where the test script - # lives, NOT where test/common_utils.py lives. This doesn't matter in - # PyTorch where all test scripts are in the same directory as - # test/common_utils.py, but it matters in onnx-pytorch - module_id = self.__class__.__module__ - munged_id = remove_prefix(self.id(), module_id + ".") - test_file = os.path.realpath(sys.modules[module_id].__file__) - expected_file = os.path.join(os.path.dirname(test_file), - "expect", - munged_id) - - subname_output = "" - if subname: - expected_file += "-" + subname - subname_output = " ({})".format(subname) - expected_file += ".expect" - expected = None - - def accept_output(update_type): - print("Accepting {} for {}{}:\n\n{}".format(update_type, munged_id, subname_output, s)) - with open(expected_file, 'w') as f: - f.write(s) - - try: - with open(expected_file) as f: - expected = f.read() - except IOError as e: - if e.errno != errno.ENOENT: - raise - elif expecttest.ACCEPT: - return accept_output("output") - else: - raise RuntimeError( - ("I got this output for {}{}:\n\n{}\n\n" - "No expect file exists; to accept the current output, run:\n" - "python {} {} --accept").format(munged_id, subname_output, s, __main__.__file__, munged_id)) - - # a hack for JIT tests - if IS_WINDOWS: - expected = re.sub(r'CppOp\[(.+?)\]', 'CppOp[]', expected) - s = re.sub(r'CppOp\[(.+?)\]', 'CppOp[]', s) - - if expecttest.ACCEPT: - if expected != s: - return accept_output("updated output") - else: - if hasattr(self, "assertMultiLineEqual"): - # Python 2.7 only - # NB: Python considers lhs "old" and rhs "new". - self.assertMultiLineEqual(expected, s) - else: - self.assertEqual(s, expected) - - if sys.version_info < (3, 2): - # assertRegexpMatches renamed to assertRegex in 3.2 - assertRegex = unittest.TestCase.assertRegexpMatches - # assertRaisesRegexp renamed to assertRaisesRegex in 3.2 - assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - - -def download_file(url, binary=True): - if sys.version_info < (3,): - from urlparse import urlsplit - import urllib2 - request = urllib2 - error = urllib2 - else: - from urllib.parse import urlsplit - from urllib import request, error - - filename = os.path.basename(urlsplit(url)[2]) - data_dir = get_writable_path(os.path.join(os.path.dirname(__file__), 'data')) - path = os.path.join(data_dir, filename) - - if os.path.exists(path): - return path - try: - data = request.urlopen(url, timeout=15).read() - with open(path, 'wb' if binary else 'w') as f: - f.write(data) - return path - except error.URLError: - msg = "could not download test file '{}'".format(url) - warnings.warn(msg, RuntimeWarning) - raise unittest.SkipTest(msg) - - -def find_free_port(): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(('localhost', 0)) - sockname = sock.getsockname() - sock.close() - return sockname[1] - - -# Methods for matrix generation -# Used in test_autograd.py and test_torch.py -def prod_single_zero(dim_size): - result = torch.randn(dim_size, dim_size) - result[0, 1] = 0 - return result - - -def random_square_matrix_of_rank(l, rank): - assert rank <= l - A = torch.randn(l, l) - u, s, v = A.svd() - for i in range(l): - if i >= rank: - s[i] = 0 - elif s[i] == 0: - s[i] = 1 - return u.mm(torch.diag(s)).mm(v.transpose(0, 1)) - - -def random_symmetric_matrix(l): - A = torch.randn(l, l) - for i in range(l): - for j in range(i): - A[i, j] = A[j, i] - return A - - -def random_symmetric_psd_matrix(l): - A = torch.randn(l, l) - return A.mm(A.transpose(0, 1)) - - -def random_symmetric_pd_matrix(l, *batches): - A = torch.randn(*(batches + (l, l))) - return A.matmul(A.transpose(-2, -1)) + torch.eye(l) * 1e-5 - - -def make_nonzero_det(A, sign=None, min_singular_value=0.1): - u, s, v = A.svd() - s[s < min_singular_value] = min_singular_value - A = u.mm(torch.diag(s)).mm(v.t()) - det = A.det().item() - if sign is not None: - if (det < 0) ^ (sign < 0): - A[0, :].neg_() - return A - - -def random_fullrank_matrix_distinct_singular_value(l, *batches, **kwargs): - silent = kwargs.get("silent", False) - if silent and not torch._C.has_lapack: - return torch.ones(l, l) - - if len(batches) == 0: - A = torch.randn(l, l) - u, _, v = A.svd() - s = torch.arange(1., l + 1).mul_(1.0 / (l + 1)) - return u.mm(torch.diag(s)).mm(v.t()) - else: - all_matrices = [] - for _ in range(0, torch.prod(torch.as_tensor(batches)).item()): - A = torch.randn(l, l) - u, _, v = A.svd() - s = torch.arange(1., l + 1).mul_(1.0 / (l + 1)) - all_matrices.append(u.mm(torch.diag(s)).mm(v.t())) - return torch.stack(all_matrices).reshape(*(batches + (l, l))) - - -def do_test_dtypes(self, dtypes, layout, device): - for dtype in dtypes: - if dtype != torch.float16: - out = torch.zeros((2, 3), dtype=dtype, layout=layout, device=device) - self.assertIs(dtype, out.dtype) - self.assertIs(layout, out.layout) - self.assertEqual(device, out.device) - - -def do_test_empty_full(self, dtypes, layout, device): - shape = torch.Size([2, 3]) - - def check_value(tensor, dtype, layout, device, value, requires_grad): - self.assertEqual(shape, tensor.shape) - self.assertIs(dtype, tensor.dtype) - self.assertIs(layout, tensor.layout) - self.assertEqual(tensor.requires_grad, requires_grad) - if tensor.is_cuda and device is not None: - self.assertEqual(device, tensor.device) - if value is not None: - fill = tensor.new(shape).fill_(value) - self.assertEqual(tensor, fill) - - def get_int64_dtype(dtype): - module = '.'.join(str(dtype).split('.')[1:-1]) - if not module: - return torch.int64 - return operator.attrgetter(module)(torch).int64 - - default_dtype = torch.get_default_dtype() - check_value(torch.empty(shape), default_dtype, torch.strided, -1, None, False) - check_value(torch.full(shape, -5), default_dtype, torch.strided, -1, None, False) - for dtype in dtypes: - for rg in {dtype.is_floating_point, False}: - int64_dtype = get_int64_dtype(dtype) - v = torch.empty(shape, dtype=dtype, device=device, layout=layout, requires_grad=rg) - check_value(v, dtype, layout, device, None, rg) - out = v.new() - check_value(torch.empty(shape, out=out, device=device, layout=layout, requires_grad=rg), - dtype, layout, device, None, rg) - check_value(v.new_empty(shape), dtype, layout, device, None, False) - check_value(v.new_empty(shape, dtype=int64_dtype, device=device, requires_grad=False), - int64_dtype, layout, device, None, False) - check_value(torch.empty_like(v), dtype, layout, device, None, False) - check_value(torch.empty_like(v, dtype=int64_dtype, layout=layout, device=device, requires_grad=False), - int64_dtype, layout, device, None, False) - - if dtype is not torch.float16 and layout != torch.sparse_coo: - fv = 3 - v = torch.full(shape, fv, dtype=dtype, layout=layout, device=device, requires_grad=rg) - check_value(v, dtype, layout, device, fv, rg) - check_value(v.new_full(shape, fv + 1), dtype, layout, device, fv + 1, False) - out = v.new() - check_value(torch.full(shape, fv + 2, out=out, device=device, layout=layout, requires_grad=rg), - dtype, layout, device, fv + 2, rg) - check_value(v.new_full(shape, fv + 3, dtype=int64_dtype, device=device, requires_grad=False), - int64_dtype, layout, device, fv + 3, False) - check_value(torch.full_like(v, fv + 4), dtype, layout, device, fv + 4, False) - check_value(torch.full_like(v, fv + 5, - dtype=int64_dtype, layout=layout, device=device, requires_grad=False), - int64_dtype, layout, device, fv + 5, False) - - -IS_SANDCASTLE = os.getenv('SANDCASTLE') == '1' or os.getenv('TW_JOB_USER') == 'sandcastle' - -THESE_TAKE_WAY_TOO_LONG = { - 'test_Conv3d_groups', - 'test_conv_double_backward_groups', - 'test_Conv3d_dilated', - 'test_Conv3d_stride_padding', - 'test_Conv3d_dilated_strided', - 'test_Conv3d', - 'test_Conv2d_dilated', - 'test_ConvTranspose3d_dilated', - 'test_ConvTranspose2d_dilated', - 'test_snli', - 'test_Conv2d', - 'test_Conv2d_padding', - 'test_ConvTranspose2d_no_bias', - 'test_ConvTranspose2d', - 'test_ConvTranspose3d', - 'test_Conv2d_no_bias', - 'test_matmul_4d_4d', - 'test_multinomial_invalid_probs', -} - - -running_script_path = None - - -def set_running_script_path(): - global running_script_path - try: - running_file = os.path.abspath(os.path.realpath(sys.argv[0])) - if running_file.endswith('.py'): # skip if the running file is not a script - running_script_path = running_file - except Exception: - pass - - -def check_test_defined_in_running_script(test_case): - if running_script_path is None: - return - test_case_class_file = os.path.abspath(os.path.realpath(inspect.getfile(test_case.__class__))) - assert test_case_class_file == running_script_path, "Class of loaded TestCase \"{}\" " \ - "is not defined in the running script \"{}\", but in \"{}\". Did you " \ - "accidentally import a unittest.TestCase from another file?".format( - test_case.id(), running_script_path, test_case_class_file) - - -num_shards = os.environ.get('TEST_NUM_SHARDS', None) -shard = os.environ.get('TEST_SHARD', None) -if num_shards is not None and shard is not None: - num_shards = int(num_shards) - shard = int(shard) - - def load_tests(loader, tests, pattern): - set_running_script_path() - test_suite = unittest.TestSuite() - for test_group in tests: - for test in test_group: - check_test_defined_in_running_script(test) - name = test.id().split('.')[-1] - if name in THESE_TAKE_WAY_TOO_LONG: - continue - hash_id = int(hashlib.sha256(str(test).encode('utf-8')).hexdigest(), 16) - if hash_id % num_shards == shard: - test_suite.addTest(test) - return test_suite -else: - - def load_tests(loader, tests, pattern): - set_running_script_path() - test_suite = unittest.TestSuite() - for test_group in tests: - for test in test_group: - check_test_defined_in_running_script(test) - test_suite.addTest(test) - return test_suite diff --git a/tests/expecttest.py b/tests/expecttest.py deleted file mode 100644 index f6f2649b5..000000000 --- a/tests/expecttest.py +++ /dev/null @@ -1,202 +0,0 @@ -import re -import unittest -import traceback -import os -import string - - -ACCEPT = os.getenv('EXPECTTEST_ACCEPT') - - -def nth_line(src, lineno): - """ - Compute the starting index of the n-th line (where n is 1-indexed) - - >>> nth_line("aaa\\nbb\\nc", 2) - 4 - """ - assert lineno >= 1 - pos = 0 - for _ in range(lineno - 1): - pos = src.find('\n', pos) + 1 - return pos - - -def nth_eol(src, lineno): - """ - Compute the ending index of the n-th line (before the newline, - where n is 1-indexed) - - >>> nth_eol("aaa\\nbb\\nc", 2) - 6 - """ - assert lineno >= 1 - pos = -1 - for _ in range(lineno): - pos = src.find('\n', pos + 1) - if pos == -1: - return len(src) - return pos - - -def normalize_nl(t): - return t.replace('\r\n', '\n').replace('\r', '\n') - - -def escape_trailing_quote(s, quote): - if s and s[-1] == quote: - return s[:-1] + '\\' + quote - else: - return s - - -class EditHistory(object): - def __init__(self): - self.state = {} - - def adjust_lineno(self, fn, lineno): - if fn not in self.state: - return lineno - for edit_loc, edit_diff in self.state[fn]: - if lineno > edit_loc: - lineno += edit_diff - return lineno - - def seen_file(self, fn): - return fn in self.state - - def record_edit(self, fn, lineno, delta): - self.state.setdefault(fn, []).append((lineno, delta)) - - -EDIT_HISTORY = EditHistory() - - -def ok_for_raw_triple_quoted_string(s, quote): - """ - Is this string representable inside a raw triple-quoted string? - Due to the fact that backslashes are always treated literally, - some strings are not representable. - - >>> ok_for_raw_triple_quoted_string("blah", quote="'") - True - >>> ok_for_raw_triple_quoted_string("'", quote="'") - False - >>> ok_for_raw_triple_quoted_string("a ''' b", quote="'") - False - """ - return quote * 3 not in s and (not s or s[-1] not in [quote, '\\']) - - -# This operates on the REVERSED string (that's why suffix is first) -RE_EXPECT = re.compile(r"^(?P[^\n]*?)" - r"(?P'''|" r'""")' - r"(?P.*?)" - r"(?P=quote)" - r"(?Pr?)", re.DOTALL) - - -def replace_string_literal(src, lineno, new_string): - r""" - Replace a triple quoted string literal with new contents. - Only handles printable ASCII correctly at the moment. This - will preserve the quote style of the original string, and - makes a best effort to preserve raw-ness (unless it is impossible - to do so.) - - Returns a tuple of the replaced string, as well as a delta of - number of lines added/removed. - - >>> replace_string_literal("'''arf'''", 1, "barf") - ("'''barf'''", 0) - >>> r = replace_string_literal(" moo = '''arf'''", 1, "'a'\n\\b\n") - >>> print(r[0]) - moo = '''\ - 'a' - \\b - ''' - >>> r[1] - 3 - >>> replace_string_literal(" moo = '''\\\narf'''", 2, "'a'\n\\b\n")[1] - 2 - >>> print(replace_string_literal(" f('''\"\"\"''')", 1, "a ''' b")[0]) - f('''a \'\'\' b''') - """ - # Haven't implemented correct escaping for non-printable characters - assert all(c in string.printable for c in new_string) - i = nth_eol(src, lineno) - new_string = normalize_nl(new_string) - - delta = [new_string.count("\n")] - if delta[0] > 0: - delta[0] += 1 # handle the extra \\\n - - def replace(m): - s = new_string - raw = m.group('raw') == 'r' - if not raw or not ok_for_raw_triple_quoted_string(s, quote=m.group('quote')[0]): - raw = False - s = s.replace('\\', '\\\\') - if m.group('quote') == "'''": - s = escape_trailing_quote(s, "'").replace("'''", r"\'\'\'") - else: - s = escape_trailing_quote(s, '"').replace('"""', r'\"\"\"') - - new_body = "\\\n" + s if "\n" in s and not raw else s - delta[0] -= m.group('body').count("\n") - - return ''.join([m.group('suffix'), - m.group('quote'), - new_body[::-1], - m.group('quote'), - 'r' if raw else '', - ]) - - # Having to do this in reverse is very irritating, but it's the - # only way to make the non-greedy matches work correctly. - return (RE_EXPECT.sub(replace, src[:i][::-1], count=1)[::-1] + src[i:], delta[0]) - - -class TestCase(unittest.TestCase): - longMessage = True - - def assertExpectedInline(self, actual, expect, skip=0): - if ACCEPT: - if actual != expect: - # current frame and parent frame, plus any requested skip - tb = traceback.extract_stack(limit=2 + skip) - fn, lineno, _, _ = tb[0] - print("Accepting new output for {} at {}:{}".format(self.id(), fn, lineno)) - with open(fn, 'r+') as f: - old = f.read() - - # compute the change in lineno - lineno = EDIT_HISTORY.adjust_lineno(fn, lineno) - new, delta = replace_string_literal(old, lineno, actual) - - assert old != new, "Failed to substitute string at {}:{}".format(fn, lineno) - - # Only write the backup file the first time we hit the - # file - if not EDIT_HISTORY.seen_file(fn): - with open(fn + ".bak", 'w') as f_bak: - f_bak.write(old) - f.seek(0) - f.truncate(0) - - f.write(new) - - EDIT_HISTORY.record_edit(fn, lineno, delta) - else: - help_text = ("To accept the new output, re-run test with " - "envvar EXPECTTEST_ACCEPT=1 (we recommend " - "staging/committing your changes before doing this)") - if hasattr(self, "assertMultiLineEqual"): - self.assertMultiLineEqual(expect, actual, msg=help_text) - else: - self.assertEqual(expect, actual, msg=help_text) - - -if __name__ == "__main__": - import doctest - doctest.testmod() diff --git a/tests/test_segmentation_mask.py b/tests/test_segmentation_mask.py index 06e32bff4..0c0a810a3 100644 --- a/tests/test_segmentation_mask.py +++ b/tests/test_segmentation_mask.py @@ -1,11 +1,11 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import torch import numpy as np -from common_utils import TestCase, run_tests +import unittest from maskrcnn_benchmark.structures.segmentation_mask import Mask, Polygons, SegmentationMask -class TestSegmentationMask(TestCase): +class TestSegmentationMask(unittest.TestCase): def __init__(self, method_name='runTest'): super(TestSegmentationMask, self).__init__(method_name) self.poly = [[423.0, 306.5, 406.5, 277.0, 400.0, 271.5, 389.5, 277.0, 387.5, 292.0, @@ -24,14 +24,14 @@ def __init__(self, method_name='runTest'): def test_crop(self): poly_crop = self.polygon.crop(self.box) mask_from_poly_crop = poly_crop.convert('mask') - mask_crop = self.mask.crop(self.box).mask + mask_crop = self.mask.crop(self.box).convert('mask') - self.assertEqual(mask_from_poly_crop, mask_crop) + self.assertTrue(torch.equal(mask_from_poly_crop, mask_crop)) def test_convert(self): mask_from_poly_convert = self.polygon.convert('mask') mask = self.mask.convert('mask') - self.assertEqual(mask_from_poly_convert, mask) + self.assertTrue(torch.equal(mask_from_poly_convert, mask)) def test_transpose(self): FLIP_LEFT_RIGHT = 0 @@ -41,14 +41,14 @@ def test_transpose(self): mask_from_poly_flip = self.polygon.transpose(method).convert('mask') mask_flip = self.mask.transpose(method).convert('mask') print(method, torch.abs(mask_flip.float() - mask_from_poly_flip.float()).sum()) - self.assertEqual(mask_flip, mask_from_poly_flip) + self.assertTrue(torch.equal(mask_flip, mask_from_poly_flip)) def test_resize(self): new_size = (600, 500) mask_from_poly_resize = self.polygon.resize(new_size).convert('mask') mask_resize = self.mask.resize(new_size).convert('mask') print('diff resize: ', torch.abs(mask_from_poly_resize.float() - mask_resize.float()).sum()) - self.assertEqual(mask_from_poly_resize, mask_resize) + self.assertTrue(torch.equal(mask_from_poly_resize, mask_resize)) if __name__ == "__main__": - run_tests() + unittest.main() From b8c5bcafa02e4a7489b5a79bf4609933f98c9599 Mon Sep 17 00:00:00 2001 From: Gu Wang Date: Sun, 17 Feb 2019 23:34:07 +0800 Subject: [PATCH 07/14] minor change --- maskrcnn_benchmark/structures/segmentation_mask.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 16f43a79a..30abbe31e 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -60,8 +60,8 @@ def transpose(self, method): return Mask(flipped_mask, self.size, self.mode) def crop(self, box): - box = [int(b) for b in box] - w, h = box[2] - box[0], box[3] - box[1] + box = [round(b) for b in box] + w, h = box[2] - box[0] + 1, box[3] - box[1] + 1 w = max(w, 1) h = max(h, 1) cropped_mask = self.mask[box[1]: box[3], box[0]: box[2]] @@ -69,7 +69,7 @@ def crop(self, box): def resize(self, size, *args, **kwargs): width, height = size - scaled_mask = interpolate(self.mask[None, None, :, :], (height, width), mode='nearest')[0, 0] + scaled_mask = interpolate(self.mask[None, None, :, :], (height, width), mode='bilinear')[0, 0] return Mask(scaled_mask, size=size, mode=self.mode) def convert(self, mode): @@ -203,7 +203,7 @@ def __init__(self, segms, size, mode=None): mode: 'polygon', 'mask'. if mode is 'mask', convert mask of any format to binary mask """ assert isinstance(segms, list) - if type(segms[0]) != list: + if not isinstance(segms[0], (list, Polygons)): mode = 'mask' if mode == 'mask': self.masks = [Mask(m, size, mode) for m in segms] From ac1b17401225ac7441b8653920891b9f6734cd52 Mon Sep 17 00:00:00 2001 From: botcs Date: Thu, 21 Feb 2019 05:58:07 +0000 Subject: [PATCH 08/14] Refactored segmentation_mask.py --- maskrcnn_benchmark/data/datasets/coco.py | 2 +- .../modeling/roi_heads/mask_head/loss.py | 6 +- .../structures/segmentation_mask.py | 597 +++++++++++++----- 3 files changed, 453 insertions(+), 152 deletions(-) diff --git a/maskrcnn_benchmark/data/datasets/coco.py b/maskrcnn_benchmark/data/datasets/coco.py index f0c8c25b4..d0e42b437 100644 --- a/maskrcnn_benchmark/data/datasets/coco.py +++ b/maskrcnn_benchmark/data/datasets/coco.py @@ -80,7 +80,7 @@ def __getitem__(self, idx): target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] - masks = SegmentationMask(masks, img.size) + masks = SegmentationMask(masks, img.size, mode='poly') target.add_field("masks", masks) if anno and "keypoints" in anno[0]: diff --git a/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py b/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py index 36dcaa325..af8fba883 100644 --- a/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py +++ b/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py @@ -27,9 +27,7 @@ def project_masks_on_boxes(segmentation_masks, proposals, discretization_size): assert segmentation_masks.size == proposals.size, "{}, {}".format( segmentation_masks, proposals ) - # TODO put the proposals on the CPU, as the representation for the - # masks is not efficient GPU-wise (possibly several small tensors for - # representing a single instance mask) + proposals = proposals.bbox.to(torch.device("cpu")) for segmentation_mask, proposal in zip(segmentation_masks, proposals): # crop the masks, resize them to the desired resolution and @@ -37,7 +35,7 @@ def project_masks_on_boxes(segmentation_masks, proposals, discretization_size): # instead of the list representation that was used cropped_mask = segmentation_mask.crop(proposal) scaled_mask = cropped_mask.resize((M, M)) - mask = scaled_mask.convert(mode="mask") + mask = scaled_mask.get_mask_tensor() masks.append(mask) if len(masks) == 0: return torch.empty(0, dtype=torch.float32, device=device) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 30abbe31e..77ca5819c 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -1,116 +1,223 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +import cv2 + import torch import numpy as np from torch.nn.functional import interpolate + import pycocotools.mask as mask_utils # transpose FLIP_LEFT_RIGHT = 0 FLIP_TOP_BOTTOM = 1 -class Mask(object): - """ - This class is unfinished and not meant for use yet - It is supposed to contain the mask for an object as - a 2d tensor - """ - def __init__(self, segm, size, mode): - width, height = size - if isinstance(segm, Mask): - mask = segm.mask - else: - if type(segm) == list: - # polygons - mask = Polygons(segm, size, 'polygon').convert('mask').to(dtype=torch.float32) - elif type(segm) == dict and 'counts' in segm: - if type(segm['counts']) == list: - # uncompressed RLE - h, w = segm['size'] - rle = mask_utils.frPyObjects(segm, h, w) - mask = mask_utils.decode(rle) - mask = torch.from_numpy(mask).to(dtype=torch.float32) - else: - # compressed RLE - mask = mask_utils.decode(segm) - mask = torch.from_numpy(mask).to(dtype=torch.float32) - else: - # binary mask - if type(segm) == np.ndarray: - mask = torch.from_numpy(segm).to(dtype=torch.float32) - else: # torch.Tensor - mask = segm.to(dtype=torch.float32) - self.mask = mask - self.size = size - self.mode = mode - def transpose(self, method): - if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): - raise NotImplementedError("Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented") +''' ABSTRACT +Segmentations come in either: +1) Binary masks +2) Polygons - width, height = self.size - if method == FLIP_LEFT_RIGHT: - max_idx = width - dim = 1 - elif method == FLIP_TOP_BOTTOM: - max_idx = height - dim = 0 +Binary masks can be represented in a contiguous array +and operations can be carried out more efficiently, +Therefore BinaryMaskList handles them together. + +Polygons are handled separately for each instance, +by PolygonInstance and instances are handled by +PolygonList. + +SegmentationList is supposed to represent both, +therefore it wraps the functions of BinaryMaskList +and PolygonList to make it transparent. +''' + + + +class BinaryMaskList(object): + ''' + This class handles binary masks for all objects in the image + ''' + + def __init__(self, masks, size): + ''' + Arguments: + masks: Either torch.tensor of [num_instances, H, W] + or list of torch.tensors of [H, W] or + BinaryMaskList. + size: absolute image size, width first + ''' + + if isinstance(masks, torch.Tensor): + pass + elif isinstance(masks, (list, tuple)): + masks = torch.stack(masks, dim=2) + elif isinstance(masks, BinaryMaskList): + masks = masks.masks + + if len(masks.shape) == 2: + masks = masks[None] + assert len(masks.shape) == 3 + + + assert masks.shape[1] == size[1],\ + '%s != %s'%(masks.shape[1], size[1]) + assert masks.shape[2] == size[0],\ + '%s != %s'%(masks.shape[2], size[0]) + + + self.masks = masks.clone() + self.size = tuple(size) + + + def transpose(self, method): + dim = 1 if method == FLIP_TOP_BOTTOM else 2 + # print(dim) + # print(self.masks) + # print(self.masks.flip(dim)) + flipped_masks = self.masks.flip(dim) + return BinaryMaskList(flipped_masks, self.size) - flip_idx = torch.tensor(list(range(max_idx)[::-1])) - flipped_mask = self.mask.index_select(dim, flip_idx) - return Mask(flipped_mask, self.size, self.mode) def crop(self, box): - box = [round(b) for b in box] - w, h = box[2] - box[0] + 1, box[3] - box[1] + 1 - w = max(w, 1) - h = max(h, 1) - cropped_mask = self.mask[box[1]: box[3], box[0]: box[2]] - return Mask(cropped_mask, size=(w, h), mode=self.mode) + assert isinstance(box, (list, tuple)), str(type(box)) - def resize(self, size, *args, **kwargs): - width, height = size - scaled_mask = interpolate(self.mask[None, None, :, :], (height, width), mode='bilinear')[0, 0] - return Mask(scaled_mask, size=size, mode=self.mode) + # box is assumed to by xyxy + current_width, current_height = self.size + xmin, ymin, xmax, ymax = map(round, box) + assert xmin >= 0 and xmax < current_width + assert ymin >= 0 and ymax < current_height + assert xmin < xmax and ymin < ymax + + width, height = xmax - xmin, ymax - ymin + #width = max(width, 1) + #height = max(height, 1) + + if (xmin >= xmax or ymin >= ymax): + print(box) + + cropped_masks = self.masks[:, ymin:ymax, xmin:xmax] + cropped_size = width, height + return BinaryMaskList(cropped_masks, cropped_size) + + + def resize(self, size): + try: + iter(size) + except TypeError: + assert isinstance(size, (int, float)) + size = size, size + width, height = map(int, size) + + assert width > 0 + assert height > 0 + + # Height comes first here! + resized_masks = torch.nn.functional.interpolate( + input=self.masks[None].float(), + size=(height, width), + mode='bilinear', + align_corners=False + )[0].type_as(self.masks) + resized_size = width, height + return BinaryMaskList(resized_masks, resized_size) + + + + def convert_to_polygon(self): + contours = self._findContours() + return PolygonList(contours, self.size) + + + def to(self, *args, **kwargs): + return self + + + def _findContours(self): + contours = [] + masks = self.masks.detach().numpy() + for mask in masks: + mask = cv2.UMat(mask) + contour, hierarchy = cv2.findContours( + mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_L1 + ) + + reshaped_contour = [] + for entity in contour: + assert len(entity.shape) == 3 + assert entity.shape[1] == 1, \ + 'Hierarchical contours are not allowed' + reshaped_contour.append(entity.reshape(-1).tolist()) + contours.append(reshaped_contour) + return contours + + + def __len__(self): + return len(self.masks) + + + def __getitem__(self, index): + # Probably it can cause some overhead + # but preserves consistency + masks = self.masks[index].clone() + return BinaryMaskList(masks, self.size) - def convert(self, mode): - mask = self.mask.to(dtype=torch.uint8) - return mask def __iter__(self): - return iter(self.mask) + return iter(self.masks) + def __repr__(self): s = self.__class__.__name__ + "(" - # s += "num_mask={}, ".format(len(self.mask)) + s += "num_instances={}, ".format(len(self.masks)) s += "image_width={}, ".format(self.size[0]) - s += "image_height={}, ".format(self.size[1]) - s += "mode={})".format(self.mode) + s += "image_height={})".format(self.size[1]) return s -class Polygons(object): - """ +class PolygonInstance(object): + ''' This class holds a set of polygons that represents a single instance of an object mask. The object can be represented as a set of polygons - """ + ''' + + def __init__(self, polygons, size): + ''' + Arguments: + a list of lists of numbers. + The first level refers to all the polygons that compose the + object, and the second level to the polygon coordinates. + ''' + if isinstance(polygons, (list, tuple)): + valid_polygons = [] + for p in polygons: + p = torch.as_tensor(p, dtype=torch.float32) + if len(p) >= 6: # 3 * 2 coordinates + valid_polygons.append(p) + polygons = valid_polygons + + elif isinstance(polygons, PolygonInstance): + polygons = polygons.polygons.copy() - def __init__(self, polygons, size, mode): - # assert isinstance(polygons, list), '{}'.format(polygons) - if isinstance(polygons, list): - polygons = [torch.as_tensor(p, dtype=torch.float32) for p in polygons] - elif isinstance(polygons, Polygons): - polygons = polygons.polygons + else: + RuntimeError('Type of argument `polygons` is not allowed:%s'\ + %(type(polygons))) + + ''' This crashes the training way too many times... + for p in polygons: + assert p[::2].min() >= 0 + assert p[::2].max() < size[0] + assert p[1::2].min() >= 0 + assert p[1::2].max() , size[1] + ''' self.polygons = polygons - self.size = size - self.mode = mode + self.size = tuple(size) + def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): raise NotImplementedError( - "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" + 'Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented' ) flipped_polygons = [] @@ -128,30 +235,56 @@ def transpose(self, method): p[idx::2] = dim - poly[idx::2] - TO_REMOVE flipped_polygons.append(p) - return Polygons(flipped_polygons, size=self.size, mode=self.mode) + return PolygonInstance(flipped_polygons, size=self.size) + def crop(self, box): - w, h = box[2] - box[0], box[3] - box[1] + assert isinstance(box, (list, tuple, torch.Tensor)), str(type(box)) + + # box is assumed to be xyxy + current_width, current_height = self.size + xmin, ymin, xmax, ymax = map(float, box) + #assert xmin >= 0 and xmax < current_width, str(box) + #assert ymin >= 0 and ymax < current_height, str(box) + xmin = max(xmin, 0) + ymin = max(ymin, 0) + + xmax = min(xmax, current_width) + ymax = min(ymax, current_height) + + assert xmin < xmax and ymin < ymax, str(box) - # TODO chck if necessary - w = max(w, 1) - h = max(h, 1) + if (xmin >= xmax or ymin >= ymax): + print(box) + + w, h = ymax - ymin, xmax - xmin + #w = max(w, 1) + #h = max(w, 1) cropped_polygons = [] for poly in self.polygons: p = poly.clone() - p[0::2] = p[0::2] - box[0] # .clamp(min=0, max=w) - p[1::2] = p[1::2] - box[1] # .clamp(min=0, max=h) + p[0::2] = p[0::2] - xmin # .clamp(min=0, max=w) + p[1::2] = p[1::2] - ymin # .clamp(min=0, max=h) cropped_polygons.append(p) - return Polygons(cropped_polygons, size=(w, h), mode=self.mode) + return PolygonInstance(cropped_polygons, size=(w, h)) + + + def resize(self, size): + try: + iter(size) + except TypeError: + assert isinstance(size, (int, float)) + size = size, size + + ratios = tuple(float(s) / float(s_orig) + for s, s_orig in zip(size, self.size)) - def resize(self, size, *args, **kwargs): - ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(size, self.size)) if ratios[0] == ratios[1]: ratio = ratios[0] scaled_polys = [p * ratio for p in self.polygons] - return Polygons(scaled_polys, size, mode=self.mode) + return PolygonInstance(scaled_polys, size) ratio_w, ratio_h = ratios scaled_polygons = [] @@ -161,104 +294,274 @@ def resize(self, size, *args, **kwargs): p[1::2] *= ratio_h scaled_polygons.append(p) - return Polygons(scaled_polygons, size=size, mode=self.mode) + return PolygonInstance(scaled_polygons, size=size) - def convert(self, mode): + + def convert_to_binarymask(self): width, height = self.size - if mode == "mask": - rles = mask_utils.frPyObjects( - [p.numpy() for p in self.polygons], height, width - ) - rle = mask_utils.merge(rles) - mask = mask_utils.decode(rle) - mask = torch.from_numpy(mask) - # TODO add squeeze? - return mask + rles = mask_utils.frPyObjects( + [p.numpy() for p in self.polygons], height, width + ) + rle = mask_utils.merge(rles) + mask = mask_utils.decode(rle) + mask = torch.from_numpy(mask) + return mask + + + def __len__(self): + return len(self.polygons) + def __repr__(self): - s = self.__class__.__name__ + "(" - s += "num_polygons={}, ".format(len(self.polygons)) - s += "image_width={}, ".format(self.size[0]) - s += "image_height={}, ".format(self.size[1]) - s += "mode={})".format(self.mode) + s = self.__class__.__name__ + '(' + s += 'num_groups={}, '.format(len(self.polygons)) + s += 'image_width={}, '.format(self.size[0]) + s += 'image_height={}, '.format(self.size[1]) return s -class SegmentationMask(object): - """ - This class stores the segmentations for all objects in the image - """ - def __init__(self, segms, size, mode=None): - """ +class PolygonList(object): + ''' + This class handles PolygonInstances for all objects in the image + ''' + + def __init__(self, polygons, size): + ''' Arguments: - segms: three types - (1) polygons: a list of list of lists of numbers. The first + polygons: + a list of list of lists of numbers. The first level of the list correspond to individual instances, the second level to all the polygons that compose the object, and the third level to the polygon coordinates. - (2) rles: COCO's run length encoding format, uncompressed or compressed - (3) binary masks - size: (width, height) - mode: 'polygon', 'mask'. if mode is 'mask', convert mask of any format to binary mask - """ - assert isinstance(segms, list) - if not isinstance(segms[0], (list, Polygons)): - mode = 'mask' - if mode == 'mask': - self.masks = [Mask(m, size, mode) for m in segms] - else: # polygons - self.masks = [Polygons(p, size, mode) for p in segms] - self.size = size - self.mode = mode + + OR + + a list of PolygonInstances. + + OR + + a PolygonList + + size: absolute image size + + ''' + if isinstance(polygons, (list, tuple)): + if len(polygons) == 0: + polygons = [[[]]] + if isinstance(polygons[0], (list, tuple)): + assert isinstance(polygons[0][0], (list, tuple)),\ + str(type(polygons[0][0])) + else: + assert isinstance(polygons[0], PolygonInstance),\ + str(type(polygons[0])) + + elif isinstance(polygons, PolygonList): + size = polygons.size + polygons = polygons.polygons + + else: + RuntimeError('Type of argument `polygons` is not allowed:%s'\ + %(type(polygons))) + + + + assert isinstance(size, (list, tuple)), str(type(size)) + + self.polygons = [] + for p in polygons: + p = PolygonInstance(p, size) + if len(p) > 0: + self.polygons.append(p) + + self.size = tuple(size) + def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): raise NotImplementedError( - "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" + 'Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented' ) - flipped = [] - for mask in self.masks: - flipped.append(mask.transpose(method)) - return SegmentationMask(flipped, size=self.size, mode=self.mode) + flipped_polygons = [] + for polygon in self.polygons: + flipped_polygons.append(polygon.transpose(method)) + + return PolygonList(flipped_polygons, size=self.size) + def crop(self, box): w, h = box[2] - box[0], box[3] - box[1] - cropped = [] - for mask in self.masks: - cropped.append(mask.crop(box)) - return SegmentationMask(cropped, size=(w, h), mode=self.mode) + cropped_polygons = [] + for polygon in self.polygons: + cropped_polygons.append(polygon.crop(box)) + + cropped_size = w, h + return PolygonList(cropped_polygons, cropped_size) + + + def resize(self, size): + resized_polygons = [] + for polygon in self.polygons: + resized_polygons.append(polygon.resize(size)) + + resized_size = size + return PolygonList(resized_polygons, resized_size) - def resize(self, size, *args, **kwargs): - scaled = [] - for mask in self.masks: - scaled.append(mask.resize(size, *args, **kwargs)) - return SegmentationMask(scaled, size=size, mode=self.mode) def to(self, *args, **kwargs): return self + + def convert_to_binarymask(self): + if self.__len__() > 0: + masks = torch.stack([ + p.convert_to_binarymask() for p in self.polygons]) + else: + size = self.size + masks = torch.empty([0, size[1], size[0]], dtype=torch.uint8) + + return BinaryMaskList(masks, size=self.size) + + + def __len__(self): + return len(self.polygons) + + def __getitem__(self, item): if isinstance(item, (int, slice)): - selected_masks = [self.masks[item]] + selected_polygons = [self.polygons[item]] else: # advanced indexing on a single dimension - selected_masks = [] + selected_polygons = [] if isinstance(item, torch.Tensor) and item.dtype == torch.uint8: item = item.nonzero() item = item.squeeze(1) if item.numel() > 0 else item item = item.tolist() for i in item: - selected_masks.append(self.masks[i]) - return SegmentationMask(selected_masks, size=self.size, mode=self.mode) + selected_polygons.append(self.polygons[i]) + return PolygonList(selected_polygons, size=self.size) + def __iter__(self): - return iter(self.masks) + return iter(self.polygons) + + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += 'num_instances={}, '.format(len(self.polygons)) + s += 'image_width={}, '.format(self.size[0]) + s += 'image_height={})'.format(self.size[1]) + return s + + + + + +class SegmentationMask(object): + + ''' + This class stores the segmentations for all objects in the image. + It wraps BinaryMaskList and PolygonList conveniently. + ''' + def __init__(self, instances, size, mode='poly'): + ''' + Arguments: + instances: two types + (1) polygon + (2) binary mask + size: (width, height) + mode: 'poly', 'mask'. if mode is 'mask', convert mask of any format to binary mask + ''' + + assert isinstance(size, (list, tuple)) + assert len(size) == 2 + if isinstance(size[0], torch.Tensor): + assert isinstance(size[1], torch.Tensor) + size = size[0].item(), size[1].item() + + assert isinstance(size[0], (int, float)) + assert isinstance(size[1], (int, float)) + + if mode == 'poly': + self.instances = PolygonList(instances, size) + elif mode == 'mask': + self.instances = BinaryMaskList(instances, size) + else: + raise NotImplementedError('Unknown mode: %s'%str(mode)) + + self.mode = mode + self.size = tuple(size) + + def transpose(self, method): + flipped_instances = self.instances.transpose(method) + return SegmentationMask(flipped_instances, self.size, self.mode) + + + def crop(self, box): + cropped_instances = self.instances.crop(box) + cropped_size = cropped_instances.size + return SegmentationMask(cropped_instances, cropped_size, self.mode) + + + def resize(self, size, *args, **kwargs): + resized_instances = self.instances.resize(size) + resized_size = size + return SegmentationMask(resized_instances, resized_size, self.mode) + + + def to(self, *args, **kwargs): + return self + + + def convert(self, mode): + if mode == self.mode: + return self + + if mode == 'poly': + converted_instances = self.instances.convert_to_polygon() + elif mode == 'mask': + converted_instances = self.instances.convert_to_binarymask() + else: + raise NotImplementedError('Unknown mode: %s'%str(mode)) + + return SegmentationMask(converted_instances, self.size, mode) + + + def get_mask_tensor(self): + instances = self.instances + if self.mode == 'poly': + instances = instances.convert_to_binarymask() + # If there is only 1 instance + return instances.masks.squeeze(0) + + + def __len__(self): + return len(self.instances) + + + def __getitem__(self, item): + selected_instances = self.instances.__getitem__(item) + return SegmentationMask(selected_instances, self.size, self.mode) + + + def __iter__(self): + self.iter_idx = 0 + return self + + + def __next__(self): + if self.iter_idx < self.__len__(): + next_segmentation = self.__getitem__(self.iter_idx) + self.iter_idx += 1 + return next_segmentation + raise StopIteration + def __repr__(self): s = self.__class__.__name__ + "(" - s += "num_instances={}, ".format(len(self.masks)) + s += "num_instances={}, ".format(len(self.instances)) s += "image_width={}, ".format(self.size[0]) - s += "image_height={})".format(self.size[1]) + s += "image_height={}, ".format(self.size[1]) + s += "mode={})".format(self.mode) return s From d79829d0438636447e0137d9df7bcb19cf9d307d Mon Sep 17 00:00:00 2001 From: botcs Date: Thu, 21 Feb 2019 07:09:09 +0000 Subject: [PATCH 09/14] Add unit test for segmentation_mask.py --- tests/test_segmentation_mask.py | 96 ++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/tests/test_segmentation_mask.py b/tests/test_segmentation_mask.py index 0c0a810a3..d01ed9452 100644 --- a/tests/test_segmentation_mask.py +++ b/tests/test_segmentation_mask.py @@ -1,54 +1,74 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import torch -import numpy as np +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. import unittest -from maskrcnn_benchmark.structures.segmentation_mask import Mask, Polygons, SegmentationMask +import torch +from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask class TestSegmentationMask(unittest.TestCase): def __init__(self, method_name='runTest'): super(TestSegmentationMask, self).__init__(method_name) - self.poly = [[423.0, 306.5, 406.5, 277.0, 400.0, 271.5, 389.5, 277.0, 387.5, 292.0, - 384.5, 295.0, 374.5, 220.0, 378.5, 210.0, 391.0, 200.5, 404.0, 199.5, - 414.0, 203.5, 425.5, 221.0, 438.5, 297.0, 423.0, 306.5], - [385.5, 240.0, 404.0, 234.5, 419.5, 234.0, 416.5, 219.0, 409.0, 209.5, - 394.0, 207.5, 385.5, 213.0, 382.5, 221.0, 385.5, 240.0]] - self.width = 640 - self.height = 480 - self.size = (self.width, self.height) - self.box = [35, 55, 540, 400] # xyxy - - self.polygon = Polygons(self.poly, self.size, 'polygon') - self.mask = Mask(self.poly, self.size, 'mask') + poly = [[[423.0, 306.5, 406.5, 277.0, 400.0, 271.5, 389.5, 277.0, + 387.5, 292.0, 384.5, 295.0, 374.5, 220.0, 378.5, 210.0, + 391.0, 200.5, 404.0, 199.5, 414.0, 203.5, 425.5, 221.0, + 438.5, 297.0, 423.0, 306.5], + [100, 100, 200, 100, 200, 200, 100, 200], + ]] + width = 640 + height = 480 + size = width, height + + self.P = SegmentationMask(poly, size, 'poly') + self.M = SegmentationMask(poly, size, 'poly').convert('mask') + + + def L1(self, A, B): + diff = A.get_mask_tensor() - B.get_mask_tensor() + diff = torch.sum(torch.abs(diff.float())).item() + return diff - def test_crop(self): - poly_crop = self.polygon.crop(self.box) - mask_from_poly_crop = poly_crop.convert('mask') - mask_crop = self.mask.crop(self.box).convert('mask') - self.assertTrue(torch.equal(mask_from_poly_crop, mask_crop)) - def test_convert(self): - mask_from_poly_convert = self.polygon.convert('mask') - mask = self.mask.convert('mask') - self.assertTrue(torch.equal(mask_from_poly_convert, mask)) + M_hat = self.M.convert('poly').convert('mask') + P_hat = self.P.convert('mask').convert('poly') + + diff_mask = self.L1(self.M, M_hat) + diff_poly = self.L1(self.P, P_hat) + self.assertTrue(diff_mask == diff_poly) + self.assertTrue(diff_mask <= 8169.) + self.assertTrue(diff_poly <= 8169.) + + + def test_crop(self): + box = [400, 250, 500, 300] # xyxy + diff = self.L1(self.M.crop(box), self.P.crop(box)) + self.assertTrue(diff <= 1.) + + + def test_resize(self): + new_size = 50, 25 + M_hat = self.M.resize(new_size) + P_hat = self.P.resize(new_size) + diff = self.L1(M_hat, P_hat) + + self.assertTrue(self.M.size == self.P.size) + self.assertTrue(M_hat.size == P_hat.size) + self.assertTrue(self.M.size != M_hat.size) + self.assertTrue(diff <= 255.) + def test_transpose(self): FLIP_LEFT_RIGHT = 0 FLIP_TOP_BOTTOM = 1 - methods = (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM) - for method in methods: - mask_from_poly_flip = self.polygon.transpose(method).convert('mask') - mask_flip = self.mask.transpose(method).convert('mask') - print(method, torch.abs(mask_flip.float() - mask_from_poly_flip.float()).sum()) - self.assertTrue(torch.equal(mask_flip, mask_from_poly_flip)) - - def test_resize(self): - new_size = (600, 500) - mask_from_poly_resize = self.polygon.resize(new_size).convert('mask') - mask_resize = self.mask.resize(new_size).convert('mask') - print('diff resize: ', torch.abs(mask_from_poly_resize.float() - mask_resize.float()).sum()) - self.assertTrue(torch.equal(mask_from_poly_resize, mask_resize)) + diff_hor = self.L1(self.M.transpose(FLIP_LEFT_RIGHT), + self.P.transpose(FLIP_LEFT_RIGHT)) + + diff_ver = self.L1(self.M.transpose(FLIP_TOP_BOTTOM), + self.P.transpose(FLIP_TOP_BOTTOM)) + + self.assertTrue(diff_hor <= 53250.) + self.assertTrue(diff_ver <= 42494.) + if __name__ == "__main__": + unittest.main() From e356f5a2a63eff3d1df728f56abed578a88fc086 Mon Sep 17 00:00:00 2001 From: botcs Date: Sun, 24 Feb 2019 11:00:07 +0000 Subject: [PATCH 10/14] Add RLE support for BinaryMaskList --- .../modeling/roi_heads/mask_head/loss.py | 4 +- .../structures/segmentation_mask.py | 221 +++++++----------- 2 files changed, 91 insertions(+), 134 deletions(-) diff --git a/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py b/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py index af8fba883..d4c5e3621 100644 --- a/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py +++ b/maskrcnn_benchmark/modeling/roi_heads/mask_head/loss.py @@ -28,11 +28,11 @@ def project_masks_on_boxes(segmentation_masks, proposals, discretization_size): segmentation_masks, proposals ) + # FIXME: CPU computation bottleneck, this should be parallelized proposals = proposals.bbox.to(torch.device("cpu")) for segmentation_mask, proposal in zip(segmentation_masks, proposals): # crop the masks, resize them to the desired resolution and - # then convert them to the tensor representation, - # instead of the list representation that was used + # then convert them to the tensor representation. cropped_mask = segmentation_mask.crop(proposal) scaled_mask = cropped_mask.resize((M, M)) mask = scaled_mask.get_mask_tensor() diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 77ca5819c..90cb5a39a 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -11,14 +11,14 @@ FLIP_TOP_BOTTOM = 1 -''' ABSTRACT +""" ABSTRACT Segmentations come in either: 1) Binary masks 2) Polygons Binary masks can be represented in a contiguous array and operations can be carried out more efficiently, -Therefore BinaryMaskList handles them together. +therefore BinaryMaskList handles them together. Polygons are handled separately for each instance, by PolygonInstance and instances are handled by @@ -27,55 +27,63 @@ SegmentationList is supposed to represent both, therefore it wraps the functions of BinaryMaskList and PolygonList to make it transparent. -''' - +""" class BinaryMaskList(object): - ''' + """ This class handles binary masks for all objects in the image - ''' + """ def __init__(self, masks, size): - ''' + """ Arguments: masks: Either torch.tensor of [num_instances, H, W] - or list of torch.tensors of [H, W] or - BinaryMaskList. + or list of torch.tensors of [H, W] with num_instances elems, + or RLE (Run Length Encoding) - interpreted as list of dicts, + or BinaryMaskList. size: absolute image size, width first - ''' + + After initialization, a hard copy will be made, to leave the + initializing source data intact. + """ if isinstance(masks, torch.Tensor): - pass + # The raw data representation is passed as argument + masks = masks.clone() elif isinstance(masks, (list, tuple)): - masks = torch.stack(masks, dim=2) + if isinstance(masks[0], torch.Tensor): + masks = torch.stack(masks, dim=2).clone() + elif isinstance(masks[0], dict) and 'count' in masks[0]: + # RLE interpretation + + masks = mask_utils + else: + RuntimeError('Type of `masks[0]` could not be interpreted: %s'\ + %type(masks)) elif isinstance(masks, BinaryMaskList): - masks = masks.masks + # just hard copy the BinaryMaskList instance's underlying data + masks = masks.masks.clone() + else: + RuntimeError('Type of `masks` argument could not be interpreted:%s'\ + %tpye(masks)) if len(masks.shape) == 2: + # if only a single instance mask is passed masks = masks[None] - assert len(masks.shape) == 3 - - - assert masks.shape[1] == size[1],\ - '%s != %s'%(masks.shape[1], size[1]) - assert masks.shape[2] == size[0],\ - '%s != %s'%(masks.shape[2], size[0]) + assert len(masks.shape) == 3 + assert masks.shape[1] == size[1], "%s != %s" % (masks.shape[1], size[1]) + assert masks.shape[2] == size[0], "%s != %s" % (masks.shape[2], size[0]) - self.masks = masks.clone() + self.masks = masks self.size = tuple(size) - def transpose(self, method): dim = 1 if method == FLIP_TOP_BOTTOM else 2 - # print(dim) - # print(self.masks) - # print(self.masks.flip(dim)) flipped_masks = self.masks.flip(dim) return BinaryMaskList(flipped_masks, self.size) - def crop(self, box): assert isinstance(box, (list, tuple)), str(type(box)) @@ -87,17 +95,11 @@ def crop(self, box): assert xmin < xmax and ymin < ymax width, height = xmax - xmin, ymax - ymin - #width = max(width, 1) - #height = max(height, 1) - - if (xmin >= xmax or ymin >= ymax): - print(box) cropped_masks = self.masks[:, ymin:ymax, xmin:xmax] cropped_size = width, height return BinaryMaskList(cropped_masks, cropped_size) - def resize(self, size): try: iter(size) @@ -113,23 +115,19 @@ def resize(self, size): resized_masks = torch.nn.functional.interpolate( input=self.masks[None].float(), size=(height, width), - mode='bilinear', - align_corners=False + mode="bilinear", + align_corners=False, )[0].type_as(self.masks) resized_size = width, height return BinaryMaskList(resized_masks, resized_size) - - def convert_to_polygon(self): contours = self._findContours() return PolygonList(contours, self.size) - def to(self, *args, **kwargs): return self - def _findContours(self): contours = [] masks = self.masks.detach().numpy() @@ -142,28 +140,24 @@ def _findContours(self): reshaped_contour = [] for entity in contour: assert len(entity.shape) == 3 - assert entity.shape[1] == 1, \ - 'Hierarchical contours are not allowed' + assert entity.shape[1] == 1,\ + "Hierarchical contours are not allowed" reshaped_contour.append(entity.reshape(-1).tolist()) contours.append(reshaped_contour) return contours - def __len__(self): return len(self.masks) - def __getitem__(self, index): # Probably it can cause some overhead # but preserves consistency masks = self.masks[index].clone() return BinaryMaskList(masks, self.size) - def __iter__(self): return iter(self.masks) - def __repr__(self): s = self.__class__.__name__ + "(" s += "num_instances={}, ".format(len(self.masks)) @@ -172,26 +166,25 @@ def __repr__(self): return s - class PolygonInstance(object): - ''' + """ This class holds a set of polygons that represents a single instance of an object mask. The object can be represented as a set of polygons - ''' + """ def __init__(self, polygons, size): - ''' + """ Arguments: a list of lists of numbers. The first level refers to all the polygons that compose the object, and the second level to the polygon coordinates. - ''' + """ if isinstance(polygons, (list, tuple)): valid_polygons = [] for p in polygons: p = torch.as_tensor(p, dtype=torch.float32) - if len(p) >= 6: # 3 * 2 coordinates + if len(p) >= 6: # 3 * 2 coordinates valid_polygons.append(p) polygons = valid_polygons @@ -199,25 +192,25 @@ def __init__(self, polygons, size): polygons = polygons.polygons.copy() else: - RuntimeError('Type of argument `polygons` is not allowed:%s'\ - %(type(polygons))) + RuntimeError( + "Type of argument `polygons` is not allowed:%s"%(type(polygons)) + ) - ''' This crashes the training way too many times... + """ This crashes the training way too many times... for p in polygons: assert p[::2].min() >= 0 assert p[::2].max() < size[0] assert p[1::2].min() >= 0 assert p[1::2].max() , size[1] - ''' + """ self.polygons = polygons self.size = tuple(size) - def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): raise NotImplementedError( - 'Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented' + "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" ) flipped_polygons = [] @@ -237,15 +230,13 @@ def transpose(self, method): return PolygonInstance(flipped_polygons, size=self.size) - def crop(self, box): assert isinstance(box, (list, tuple, torch.Tensor)), str(type(box)) # box is assumed to be xyxy current_width, current_height = self.size xmin, ymin, xmax, ymax = map(float, box) - #assert xmin >= 0 and xmax < current_width, str(box) - #assert ymin >= 0 and ymax < current_height, str(box) + xmin = max(xmin, 0) ymin = max(ymin, 0) @@ -253,13 +244,7 @@ def crop(self, box): ymax = min(ymax, current_height) assert xmin < xmax and ymin < ymax, str(box) - - if (xmin >= xmax or ymin >= ymax): - print(box) - w, h = ymax - ymin, xmax - xmin - #w = max(w, 1) - #h = max(w, 1) cropped_polygons = [] for poly in self.polygons: @@ -270,7 +255,6 @@ def crop(self, box): return PolygonInstance(cropped_polygons, size=(w, h)) - def resize(self, size): try: iter(size) @@ -278,7 +262,7 @@ def resize(self, size): assert isinstance(size, (int, float)) size = size, size - ratios = tuple(float(s) / float(s_orig) + ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(size, self.size)) if ratios[0] == ratios[1]: @@ -296,38 +280,34 @@ def resize(self, size): return PolygonInstance(scaled_polygons, size=size) - def convert_to_binarymask(self): width, height = self.size - rles = mask_utils.frPyObjects( - [p.numpy() for p in self.polygons], height, width - ) + # formatting for COCO PythonAPI + polygons = [p.numpy() for p in self.polygons] + rles = mask_utils.frPyObjects(polygons, height, width) rle = mask_utils.merge(rles) mask = mask_utils.decode(rle) mask = torch.from_numpy(mask) return mask - def __len__(self): return len(self.polygons) - def __repr__(self): - s = self.__class__.__name__ + '(' - s += 'num_groups={}, '.format(len(self.polygons)) - s += 'image_width={}, '.format(self.size[0]) - s += 'image_height={}, '.format(self.size[1]) + s = self.__class__.__name__ + "(" + s += "num_groups={}, ".format(len(self.polygons)) + s += "image_width={}, ".format(self.size[0]) + s += "image_height={}, ".format(self.size[1]) return s - class PolygonList(object): - ''' + """ This class handles PolygonInstances for all objects in the image - ''' + """ def __init__(self, polygons, size): - ''' + """ Arguments: polygons: a list of list of lists of numbers. The first @@ -345,26 +325,25 @@ def __init__(self, polygons, size): size: absolute image size - ''' + """ if isinstance(polygons, (list, tuple)): if len(polygons) == 0: polygons = [[[]]] if isinstance(polygons[0], (list, tuple)): - assert isinstance(polygons[0][0], (list, tuple)),\ - str(type(polygons[0][0])) + assert isinstance(polygons[0][0], (list, tuple)), str( + type(polygons[0][0]) + ) else: - assert isinstance(polygons[0], PolygonInstance),\ - str(type(polygons[0])) + assert isinstance(polygons[0], PolygonInstance), str(type(polygons[0])) elif isinstance(polygons, PolygonList): size = polygons.size polygons = polygons.polygons else: - RuntimeError('Type of argument `polygons` is not allowed:%s'\ - %(type(polygons))) - - + RuntimeError( + "Type of argument `polygons` is not allowed:%s" % (type(polygons)) + ) assert isinstance(size, (list, tuple)), str(type(size)) @@ -376,11 +355,10 @@ def __init__(self, polygons, size): self.size = tuple(size) - def transpose(self, method): if method not in (FLIP_LEFT_RIGHT, FLIP_TOP_BOTTOM): raise NotImplementedError( - 'Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented' + "Only FLIP_LEFT_RIGHT and FLIP_TOP_BOTTOM implemented" ) flipped_polygons = [] @@ -389,7 +367,6 @@ def transpose(self, method): return PolygonList(flipped_polygons, size=self.size) - def crop(self, box): w, h = box[2] - box[0], box[3] - box[1] cropped_polygons = [] @@ -399,7 +376,6 @@ def crop(self, box): cropped_size = w, h return PolygonList(cropped_polygons, cropped_size) - def resize(self, size): resized_polygons = [] for polygon in self.polygons: @@ -408,26 +384,21 @@ def resize(self, size): resized_size = size return PolygonList(resized_polygons, resized_size) - def to(self, *args, **kwargs): return self - def convert_to_binarymask(self): - if self.__len__() > 0: - masks = torch.stack([ - p.convert_to_binarymask() for p in self.polygons]) + if len(self) > 0: + masks = torch.stack([p.convert_to_binarymask() for p in self.polygons]) else: size = self.size masks = torch.empty([0, size[1], size[0]], dtype=torch.uint8) return BinaryMaskList(masks, size=self.size) - def __len__(self): return len(self.polygons) - def __getitem__(self, item): if isinstance(item, (int, slice)): selected_polygons = [self.polygons[item]] @@ -442,37 +413,33 @@ def __getitem__(self, item): selected_polygons.append(self.polygons[i]) return PolygonList(selected_polygons, size=self.size) - def __iter__(self): return iter(self.polygons) - def __repr__(self): - s = self.__class__.__name__ + '(' - s += 'num_instances={}, '.format(len(self.polygons)) - s += 'image_width={}, '.format(self.size[0]) - s += 'image_height={})'.format(self.size[1]) + s = self.__class__.__name__ + "(" + s += "num_instances={}, ".format(len(self.polygons)) + s += "image_width={}, ".format(self.size[0]) + s += "image_height={})".format(self.size[1]) return s - - - class SegmentationMask(object): - ''' + """ This class stores the segmentations for all objects in the image. It wraps BinaryMaskList and PolygonList conveniently. - ''' - def __init__(self, instances, size, mode='poly'): - ''' + """ + + def __init__(self, instances, size, mode="poly"): + """ Arguments: instances: two types (1) polygon (2) binary mask size: (width, height) mode: 'poly', 'mask'. if mode is 'mask', convert mask of any format to binary mask - ''' + """ assert isinstance(size, (list, tuple)) assert len(size) == 2 @@ -483,12 +450,12 @@ def __init__(self, instances, size, mode='poly'): assert isinstance(size[0], (int, float)) assert isinstance(size[1], (int, float)) - if mode == 'poly': + if mode == "poly": self.instances = PolygonList(instances, size) - elif mode == 'mask': + elif mode == "mask": self.instances = BinaryMaskList(instances, size) else: - raise NotImplementedError('Unknown mode: %s'%str(mode)) + raise NotImplementedError("Unknown mode: %s" % str(mode)) self.mode = mode self.size = tuple(size) @@ -497,59 +464,50 @@ def transpose(self, method): flipped_instances = self.instances.transpose(method) return SegmentationMask(flipped_instances, self.size, self.mode) - def crop(self, box): cropped_instances = self.instances.crop(box) cropped_size = cropped_instances.size return SegmentationMask(cropped_instances, cropped_size, self.mode) - def resize(self, size, *args, **kwargs): resized_instances = self.instances.resize(size) resized_size = size return SegmentationMask(resized_instances, resized_size, self.mode) - def to(self, *args, **kwargs): return self - def convert(self, mode): if mode == self.mode: return self - if mode == 'poly': + if mode == "poly": converted_instances = self.instances.convert_to_polygon() - elif mode == 'mask': + elif mode == "mask": converted_instances = self.instances.convert_to_binarymask() else: - raise NotImplementedError('Unknown mode: %s'%str(mode)) + raise NotImplementedError("Unknown mode: %s" % str(mode)) return SegmentationMask(converted_instances, self.size, mode) - def get_mask_tensor(self): instances = self.instances - if self.mode == 'poly': + if self.mode == "poly": instances = instances.convert_to_binarymask() # If there is only 1 instance return instances.masks.squeeze(0) - def __len__(self): return len(self.instances) - def __getitem__(self, item): selected_instances = self.instances.__getitem__(item) return SegmentationMask(selected_instances, self.size, self.mode) - def __iter__(self): self.iter_idx = 0 return self - def __next__(self): if self.iter_idx < self.__len__(): next_segmentation = self.__getitem__(self.iter_idx) @@ -557,7 +515,6 @@ def __next__(self): return next_segmentation raise StopIteration - def __repr__(self): s = self.__class__.__name__ + "(" s += "num_instances={}, ".format(len(self.instances)) From b701768252cb110e082db7f1f2535ea53ad23fe6 Mon Sep 17 00:00:00 2001 From: botcs Date: Sun, 24 Feb 2019 11:02:24 +0000 Subject: [PATCH 11/14] PEP8 black formatting --- .../structures/segmentation_mask.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 90cb5a39a..4f85b056f 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -54,19 +54,21 @@ def __init__(self, masks, size): elif isinstance(masks, (list, tuple)): if isinstance(masks[0], torch.Tensor): masks = torch.stack(masks, dim=2).clone() - elif isinstance(masks[0], dict) and 'count' in masks[0]: + elif isinstance(masks[0], dict) and "count" in masks[0]: # RLE interpretation masks = mask_utils else: - RuntimeError('Type of `masks[0]` could not be interpreted: %s'\ - %type(masks)) + RuntimeError( + "Type of `masks[0]` could not be interpreted: %s" % type(masks) + ) elif isinstance(masks, BinaryMaskList): # just hard copy the BinaryMaskList instance's underlying data masks = masks.masks.clone() else: - RuntimeError('Type of `masks` argument could not be interpreted:%s'\ - %tpye(masks)) + RuntimeError( + "Type of `masks` argument could not be interpreted:%s" % tpye(masks) + ) if len(masks.shape) == 2: # if only a single instance mask is passed @@ -140,8 +142,7 @@ def _findContours(self): reshaped_contour = [] for entity in contour: assert len(entity.shape) == 3 - assert entity.shape[1] == 1,\ - "Hierarchical contours are not allowed" + assert entity.shape[1] == 1, "Hierarchical contours are not allowed" reshaped_contour.append(entity.reshape(-1).tolist()) contours.append(reshaped_contour) return contours @@ -193,7 +194,7 @@ def __init__(self, polygons, size): else: RuntimeError( - "Type of argument `polygons` is not allowed:%s"%(type(polygons)) + "Type of argument `polygons` is not allowed:%s" % (type(polygons)) ) """ This crashes the training way too many times... @@ -262,8 +263,7 @@ def resize(self, size): assert isinstance(size, (int, float)) size = size, size - ratios = tuple(float(s) / float(s_orig) - for s, s_orig in zip(size, self.size)) + ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(size, self.size)) if ratios[0] == ratios[1]: ratio = ratios[0] From 771d75029742cc39f808519cc5683986a0a9344e Mon Sep 17 00:00:00 2001 From: botcs Date: Mon, 25 Feb 2019 20:43:47 +0000 Subject: [PATCH 12/14] Minor patch --- .../structures/segmentation_mask.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 4f85b056f..5f6bf4f9c 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -87,17 +87,22 @@ def transpose(self, method): return BinaryMaskList(flipped_masks, self.size) def crop(self, box): - assert isinstance(box, (list, tuple)), str(type(box)) - - # box is assumed to by xyxy + assert isinstance(box, (list, tuple, torch.Tensor)), str(type(box)) + # box is assumed to be xyxy current_width, current_height = self.size - xmin, ymin, xmax, ymax = map(round, box) - assert xmin >= 0 and xmax < current_width - assert ymin >= 0 and ymax < current_height - assert xmin < xmax and ymin < ymax + xmin, ymin, xmax, ymax = [round(float(b)) for b in box] - width, height = xmax - xmin, ymax - ymin + assert xmin <= xmax and ymin <= ymax, str(box) + xmin = min(max(xmin, 0), current_width - 1) + ymin = min(max(ymin, 0), current_height - 1) + xmax = min(max(xmax, 0), current_width) + ymax = min(max(ymax, 0), current_height) + + xmax = max(xmax, xmin + 1) + ymax = max(ymax, ymin + 1) + + width, height = xmax - xmin, ymax - ymin cropped_masks = self.masks[:, ymin:ymax, xmin:xmax] cropped_size = width, height return BinaryMaskList(cropped_masks, cropped_size) @@ -238,14 +243,17 @@ def crop(self, box): current_width, current_height = self.size xmin, ymin, xmax, ymax = map(float, box) - xmin = max(xmin, 0) - ymin = max(ymin, 0) + assert xmin <= xmax and ymin <= ymax, str(box) + xmin = min(max(xmin, 0), current_width - 1) + ymin = min(max(ymin, 0), current_height - 1) + + xmax = min(max(xmax, 0), current_width) + ymax = min(max(ymax, 0), current_height) - xmax = min(xmax, current_width) - ymax = min(ymax, current_height) + xmax = max(xmax, xmin + 1) + ymax = max(ymax, ymin + 1) - assert xmin < xmax and ymin < ymax, str(box) - w, h = ymax - ymin, xmax - xmin + w, h = xmax - xmin, ymax - ymin cropped_polygons = [] for poly in self.polygons: From 493e3f6a1b660c212933e24cdb8a7c217c7dc45b Mon Sep 17 00:00:00 2001 From: botcs Date: Thu, 28 Feb 2019 12:06:05 +0000 Subject: [PATCH 13/14] Use internal that handles 0 channels --- maskrcnn_benchmark/structures/segmentation_mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 5f6bf4f9c..6f98f4505 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -2,7 +2,7 @@ import torch import numpy as np -from torch.nn.functional import interpolate +from maskrcnn_benchmark.layers.misc import interpolate import pycocotools.mask as mask_utils From a037720f787b367637006a1954187ce78b8c45aa Mon Sep 17 00:00:00 2001 From: botcs Date: Thu, 7 Mar 2019 03:48:21 +0000 Subject: [PATCH 14/14] Fix polygon slicing --- maskrcnn_benchmark/structures/segmentation_mask.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maskrcnn_benchmark/structures/segmentation_mask.py b/maskrcnn_benchmark/structures/segmentation_mask.py index 6f98f4505..bf91d5568 100644 --- a/maskrcnn_benchmark/structures/segmentation_mask.py +++ b/maskrcnn_benchmark/structures/segmentation_mask.py @@ -408,8 +408,10 @@ def __len__(self): return len(self.polygons) def __getitem__(self, item): - if isinstance(item, (int, slice)): + if isinstance(item, int): selected_polygons = [self.polygons[item]] + elif isinstance(item, slice): + selected_polygons = self.polygons[item] else: # advanced indexing on a single dimension selected_polygons = []