From 7d8000a1e48145fd17b50ce9c526f98649d84592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=B0=B9=E7=A1=95?= Date: Sat, 13 Mar 2021 19:45:58 +0800 Subject: [PATCH 1/5] Add down_sampling.py for generating LQ image from GT image, which is required in LIIF. --- mmedit/datasets/pipelines/__init__.py | 3 +- mmedit/datasets/pipelines/down_sampling.py | 73 ++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 mmedit/datasets/pipelines/down_sampling.py diff --git a/mmedit/datasets/pipelines/__init__.py b/mmedit/datasets/pipelines/__init__.py index d15aebbe37..fafbaefa8f 100644 --- a/mmedit/datasets/pipelines/__init__.py +++ b/mmedit/datasets/pipelines/__init__.py @@ -5,6 +5,7 @@ from .compose import Compose from .crop import (Crop, CropAroundCenter, CropAroundFg, CropAroundUnknown, FixedCrop, ModCrop, PairedRandomCrop) +from .down_sampling import DownSampling from .formating import (Collect, FormatTrimap, GetMaskedImage, ImageToTensor, ToTensor) from .loading import (GetSpatialDiscountMask, LoadImageFromFile, @@ -25,6 +26,6 @@ 'MergeFgAndBg', 'CompositeFg', 'TemporalReverse', 'LoadImageFromFileList', 'GenerateFrameIndices', 'GenerateFrameIndiceswithPadding', 'FixedCrop', 'LoadPairedImageFromFile', 'GenerateSoftSeg', 'GenerateSeg', 'PerturbBg', - 'CropAroundFg', 'GetSpatialDiscountMask', + 'CropAroundFg', 'GetSpatialDiscountMask', 'DownSampling', 'GenerateTrimapWithDistTransform', 'TransformTrimap' ] diff --git a/mmedit/datasets/pipelines/down_sampling.py b/mmedit/datasets/pipelines/down_sampling.py new file mode 100644 index 0000000000..ad76bbfa20 --- /dev/null +++ b/mmedit/datasets/pipelines/down_sampling.py @@ -0,0 +1,73 @@ +import math +import random + +import numpy as np +import torch +from PIL import Image +from torchvision import transforms + +from ..registry import PIPELINES + + +@PIPELINES.register_module() +class DownSampling: + """Generate LQ image from GT (and crop). + + Args: + scale_min (int): The minimum of upsampling scale. Default: 1. + scale_max (int): The maximum of upsampling scale. Default: 4. + inp_size (int): The input size, i.e. cropped lr patch size. + Default: None, means no crop. + """ + + def __init__(self, scale_min=1, scale_max=4, inp_size=None): + assert scale_max >= scale_min + self.scale_min = scale_min + self.scale_max = scale_max + self.inp_size = inp_size + + def __call__(self, results): + """Call function. + + Args: + results (dict): A dict containing the necessary information and + data for augmentation. + + Returns: + dict: A dict containing the processed data and information. + """ + img = results['gt'] + scale = random.uniform(self.scale_min, self.scale_max) + if self.inp_size is None: + h_lr = math.floor(img.shape[-3] / scale + 1e-9) + w_lr = math.floor(img.shape[-2] / scale + 1e-9) + img = img[:round(h_lr * scale), :round(w_lr * scale), :] + img_down = resize_fn(img, (w_lr, h_lr)) + crop_lr, crop_hr = img_down, img + else: + w_lr = self.inp_size + w_hr = round(w_lr * scale) + x0 = random.randint(0, img.shape[-3] - w_hr) + y0 = random.randint(0, img.shape[-2] - w_hr) + crop_hr = img[x0:x0 + w_hr, y0:y0 + w_hr, :] + crop_lr = resize_fn(crop_hr, w_lr) + results['gt'] = crop_hr + results['lq'] = crop_lr + results['scale'] = scale + + return results + + +def resize_fn(img, size): + if isinstance(size, int): + size = (size, size) + if isinstance(img, np.ndarray): + return np.asarray(Image.fromarray(img).resize(size, Image.BICUBIC)) + elif isinstance(img, torch.Tensor): + return transforms.ToTensor()( + transforms.Resize(size, + Image.BICUBIC)(transforms.ToPILImage()(img))) + + else: + raise TypeError('img should got np.ndarray or torch.Tensor,' + f'but got {type(img)}') From d19716772c3a4ce14b954ed5bf8088cdaad9b778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=B0=B9=E7=A1=95?= Date: Sat, 13 Mar 2021 21:11:14 +0800 Subject: [PATCH 2/5] Add '__repr__' and test_down_sampling.py. --- mmedit/datasets/pipelines/down_sampling.py | 8 +++++++ tests/test_down_sampling.py | 26 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/test_down_sampling.py diff --git a/mmedit/datasets/pipelines/down_sampling.py b/mmedit/datasets/pipelines/down_sampling.py index ad76bbfa20..19b183fe91 100644 --- a/mmedit/datasets/pipelines/down_sampling.py +++ b/mmedit/datasets/pipelines/down_sampling.py @@ -57,6 +57,14 @@ def __call__(self, results): return results + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'scale_min={self.scale_min}, ' + f'scale_max={self.scale_max}, ' + f'inp_size={self.inp_size}') + + return repr_str + def resize_fn(img, size): if isinstance(size, int): diff --git a/tests/test_down_sampling.py b/tests/test_down_sampling.py new file mode 100644 index 0000000000..e4545b3e34 --- /dev/null +++ b/tests/test_down_sampling.py @@ -0,0 +1,26 @@ +import numpy as np + +from mmedit.datasets.pipelines import DownSampling + + +def test_down_sampling(): + img = np.uint8(np.random.randn(480, 640, 3) * 255) + inputs = dict(gt=img) + + down_sampling1 = DownSampling(scale_min=1, scale_max=4, inp_size=None) + results1 = down_sampling1(inputs) + assert set(list(results1.keys())) == set(['gt', 'lq', 'scale']) + assert repr(down_sampling1) == ( + down_sampling1.__class__.__name__ + + f'scale_min={down_sampling1.scale_min}, ' + + f'scale_max={down_sampling1.scale_max}, ' + + f'inp_size={down_sampling1.inp_size}') + + down_sampling2 = DownSampling(scale_min=1, scale_max=4, inp_size=48) + results2 = down_sampling2(inputs) + assert set(list(results2.keys())) == set(['gt', 'lq', 'scale']) + assert repr(down_sampling2) == ( + down_sampling2.__class__.__name__ + + f'scale_min={down_sampling2.scale_min}, ' + + f'scale_max={down_sampling2.scale_max}, ' + + f'inp_size={down_sampling2.inp_size}') From 1c397ce32f14cebc640ea4913f6d93f4e74e9882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=B0=B9=E7=A1=95?= Date: Sat, 13 Mar 2021 21:46:21 +0800 Subject: [PATCH 3/5] Add docstring, rename parameter and change the function of resize. --- mmedit/datasets/pipelines/down_sampling.py | 35 ++++++++++++---------- tests/test_down_sampling.py | 19 ++++++------ 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/mmedit/datasets/pipelines/down_sampling.py b/mmedit/datasets/pipelines/down_sampling.py index 19b183fe91..8de5b19331 100644 --- a/mmedit/datasets/pipelines/down_sampling.py +++ b/mmedit/datasets/pipelines/down_sampling.py @@ -3,8 +3,7 @@ import numpy as np import torch -from PIL import Image -from torchvision import transforms +from mmcv import imresize from ..registry import PIPELINES @@ -14,38 +13,44 @@ class DownSampling: """Generate LQ image from GT (and crop). Args: - scale_min (int): The minimum of upsampling scale. Default: 1. - scale_max (int): The maximum of upsampling scale. Default: 4. - inp_size (int): The input size, i.e. cropped lr patch size. + scale_min (int): The minimum of upsampling scale, inclusive. + Default: 1. + scale_max (int): The maximum of upsampling scale, exclusive. + Default: 4. + patch_size (int): The cropped lr patch size. Default: None, means no crop. + + Scale will be in the range of [scale_min, scale_max). """ - def __init__(self, scale_min=1, scale_max=4, inp_size=None): + def __init__(self, scale_min=1, scale_max=4, patch_size=None): assert scale_max >= scale_min self.scale_min = scale_min self.scale_max = scale_max - self.inp_size = inp_size + self.patch_size = patch_size def __call__(self, results): """Call function. Args: results (dict): A dict containing the necessary information and - data for augmentation. + data for augmentation. 'gt' is required. Returns: dict: A dict containing the processed data and information. + modified 'gt', supplement 'lq' and 'scale' to keys. """ img = results['gt'] scale = random.uniform(self.scale_min, self.scale_max) - if self.inp_size is None: + + if self.patch_size is None: h_lr = math.floor(img.shape[-3] / scale + 1e-9) w_lr = math.floor(img.shape[-2] / scale + 1e-9) img = img[:round(h_lr * scale), :round(w_lr * scale), :] img_down = resize_fn(img, (w_lr, h_lr)) crop_lr, crop_hr = img_down, img else: - w_lr = self.inp_size + w_lr = self.patch_size w_hr = round(w_lr * scale) x0 = random.randint(0, img.shape[-3] - w_hr) y0 = random.randint(0, img.shape[-2] - w_hr) @@ -61,7 +66,7 @@ def __repr__(self): repr_str = self.__class__.__name__ repr_str += (f'scale_min={self.scale_min}, ' f'scale_max={self.scale_max}, ' - f'inp_size={self.inp_size}') + f'patch_size={self.patch_size}') return repr_str @@ -70,11 +75,11 @@ def resize_fn(img, size): if isinstance(size, int): size = (size, size) if isinstance(img, np.ndarray): - return np.asarray(Image.fromarray(img).resize(size, Image.BICUBIC)) + return imresize(img, size, interpolation='bicubic', backend='pillow') elif isinstance(img, torch.Tensor): - return transforms.ToTensor()( - transforms.Resize(size, - Image.BICUBIC)(transforms.ToPILImage()(img))) + image = imresize( + img.numpy(), size, interpolation='bicubic', backend='pillow') + return torch.from_numpy(image) else: raise TypeError('img should got np.ndarray or torch.Tensor,' diff --git a/tests/test_down_sampling.py b/tests/test_down_sampling.py index e4545b3e34..d814e0e58e 100644 --- a/tests/test_down_sampling.py +++ b/tests/test_down_sampling.py @@ -4,23 +4,24 @@ def test_down_sampling(): - img = np.uint8(np.random.randn(480, 640, 3) * 255) - inputs = dict(gt=img) - - down_sampling1 = DownSampling(scale_min=1, scale_max=4, inp_size=None) - results1 = down_sampling1(inputs) + img1 = np.uint8(np.random.randn(480, 640, 3) * 255) + inputs1 = dict(gt=img1) + down_sampling1 = DownSampling(scale_min=1, scale_max=4, patch_size=None) + results1 = down_sampling1(inputs1) assert set(list(results1.keys())) == set(['gt', 'lq', 'scale']) assert repr(down_sampling1) == ( down_sampling1.__class__.__name__ + f'scale_min={down_sampling1.scale_min}, ' + f'scale_max={down_sampling1.scale_max}, ' + - f'inp_size={down_sampling1.inp_size}') + f'patch_size={down_sampling1.patch_size}') - down_sampling2 = DownSampling(scale_min=1, scale_max=4, inp_size=48) - results2 = down_sampling2(inputs) + img2 = np.uint8(np.random.randn(480, 640, 3) * 255) + inputs2 = dict(gt=img2) + down_sampling2 = DownSampling(scale_min=1, scale_max=4, patch_size=48) + results2 = down_sampling2(inputs2) assert set(list(results2.keys())) == set(['gt', 'lq', 'scale']) assert repr(down_sampling2) == ( down_sampling2.__class__.__name__ + f'scale_min={down_sampling2.scale_min}, ' + f'scale_max={down_sampling2.scale_max}, ' + - f'inp_size={down_sampling2.inp_size}') + f'patch_size={down_sampling2.patch_size}') From 3963b73abbf44d4240e0da9e38eee1c501ad9dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=B0=B9=E7=A1=95?= Date: Sun, 14 Mar 2021 13:00:30 +0800 Subject: [PATCH 4/5] Fine-tuning code and docstring of RandomDownSampling class. --- mmedit/datasets/pipelines/__init__.py | 4 +- mmedit/datasets/pipelines/down_sampling.py | 67 ++++++++++++++++------ tests/test_down_sampling.py | 8 ++- 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/mmedit/datasets/pipelines/__init__.py b/mmedit/datasets/pipelines/__init__.py index fafbaefa8f..fbb558196c 100644 --- a/mmedit/datasets/pipelines/__init__.py +++ b/mmedit/datasets/pipelines/__init__.py @@ -5,7 +5,7 @@ from .compose import Compose from .crop import (Crop, CropAroundCenter, CropAroundFg, CropAroundUnknown, FixedCrop, ModCrop, PairedRandomCrop) -from .down_sampling import DownSampling +from .down_sampling import RandomDownSampling from .formating import (Collect, FormatTrimap, GetMaskedImage, ImageToTensor, ToTensor) from .loading import (GetSpatialDiscountMask, LoadImageFromFile, @@ -26,6 +26,6 @@ 'MergeFgAndBg', 'CompositeFg', 'TemporalReverse', 'LoadImageFromFileList', 'GenerateFrameIndices', 'GenerateFrameIndiceswithPadding', 'FixedCrop', 'LoadPairedImageFromFile', 'GenerateSoftSeg', 'GenerateSeg', 'PerturbBg', - 'CropAroundFg', 'GetSpatialDiscountMask', 'DownSampling', + 'CropAroundFg', 'GetSpatialDiscountMask', 'RandomDownSampling', 'GenerateTrimapWithDistTransform', 'TransformTrimap' ] diff --git a/mmedit/datasets/pipelines/down_sampling.py b/mmedit/datasets/pipelines/down_sampling.py index 8de5b19331..7c8056413e 100644 --- a/mmedit/datasets/pipelines/down_sampling.py +++ b/mmedit/datasets/pipelines/down_sampling.py @@ -1,5 +1,4 @@ import math -import random import numpy as np import torch @@ -9,25 +8,41 @@ @PIPELINES.register_module() -class DownSampling: - """Generate LQ image from GT (and crop). +class RandomDownSampling: + """Generate LQ image from GT (and crop), which will randomly pick a scale. Args: - scale_min (int): The minimum of upsampling scale, inclusive. - Default: 1. - scale_max (int): The maximum of upsampling scale, exclusive. - Default: 4. + scale_min (float): The minimum of upsampling scale, inclusive. + Default: 1.0. + scale_max (float): The maximum of upsampling scale, exclusive. + Default: 4.0. patch_size (int): The cropped lr patch size. Default: None, means no crop. - - Scale will be in the range of [scale_min, scale_max). + interpolation (str): Interpolation method, accepted values are + "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' + backend, "nearest", "bilinear", "bicubic", "box", "lanczos", + "hamming" for 'pillow' backend. + Default: "bicubic". + backend (str | None): The image resize backend type. Options are `cv2`, + `pillow`, `None`. If backend is None, the global imread_backend + specified by ``mmcv.use_backend()`` will be used. + Default: "pillow". + + Scale will be picked in the range of [scale_min, scale_max). """ - def __init__(self, scale_min=1, scale_max=4, patch_size=None): + def __init__(self, + scale_min=1.0, + scale_max=4.0, + patch_size=None, + interpolation='bicubic', + backend='pillow'): assert scale_max >= scale_min self.scale_min = scale_min self.scale_max = scale_max self.patch_size = patch_size + self.interpolation = interpolation + self.backend = backend def __call__(self, results): """Call function. @@ -41,21 +56,23 @@ def __call__(self, results): modified 'gt', supplement 'lq' and 'scale' to keys. """ img = results['gt'] - scale = random.uniform(self.scale_min, self.scale_max) + scale = np.random.uniform(self.scale_min, self.scale_max) if self.patch_size is None: h_lr = math.floor(img.shape[-3] / scale + 1e-9) w_lr = math.floor(img.shape[-2] / scale + 1e-9) img = img[:round(h_lr * scale), :round(w_lr * scale), :] - img_down = resize_fn(img, (w_lr, h_lr)) + img_down = resize_fn(img, (w_lr, h_lr), self.interpolation, + self.backend) crop_lr, crop_hr = img_down, img else: w_lr = self.patch_size w_hr = round(w_lr * scale) - x0 = random.randint(0, img.shape[-3] - w_hr) - y0 = random.randint(0, img.shape[-2] - w_hr) + x0 = np.random.randint(0, img.shape[-3] - w_hr) + y0 = np.random.randint(0, img.shape[-2] - w_hr) crop_hr = img[x0:x0 + w_hr, y0:y0 + w_hr, :] - crop_lr = resize_fn(crop_hr, w_lr) + crop_lr = resize_fn(crop_hr, w_lr, self.interpolation, + self.backend) results['gt'] = crop_hr results['lq'] = crop_lr results['scale'] = scale @@ -71,7 +88,25 @@ def __repr__(self): return repr_str -def resize_fn(img, size): +def resize_fn(img, size, interpolation='bicubic', backend='pillow'): + """Resize the given image to a given size. + + Args: + img (ndarray | torch.Tensor): The input image. + size (int | tuple[int]): Target size w or (w, h). + interpolation (str): Interpolation method, accepted values are + "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' + backend, "nearest", "bilinear", "bicubic", "box", "lanczos", + "hamming" for 'pillow' backend. + Default: "bicubic". + backend (str | None): The image resize backend type. Options are `cv2`, + `pillow`, `None`. If backend is None, the global imread_backend + specified by ``mmcv.use_backend()`` will be used. + Default: "pillow". + + Returns: + ndarray | torch.Tensor: `resized_img`, whose type is same as `img`. + """ if isinstance(size, int): size = (size, size) if isinstance(img, np.ndarray): diff --git a/tests/test_down_sampling.py b/tests/test_down_sampling.py index d814e0e58e..d761f9aaa2 100644 --- a/tests/test_down_sampling.py +++ b/tests/test_down_sampling.py @@ -1,12 +1,13 @@ import numpy as np -from mmedit.datasets.pipelines import DownSampling +from mmedit.datasets.pipelines import RandomDownSampling def test_down_sampling(): img1 = np.uint8(np.random.randn(480, 640, 3) * 255) inputs1 = dict(gt=img1) - down_sampling1 = DownSampling(scale_min=1, scale_max=4, patch_size=None) + down_sampling1 = RandomDownSampling( + scale_min=1, scale_max=4, patch_size=None) results1 = down_sampling1(inputs1) assert set(list(results1.keys())) == set(['gt', 'lq', 'scale']) assert repr(down_sampling1) == ( @@ -17,7 +18,8 @@ def test_down_sampling(): img2 = np.uint8(np.random.randn(480, 640, 3) * 255) inputs2 = dict(gt=img2) - down_sampling2 = DownSampling(scale_min=1, scale_max=4, patch_size=48) + down_sampling2 = RandomDownSampling( + scale_min=1, scale_max=4, patch_size=48) results2 = down_sampling2(inputs2) assert set(list(results2.keys())) == set(['gt', 'lq', 'scale']) assert repr(down_sampling2) == ( From 18b8c702d6212f1796e87085fd44658bf3a9264a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=B0=B9=E7=A1=95?= Date: Mon, 15 Mar 2021 13:38:19 +0800 Subject: [PATCH 5/5] Remove hardcode of bicubic and pillow. --- mmedit/datasets/pipelines/down_sampling.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mmedit/datasets/pipelines/down_sampling.py b/mmedit/datasets/pipelines/down_sampling.py index 7c8056413e..346a7ff024 100644 --- a/mmedit/datasets/pipelines/down_sampling.py +++ b/mmedit/datasets/pipelines/down_sampling.py @@ -110,10 +110,11 @@ def resize_fn(img, size, interpolation='bicubic', backend='pillow'): if isinstance(size, int): size = (size, size) if isinstance(img, np.ndarray): - return imresize(img, size, interpolation='bicubic', backend='pillow') + return imresize( + img, size, interpolation=interpolation, backend=backend) elif isinstance(img, torch.Tensor): image = imresize( - img.numpy(), size, interpolation='bicubic', backend='pillow') + img.numpy(), size, interpolation=interpolation, backend=backend) return torch.from_numpy(image) else: