Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DeformableConv v8: reference implementation #6514

Merged
merged 15 commits into from
Jul 19, 2021
Merged

DeformableConv v8: reference implementation #6514

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f819e33
DeformableConv v8:reference implementation
itikhono Jul 5, 2021
c660d54
ngraph codestyle
itikhono Jul 5, 2021
d0b1ef0
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 7, 2021
51717a2
update reference iplementation
itikhono Jul 7, 2021
737527c
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 7, 2021
7e97693
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 7, 2021
937be7b
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 8, 2021
be885a4
Disable tests on GPU
itikhono Jul 8, 2021
9be43e3
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 13, 2021
76bb00d
fix unit tests
itikhono Jul 14, 2021
4280835
Update DeformConv single layer tests
itikhono Jul 15, 2021
c0fd6be
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 15, 2021
25ba69c
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 16, 2021
ca738df
Merge remote-tracking branch 'upstream/master' into itikhono/ops/defo…
itikhono Jul 16, 2021
85c1915
fix serialization tests
itikhono Jul 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ngraph/core/include/ngraph/op/deformable_convolution.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ namespace ngraph

void validate_and_infer_types() override;

bool evaluate(const HostTensorVector& outputs,
itikhono marked this conversation as resolved.
Show resolved Hide resolved
const HostTensorVector& inputs) const override;

bool has_evaluate() const override;

std::shared_ptr<Node>
clone_with_new_inputs(const OutputVector& new_args) const override;

Expand Down
148 changes: 123 additions & 25 deletions ngraph/core/reference/include/ngraph/runtime/reference/deformable_convolution.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ namespace ngraph
inline void validate_deformable_convolution_params(const Shape& in_shape,
const Shape& o_shape,
const Shape& f_shape,
const Shape& m_shape,
const Shape& out_shape,
const Strides& strides,
const Strides& dilations,
Expand All @@ -29,6 +30,7 @@ namespace ngraph
NGRAPH_CHECK(in_shape.size() == 4, "Unsupported input rank: ", in_shape);
NGRAPH_CHECK(o_shape.size() == 4, "Unsupported offset rank: ", o_shape);
NGRAPH_CHECK(f_shape.size() == 4, "Unsupported kernel rank: ", f_shape);
NGRAPH_CHECK(m_shape.size() == 4, "Unsupported mask rank: ", m_shape);

NGRAPH_CHECK(in_shape[1] % groups == 0,
"Input channels of data batch input must be evenly divisible by "
Expand All @@ -53,14 +55,20 @@ namespace ngraph

const Shape f_spatial_shape{std::next(f_shape.begin(), 2), std::end(f_shape)};
const Shape o_spatial_shape{std::next(o_shape.begin(), 2), std::end(o_shape)};
const Shape m_spatial_shape{std::next(m_shape.begin(), 2), std::end(m_shape)};
const Shape out_spatial_shape{std::next(out_shape.begin(), 2),
std::end(out_shape)};

NGRAPH_CHECK(o_shape[1] == deformable_groups * shape_size(f_spatial_shape) * 2,
"The channels dimension of offsets input is not "
"compatible with filters and 'deformable group' attribute");
NGRAPH_CHECK(m_shape[1] == deformable_groups * shape_size(f_spatial_shape),
"The channels dimension of mask input is not "
"compatible with filters and 'deformable group' attribute");
NGRAPH_CHECK(out_spatial_shape == o_spatial_shape,
"Spatial dimensions of output and offsets values must be equal");
NGRAPH_CHECK(out_spatial_shape == m_spatial_shape,
"Spatial dimensions of output and mask values must be equal");
}

inline Shape shape_reduce(const Shape& s) { return Shape(++s.begin(), s.end()); }
Expand All @@ -76,20 +84,37 @@ namespace ngraph
const float x_idx,
const float y_idx,
const int x_size,
const int y_size)
const int y_size,
const bool use_pad)
{
const int x1 = std::max(static_cast<int>(std::floor(x_idx)), 0);
const int x2 = std::min(static_cast<int>(std::ceil(x_idx)), x_size - 1);
const int y1 = std::max(static_cast<int>(std::floor(y_idx)), 0);
const int y2 = std::min(static_cast<int>(std::ceil(y_idx)), y_size - 1);
const int y1 = use_pad ? static_cast<int>(std::floor(y_idx))
: std::max(static_cast<int>(std::floor(y_idx)), 0);
const int x1 = use_pad ? static_cast<int>(std::floor(x_idx))
: std::max(static_cast<int>(std::floor(x_idx)), 0);

const int y2 =
use_pad ? y1 + 1 : std::min(static_cast<int>(std::ceil(y_idx)), y_size - 1);
const int x2 =
use_pad ? x1 + 1 : std::min(static_cast<int>(std::ceil(x_idx)), x_size - 1);

const float distX = x_idx - x1;
const float distY = y_idx - y1;

const float value11 = data[y1 * x_size + x1];
const float value12 = data[y2 * x_size + x1];
const float value21 = data[y1 * x_size + x2];
const float value22 = data[y2 * x_size + x2];
float value11 = 0;
itikhono marked this conversation as resolved.
Show resolved Hide resolved
if (y1 >= 0 && x1 >= 0)
value11 = data[y1 * x_size + x1];

float value21 = 0;
itikhono marked this conversation as resolved.
Show resolved Hide resolved
if (y1 >= 0 && x2 < x_size)
value21 = data[y1 * x_size + x2];

float value12 = 0;
if (y2 < y_size && x1 >= 0)
value12 = data[y2 * x_size + x1];

float value22 = 0;
if (y2 < y_size && x2 < x_size)
value22 = data[y2 * x_size + x2];

const float value = (1 - distX) * (1 - distY) * value11 +
(1 - distX) * distY * value12 +
Expand All @@ -105,10 +130,13 @@ namespace ngraph
const Shape& offset_shape,
const T* filter,
const Shape& filter_shape,
const T* mask,
const Shape& mask_shape,
T* out,
size_t group_idx,
int64_t groups,
int64_t deformable_groups)
int64_t deformable_groups,
bool bilinear_interpolation_pad)
{
const int input_size_y = batch_shape[1];
const int input_size_x = batch_shape[2];
Expand All @@ -124,6 +152,8 @@ namespace ngraph
const int offsets_size = shape_size(offset_shape);
const int offsets_spatial_size = shape_size(shape_reduce(offset_shape));
const int filter_channels_count = filter_shape[0];
const int mask_size = shape_size(mask_shape);
const int mask_spatial_size = shape_size(shape_reduce(mask_shape));

int out_idx = 0;
for (int i_y = -p.pads_begin[0];
Expand All @@ -146,29 +176,45 @@ namespace ngraph
{
for (int f_x = 0; f_x < filter_size_x; ++f_x)
{
T y_offset = offsets[deformable_group_idx * offsets_size +
(f_y * filter_size_x + f_x) * 2 *
offsets_spatial_size +
out_idx];
T x_offset = offsets[deformable_group_idx * offsets_size +
((f_y * filter_size_x + f_x) * 2 + 1) *
offsets_spatial_size +
out_idx];
int f_buf_idx = (f_y * filter_size_x) + f_x;
T y_offset =
offsets[deformable_group_idx * offsets_size +
f_buf_idx * 2 * offsets_spatial_size + out_idx];
T x_offset =
offsets[deformable_group_idx * offsets_size +
(f_buf_idx * 2 + 1) * offsets_spatial_size +
out_idx];
T rel_i_y = i_y + (f_y * p.dilation[0]) + y_offset;
T rel_i_x = i_x + (f_x * p.dilation[1]) + x_offset;

bool padding = !(in_range(rel_i_x, {0, input_size_x}) &&
in_range(rel_i_y, {0, input_size_y}));
bool padding;
if (bilinear_interpolation_pad)
{
padding =
!((static_cast<int>(rel_i_x) > -1 &&
static_cast<int>(rel_i_x) < input_size_x) &&
(static_cast<int>(rel_i_y) > -1 &&
static_cast<int>(rel_i_y) < input_size_y));
}
else
{
padding = !(in_range(rel_i_x, {0, input_size_x}) &&
in_range(rel_i_y, {0, input_size_y}));
}

if (padding)
continue;

int f_buf_idx = (f_y * filter_size_x) + f_x;
T mask_scalar =
mask[deformable_group_idx * mask_size +
f_buf_idx * mask_spatial_size + out_idx];
sum += bilinear_interpolation(input_channel,
rel_i_x,
rel_i_y,
input_size_x,
input_size_y) *
filter_channel[f_buf_idx];
input_size_y,
bilinear_interpolation_pad) *
filter_channel[f_buf_idx] * mask_scalar;
}
}
input_channel += input_channel_size;
Expand All @@ -180,28 +226,33 @@ namespace ngraph
}

} // namespace def_conv_impl

template <typename T>
void deformable_convolution(const T* in,
const T* offsets,
const T* filters,
const T* mask,
T* out,
const Shape& in_shape,
const Shape& o_shape,
const Shape& f_shape,
const Shape& m_shape,
const Shape& out_shape,
const Strides& strides,
const Strides& dilation,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const int64_t groups,
const int64_t deformable_groups)
const int64_t deformable_groups,
const bool bilinear_interpolation_pad)

{
using namespace def_conv_impl;

validate_deformable_convolution_params(in_shape,
o_shape,
f_shape,
m_shape,
out_shape,
strides,
dilation,
Expand All @@ -227,12 +278,17 @@ namespace ngraph
const Shape group_filter_shape = shape_reduce(f_shape);
const size_t group_filter_size = shape_size(group_filter_shape);

const Shape group_mask_shape =
shape_scale(shape_reduce(m_shape), deformable_groups);
const size_t group_mask_batch_size = shape_size(shape_reduce(m_shape));

const size_t out_ch_size = shape_size(shape_reduce(shape_reduce(out_shape)));

for (size_t batch_idx = 0; batch_idx < batches_count; ++batch_idx)
{
const T* group_filters = filters;
const T* group_offsets = offsets;
const T* group_mask = mask;
for (size_t group_idx = 0; group_idx < groups_count; ++group_idx)
{
for (size_t f_idx = 0; f_idx < group_filters_count; ++f_idx)
Expand All @@ -244,18 +300,60 @@ namespace ngraph
group_offset_shape,
group_filters,
group_filter_shape,
group_mask,
group_mask_shape,
out,
group_idx,
groups,
deformable_groups);
deformable_groups,
bilinear_interpolation_pad);
group_filters += group_filter_size;
out += out_ch_size;
}
in += group_in_size;
}
offsets += group_offset_batch_size;
mask += group_mask_batch_size;
}
}

template <typename T>
void deformable_convolution(const T* in,
const T* offsets,
const T* filters,
T* out,
const Shape& in_shape,
const Shape& o_shape,
const Shape& f_shape,
const Shape& out_shape,
const Strides& strides,
const Strides& dilation,
const CoordinateDiff& pads_begin,
const CoordinateDiff& pads_end,
const int64_t groups,
const int64_t deformable_groups,
const bool bilinear_interpolation_pad = false)
{
Shape m_shape = {o_shape[0], o_shape[1] / 2, o_shape[2], o_shape[3]};
std::vector<T> mask(ngraph::shape_size(m_shape), 1);
deformable_convolution(in,
offsets,
filters,
mask.data(),
out,
in_shape,
o_shape,
f_shape,
m_shape,
out_shape,
strides,
dilation,
pads_begin,
pads_end,
groups,
deformable_groups,
bilinear_interpolation_pad);
}
} // namespace reference
} // namespace runtime
} // namespace ngraph
Loading