Skip to content

Commit

Permalink
DeformableConv v8: reference implementation (openvinotoolkit#6514)
Browse files Browse the repository at this point in the history 
* DeformableConv v8:reference implementation

* ngraph codestyle

* update reference iplementation

* Disable tests on GPU

* fix unit tests

* Update DeformConv single layer tests

* fix serialization tests
  • Loading branch information
@itikhono @akuporos
itikhono authored and akuporos committed Sep 29, 2021
1 parent a5be058 commit cc18fb9
Show file tree
Hide file tree
Showing 12 changed files with 3,367 additions and 66 deletions.
1 change: 1 addition & 0 deletions inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ static void Transformation(CNNNetwork& clonedNetwork, const Config& conf) {
pass_config->disable<ngraph::pass::WeightsDequantizeToFakeQuantize>();
pass_config->disable<ngraph::pass::SimplifyCTCGreedyDecoderSeqLen>();
pass_config->disable<ngraph::pass::ConvertGather7ToGather1>();
pass_config->disable<ngraph::pass::ConvertDeformableConv8To1>();

pass_config->enable<ngraph::pass::ConvertInterpolate1ToInterpolate4>();
pass_config->enable<ngraph::pass::ConvertGather1ToGather7>();
Expand Down
10 changes: 8 additions & 2 deletions ...e/tests/functional/inference_engine/serialization/single_layer/deformable_convolution.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,22 +25,28 @@ const std::vector<std::vector<size_t>> dilations = {{1, 1}};
const std::vector<size_t> groups = {1};
const std::vector<size_t> defor_groups = {1};
const std::vector<size_t> numOutChannels = {1};
const std::vector<bool> with_bilinear_interpolation_pad = { false, true };
const std::vector<bool> with_modulated_scalar = { false, true };

const auto conv2DParams_ExplicitPadding = ::testing::Combine(
::testing::ValuesIn(offsets), ::testing::ValuesIn(filters),
::testing::ValuesIn(strides), ::testing::ValuesIn(padBegins),
::testing::ValuesIn(padEnds), ::testing::ValuesIn(dilations),
::testing::ValuesIn(groups), ::testing::ValuesIn(defor_groups),
::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));
const auto conv2DParams_AutoPadValid = ::testing::Combine(
::testing::ValuesIn(offsets), ::testing::ValuesIn(filters),
::testing::ValuesIn(strides),
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::VALID));
::testing::Values(ngraph::op::PadType::VALID),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));

INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_Serialization_ExplicitPadding, DeformableConvolutionLayerTest,
Expand Down
43 changes: 26 additions & 17 deletions ...unctional/plugin/cpu/shared_tests_instances/single_layer_tests/deformable_convolution.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,49 +1,52 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include <vector>

#include "common_test_utils/test_constants.hpp"
#include "single_layer_tests/deformable_convolution.hpp"

using namespace LayerTestsDefinitions;

namespace {

const std::vector<InferenceEngine::Precision> netPrecisions = {
InferenceEngine::Precision::FP32, InferenceEngine::Precision::FP16,
InferenceEngine::Precision::I32, InferenceEngine::Precision::I16};

/* ============= 2D DeformableConvolution ============= */
const std::vector<std::vector<size_t>> deformable_vals = {{1, 18, 28, 28}};
const std::vector<std::vector<size_t>> kernels = {{1, 1, 3, 3}};
const std::vector<std::vector<size_t>> deformable_vals = {{1, 16, 2, 2}};
const std::vector<std::vector<size_t>> kernels = {{2, 2, 2, 2}};
const std::vector<std::vector<size_t>> strides = {{1, 1}};
const std::vector<std::vector<ptrdiff_t>> padBegins = {{0, 0}};
const std::vector<std::vector<ptrdiff_t>> padEnds ={{0, 0}};
const std::vector<std::vector<size_t>> dilations = {{1, 1}};
const std::vector<size_t> groups = {1};
const std::vector<size_t> defor_groups = {1};
const std::vector<size_t> defor_groups = {2};
const std::vector<size_t> numOutChannels = {1, 5};
const std::vector<size_t> multiple_defor_groups = {4};
const std::vector<std::vector<size_t>> deform_vals = {{1, 200, 220, 220}};
const std::vector<std::vector<size_t>> kernel = {{64, 4, 5, 5}};
const std::vector<std::vector<size_t>> kernel = {{64, 16, 5, 5}};

const std::vector<bool> with_bilinear_interpolation_pad = { false, true };
const std::vector<bool> with_modulated_scalar = { false, true };

const auto deformableConv2DParams_ExplicitPadding = ::testing::Combine(
::testing::ValuesIn(deformable_vals),
::testing::ValuesIn(kernels), ::testing::ValuesIn(strides),
::testing::ValuesIn(padBegins), ::testing::ValuesIn(padEnds),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT), ::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));

const auto deformableConv2DParams_AutoPadValid = ::testing::Combine(
::testing::ValuesIn(deformable_vals),
::testing::ValuesIn(kernels), ::testing::ValuesIn(strides),
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::VALID));
::testing::Values(ngraph::op::PadType::VALID),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));

const auto deformableConv2DParams_DeformableGroups_AutoPadExplicit = ::testing::Combine(
::testing::ValuesIn(deform_vals),
Expand All @@ -52,7 +55,9 @@ const auto deformableConv2DParams_DeformableGroups_AutoPadExplicit = ::testing::
::testing::Values(std::vector<ptrdiff_t>({0, 0})),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(multiple_defor_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));

INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_ExplicitPadding, DeformableConvolutionLayerTest,
Expand All @@ -62,7 +67,7 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 1, 30, 30})),
::testing::Values(std::vector<size_t>({1, 2, 3, 3})),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
DeformableConvolutionLayerTest::getTestCaseName);

Expand All @@ -74,19 +79,20 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 1, 30, 30})),
::testing::Values(std::vector<size_t>({1, 2, 3, 3})),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
DeformableConvolutionLayerTest::getTestCaseName);

INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_DeformableGroups_ExplicitPadding, DeformableConvolutionLayerTest,
::testing::Combine(
deformableConv2DParams_DeformableGroups_AutoPadExplicit, ::testing::ValuesIn(netPrecisions),
deformableConv2DParams_DeformableGroups_AutoPadExplicit,
::testing::ValuesIn(netPrecisions),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(InferenceEngine::Layout::ANY),
::testing::Values(std::vector<size_t>({1, 4, 224, 224})),
::testing::Values(std::vector<size_t>({1, 16, 224, 224})),
::testing::Values(CommonTestUtils::DEVICE_CPU)),
DeformableConvolutionLayerTest::getTestCaseName);

Expand All @@ -101,12 +107,15 @@ const auto deformableConv2DParams_SingleTestCase = ::testing::Combine(
::testing::ValuesIn(padBegins), ::testing::ValuesIn(padEnds),
::testing::ValuesIn(dilations), ::testing::ValuesIn(groups),
::testing::ValuesIn(single_deform_groups), ::testing::ValuesIn(numOutChannels),
::testing::Values(ngraph::op::PadType::EXPLICIT));
::testing::Values(ngraph::op::PadType::EXPLICIT),
::testing::ValuesIn(with_bilinear_interpolation_pad),
::testing::ValuesIn(with_modulated_scalar));

INSTANTIATE_TEST_SUITE_P(
smoke_DeformableConvolution2D_SingleTestCase, DeformableConvolutionLayerTest,
::testing::Combine(
deformableConv2DParams_SingleTestCase, ::testing::ValuesIn(netPrecisions),
deformableConv2DParams_SingleTestCase,
::testing::ValuesIn(netPrecisions),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Precision::UNSPECIFIED),
::testing::Values(InferenceEngine::Layout::ANY),
Expand Down
4 changes: 3 additions & 1 deletion ...l/shared_test_classes/include/shared_test_classes/single_layer/deformable_convolution.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ typedef std::tuple<
size_t, // Groups
size_t, // Deformable groups
size_t, // Num out channels
ngraph::op::PadType // Padding type
ngraph::op::PadType, // Padding type
bool, // Bilinear interpolation pad
bool // Modulation
> deformableConvSpecificParams;
typedef std::tuple<
deformableConvSpecificParams,
Expand Down
62 changes: 41 additions & 21 deletions ...e-engine/tests/functional/shared_test_classes/src/single_layer/deformable_convolution.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
// Copyright (C) 2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "shared_test_classes/single_layer/deformable_convolution.hpp"

namespace LayerTestsDefinitions {

std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamInfo<deformableConvLayerTestParamsSet> obj) {
deformableConvSpecificParams convParams;
InferenceEngine::Precision netPrecision;
Expand All @@ -14,12 +12,14 @@ std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamIn
InferenceEngine::SizeVector inputShapes;
std::string targetDevice;
std::tie(convParams, netPrecision, inPrc, outPrc, inLayout, outLayout, inputShapes, targetDevice) =
obj.param;
obj.param;
ngraph::op::PadType padType;
InferenceEngine::SizeVector offsets, filter, stride, dilation;
std::vector<ptrdiff_t> padBegin, padEnd;
size_t groups, deformable_groups, convOutChannels;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType) = convParams;
bool with_bilinear_interpolation_pad, with_modulation;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType,
with_bilinear_interpolation_pad, with_modulation) = convParams;

std::ostringstream result;
result << "IS=" << CommonTestUtils::vec2str(inputShapes) << "_";
Expand All @@ -33,6 +33,8 @@ std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamIn
result << "DG=" << deformable_groups << "_";
result << "O=" << convOutChannels << "_";
result << "AP=" << padType << "_";
result << "BI_PAD=" << with_bilinear_interpolation_pad << "_";
result << "MODULATION=" << with_modulation << "_";
result << "netPRC=" << netPrecision.name() << "_";
result << "inPRC=" << inPrc.name() << "_";
result << "outPRC=" << outPrc.name() << "_";
Expand All @@ -43,29 +45,32 @@ std::string DeformableConvolutionLayerTest::getTestCaseName(testing::TestParamIn
}

InferenceEngine::Blob::Ptr DeformableConvolutionLayerTest::GenerateInput(const InferenceEngine::InputInfo &info) const {
InferenceEngine::Blob::Ptr blobPtr;
const std::string& name = info.name();
if (name == "a_data") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
} else if (name == "b_offset_vals") {
blobPtr = FuncTestUtils::createAndFillBlobFloat(info.getTensorDesc(), 2, 0, 10);
} else if (name == "c_filter_vals") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
}
return blobPtr;
InferenceEngine::Blob::Ptr blobPtr;
const std::string& name = info.name();
if (name == "a_data") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
} else if (name == "b_offset_vals") {
blobPtr = FuncTestUtils::createAndFillBlobFloat(info.getTensorDesc(), 2, 0, 10);
} else if (name == "c_filter_vals") {
blobPtr = LayerTestsUtils::LayerTestsCommon::GenerateInput(info);
} else if (name == "c_modulation_scalars") {
blobPtr = FuncTestUtils::createAndFillBlobFloat(info.getTensorDesc(), 1, 0, 20);
}
return blobPtr;
}

void DeformableConvolutionLayerTest::SetUp() {
deformableConvSpecificParams convParams;
std::vector<size_t> inputShape;
InferenceEngine::Precision netPrecision;
std::tie(convParams, netPrecision, inPrc, outPrc, inLayout, outLayout, inputShape, targetDevice) =
this->GetParam();
this->GetParam();
ngraph::op::PadType padType;
InferenceEngine::SizeVector offsets, filter, stride, dilation;
std::vector<ptrdiff_t> padBegin, padEnd;
size_t groups, deformable_groups, convOutChannels;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType) = convParams;
bool with_bilinear_interpolation_pad, with_modulation;
std::tie(offsets, filter, stride, padBegin, padEnd, dilation, groups, deformable_groups, convOutChannels, padType,
with_bilinear_interpolation_pad, with_modulation) = convParams;
auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(netPrecision);
auto params = ngraph::builder::makeParams(ngPrc, {inputShape, offsets, filter});
auto paramOuts = ngraph::helpers::convert2OutputVector(
Expand All @@ -76,9 +81,24 @@ void DeformableConvolutionLayerTest::SetUp() {
offset_vals->set_friendly_name("b_offset_vals");
auto filter_vals = std::make_shared<ngraph::op::Parameter>(ngPrc, ngraph::Shape(filter));
filter_vals->set_friendly_name("c_filter_vals");
auto deformable_conv = std::make_shared<ngraph::opset1::DeformableConvolution>(data, offset_vals, filter_vals,
stride, padBegin, padEnd, dilation, padType, groups, deformable_groups);
ngraph::ParameterVector parameters{data, offset_vals, filter_vals};
std::shared_ptr<ngraph::Node> deformable_conv;
if (with_modulation) {
auto modulation_shape = ngraph::Shape(offsets);
modulation_shape[1] = offsets[1] / 2;
auto modulation_scalars = std::make_shared<ngraph::op::Parameter>(ngPrc, modulation_shape);
modulation_scalars->set_friendly_name("c_modulation_scalars");

deformable_conv = std::make_shared<ngraph::op::v8::DeformableConvolution>(data, offset_vals, filter_vals, modulation_scalars, stride, padBegin,
padEnd, dilation, padType, groups, deformable_groups,
with_bilinear_interpolation_pad);
parameters.push_back(modulation_scalars);
} else {
deformable_conv = std::make_shared<ngraph::op::v8::DeformableConvolution>(data, offset_vals, filter_vals, stride, padBegin, padEnd, dilation,
padType, groups, deformable_groups, with_bilinear_interpolation_pad);
}

ngraph::ResultVector results{std::make_shared<ngraph::opset1::Result>(deformable_conv)};
function = std::make_shared<ngraph::Function>(results, ngraph::ParameterVector{data, offset_vals, filter_vals}, "deformable_convolution");
function = std::make_shared<ngraph::Function>(results, parameters, "deformable_convolution");
}
} // namespace LayerTestsDefinitions
} // namespace LayerTestsDefinitions
5 changes: 5 additions & 0 deletions ngraph/core/include/ngraph/op/deformable_convolution.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ namespace ngraph

void validate_and_infer_types() override;

bool evaluate(const HostTensorVector& outputs,
const HostTensorVector& inputs) const override;

bool has_evaluate() const override;

std::shared_ptr<Node>
clone_with_new_inputs(const OutputVector& new_args) const override;

Expand Down
Loading

0 comments on commit cc18fb9

Please sign in to comment.