Skip to content

Commit

Permalink
[PP GAPI] Extended preprocessing graph to support precision conversio…
Browse files Browse the repository at this point in the history 
…ns (openvinotoolkit#2290)

- not yet visible via plugin interface
- for resize non U8 input is converted to  FP32
- tests
  • Loading branch information
@anton-potapov @mryzhov
anton-potapov authored and mryzhov committed Dec 15, 2020
1 parent 7d9ffbc commit f3362fa
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 58 deletions.
63 changes: 45 additions & 18 deletions inference-engine/src/preprocessing/ie_preprocess_gapi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@

namespace InferenceEngine {
namespace {
int get_cv_depth(const TensorDesc &ie_desc);

namespace G {
struct Strides {int N; int C; int H; int W;};
struct Dims {int N; int C; int H; int W;};
struct Desc {Dims d; Strides s;};
struct Desc {Dims d; Strides s; int prec;};

void fix_strides_nhwc(const Dims &d, Strides &s) {
if (s.W > d.C) {
Expand Down Expand Up @@ -66,7 +68,7 @@ namespace G {

if (nhwc_layout) fix_strides_nhwc(d, s);

return Desc{d, s};
return Desc{d, s, get_cv_depth(ie_desc)};
}

Desc decompose(const Blob::Ptr& blob) {
Expand All @@ -78,6 +80,8 @@ inline int get_cv_depth(const TensorDesc &ie_desc) {
switch (ie_desc.getPrecision()) {
case Precision::U8: return CV_8U;
case Precision::FP32: return CV_32F;
case Precision::U16: return CV_16U;

default: THROW_IE_EXCEPTION << "Unsupported data type";
}
}
Expand Down Expand Up @@ -371,6 +375,7 @@ G::Desc getGDesc(G::Desc in_desc_y, const NV12Blob::Ptr &) {
auto nv12_desc = G::Desc{};
nv12_desc.d = in_desc_y.d;
nv12_desc.d.C = 2;
nv12_desc.prec = in_desc_y.prec;

return nv12_desc;
}
Expand All @@ -379,6 +384,7 @@ G::Desc getGDesc(G::Desc in_desc_y, const I420Blob::Ptr &) {
auto i420_desc = G::Desc{};
i420_desc.d = in_desc_y.d;
i420_desc.d.C = 3;
i420_desc.prec = in_desc_y.prec;

return i420_desc;
}
Expand Down Expand Up @@ -543,8 +549,7 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
Layout out_layout,
ResizeAlgorithm algorithm,
ColorFormat input_color_format,
ColorFormat output_color_format,
int precision) {
ColorFormat output_color_format) {
// perform basic validation to ensure our assumptions about input and output are correct
validateColorFormats(in_desc, out_desc, in_layout, out_layout, input_color_format,
output_color_format);
Expand All @@ -569,7 +574,7 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
(io_color_formats == std::make_tuple(ColorFormat::BGRX, ColorFormat::BGR));
const bool specific_case_of_preproc = ((in_layout == NHWC || specific_yuv420_input_handling)
&& (in_desc.d.C == 3 || specific_yuv420_input_handling || drop_channel)
&& (precision == CV_8U)
&& ((in_desc.prec == CV_8U) && (in_desc.prec == out_desc.prec))
&& (algorithm == RESIZE_BILINEAR)
&& (input_color_format == ColorFormat::RAW
|| input_color_format == output_color_format
Expand All @@ -591,9 +596,9 @@ cv::GComputation buildGraph(const G::Desc &in_desc,

auto planes = drop_channel ?
to_vec(gapi::ScalePlanes4:: on(
color_converted_input[0], precision, input_sz, scale_sz, cv::INTER_LINEAR))
color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR))
: to_vec(gapi::ScalePlanes ::on(
color_converted_input[0], precision, input_sz, scale_sz, cv::INTER_LINEAR));
color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR));

if (drop_channel) {
planes.pop_back();
Expand Down Expand Up @@ -623,8 +628,13 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
<< number_of_planes << " != " << out_desc.d.C;
}

const int tmp_prec = CV_32F;

std::vector<cv::GMat> outputs;
if (algorithm != NO_RESIZE) {
const bool resize_needed = (algorithm != NO_RESIZE);
const bool need_tmp_prec_conv = resize_needed && (in_desc.prec != CV_8U) && (in_desc.prec != CV_32F);

if (resize_needed) {
// resize every plane
std::vector<cv::GMat> out_planes;
out_planes.reserve(planes.size());
Expand All @@ -635,18 +645,36 @@ cv::GComputation buildGraph(const G::Desc &in_desc,
default: THROW_IE_EXCEPTION << "Unsupported resize operation";
}
} (algorithm);
const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H);
const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H);
const auto scale_fcn = std::bind(&gapi::ScalePlane::on,
std::placeholders::_1,
precision,
input_sz, scale_sz, interp_type);
std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), scale_fcn);

std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), [&](const cv::GMat& m) {
const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H);
const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H);

cv::GMat converted = m;
int prec = in_desc.prec;

if (need_tmp_prec_conv) {
std::tie(converted, prec) = std::make_tuple(gapi::ConvertDepth::on(m, tmp_prec), tmp_prec);
}

return gapi::ScalePlane::on(converted, prec, input_sz, scale_sz, interp_type);
});
outputs = out_planes;
} else {
outputs = planes;
}

if ((in_desc.prec != out_desc.prec) || need_tmp_prec_conv) {
auto convert_prec = [](const std::vector<cv::GMat> & src_gmats, int dst_precision) {
std::vector<cv::GMat> dst_gmats;
std::transform(src_gmats.begin(), src_gmats.end(), std::back_inserter(dst_gmats), [&](cv::GMat const& m){
return gapi::ConvertDepth::on(m, dst_precision);
});
return dst_gmats;
};

outputs = convert_prec(outputs, out_desc.prec);
}
// convert to interleaved if NHWC is required as output
if (out_layout == NHWC) {
outputs = merge(outputs, out_desc.d.C);
Expand Down Expand Up @@ -939,8 +967,7 @@ bool PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &o
out_layout,
algorithm,
in_fmt,
out_fmt,
get_cv_depth(in_desc_ie)));
out_fmt));
}
}

Expand All @@ -953,7 +980,7 @@ bool PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &o
return true;
}

bool PreprocEngine::preprocessWithGAPI(Blob::Ptr &inBlob, Blob::Ptr &outBlob,
bool PreprocEngine::preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob,
const ResizeAlgorithm& algorithm, ColorFormat in_fmt, bool omp_serial, int batch_size) {
if (!useGAPI()) {
return false;
Expand Down
2 changes: 1 addition & 1 deletion inference-engine/src/preprocessing/ie_preprocess_gapi.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class PreprocEngine {
static bool useGAPI();
static void checkApplicabilityGAPI(const Blob::Ptr &src, const Blob::Ptr &dst);
static int getCorrectBatchSize(int batch_size, const Blob::Ptr& roiBlob);
bool preprocessWithGAPI(Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm,
bool preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm,
ColorFormat in_fmt, bool omp_serial, int batch_size = -1);
};

Expand Down
96 changes: 65 additions & 31 deletions inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

#include <map>

#include <stdexcept>

#include <fluid_test_computations.hpp>

// Can be set externally (via CMake) if built with -DGAPI_TEST_PERF=ON
Expand Down Expand Up @@ -82,6 +84,7 @@ cv::String depthToString(int depth)
{
case CV_8U : return "CV_8U";
case CV_32F : return "CV_32F";
case CV_16U : return "CV_16U";
}
CV_Assert(!"ERROR: unsupported depth!");
return nullptr;
Expand Down Expand Up @@ -1078,14 +1081,17 @@ TEST_P(MergeTestIE, AccuracyTest)
TEST_P(PreprocTest, Performance)
{
using namespace InferenceEngine;
Precision prec;
std::pair<Precision, Precision> precisions;
ResizeAlgorithm interp;
Layout in_layout, out_layout;
std::pair<int, int> ocv_channels{-1, -1};
std::pair<cv::Size, cv::Size> sizes;
ColorFormat in_fmt = ColorFormat::RAW;
ColorFormat out_fmt = ColorFormat::BGR;
std::tie(prec, interp, in_fmt, in_layout, out_layout, ocv_channels, sizes) = GetParam();
std::tie(precisions, interp, in_fmt, in_layout, out_layout, ocv_channels, sizes) = GetParam();

Precision in_prec, out_prec;
std::tie(in_prec, out_prec) = precisions;
cv::Size in_size, out_size;
std::tie(in_size, out_size) = sizes;
int in_ocv_chan = -1, out_ocv_chan = -1;
Expand All @@ -1096,11 +1102,21 @@ TEST_P(PreprocTest, Performance)
double tolerance = Precision::U8 ? 1 : 0.015;
#endif

const int ocv_depth = prec == Precision::U8 ? CV_8U :
prec == Precision::FP32 ? CV_32F : -1;
const int in_ocv_type = CV_MAKETYPE(ocv_depth, in_ocv_chan);
const int out_ocv_type = CV_MAKETYPE(ocv_depth, out_ocv_chan);
initMatrixRandU(in_ocv_type, in_size, in_ocv_type, false);
auto precision_to_depth = [](Precision::ePrecision prec) -> int{
switch (prec)
{
case Precision::U8: return CV_8U;
case Precision::U16: return CV_16U;
case Precision::FP32: return CV_32F;
default:
throw std::logic_error("Unsupported configuration");
}
return -1;
};

const int in_ocv_type = CV_MAKETYPE(precision_to_depth(in_prec), in_ocv_chan);
const int out_ocv_type = CV_MAKETYPE(precision_to_depth(out_prec), out_ocv_chan);
initMatrixRandU(in_ocv_type, in_size, out_ocv_type, false);

cv::Mat out_mat(out_size, out_ocv_type);

Expand All @@ -1115,28 +1131,37 @@ TEST_P(PreprocTest, Performance)
cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255));
}

Blob::Ptr in_blob, out_blob;
switch (prec)
{
case Precision::U8:
if (in_fmt == ColorFormat::NV12) {
auto y_blob = img2Blob<Precision::U8>(in_mat1, Layout::NHWC);
auto uv_blob = img2Blob<Precision::U8>(in_mat2, Layout::NHWC);
in_blob = make_shared_blob<NV12Blob>(y_blob, uv_blob);
} else {
in_blob = img2Blob<Precision::U8>(in_mat1, in_layout);
}
out_blob = img2Blob<Precision::U8>(out_mat, out_layout);
break;

case Precision::FP32:
in_blob = img2Blob<Precision::FP32>(in_mat1, in_layout);
out_blob = img2Blob<Precision::FP32>(out_mat, out_layout);
break;
auto create_blob = [](Precision::ePrecision prec, ColorFormat fmt, Layout layout, cv::Mat& m1, cv::util::optional<cv::Mat> m2 = {}){
Blob::Ptr blob;
switch (prec)
{
case Precision::U8:
if (fmt == ColorFormat::NV12) {
auto y_blob = img2Blob<Precision::U8>(m1, Layout::NHWC);
auto uv_blob = img2Blob<Precision::U8>(m2.value(), Layout::NHWC);
blob = make_shared_blob<NV12Blob>(y_blob, uv_blob);
} else {
blob = img2Blob<Precision::U8>(m1, layout);
}
break;

default:
FAIL() << "Unsupported configuration";
}
case Precision::U16:
blob = img2Blob<Precision::U16>(m1, layout);
break;

case Precision::FP32:
blob = img2Blob<Precision::FP32>(m1, layout);
break;

default:
throw std::logic_error("Unsupported configuration");
}
return blob;
};

auto in_blob = create_blob(in_prec, in_fmt, in_layout, in_mat1, cv::util::make_optional(in_mat2));
auto out_blob = create_blob(out_prec, out_fmt, out_layout, out_mat);

PreProcessDataPtr preprocess = CreatePreprocDataHelper();
preprocess->setRoiBlob(in_blob);
Expand All @@ -1148,10 +1173,11 @@ TEST_P(PreprocTest, Performance)
// test once to warm-up cache
preprocess->execute(out_blob, info, false);

switch (prec)
switch (out_prec)
{
case Precision::U8: Blob2Img<Precision::U8> (out_blob, out_mat, out_layout); break;
case Precision::FP32: Blob2Img<Precision::FP32>(out_blob, out_mat, out_layout); break;
case Precision::U16: Blob2Img<Precision::FP32>(out_blob, out_mat, out_layout); break;
default: FAIL() << "Unsupported configuration";
}

Expand All @@ -1166,20 +1192,28 @@ TEST_P(PreprocTest, Performance)
auto cv_interp = interp == RESIZE_AREA ? cv::INTER_AREA : cv::INTER_LINEAR;
cv::resize(ocv_out_mat, ocv_out_mat, out_size, 0, 0, cv_interp);

if (in_prec != out_prec) {
cv::Mat ocv_converted;
ocv_out_mat.convertTo(ocv_converted, out_ocv_type);
ocv_out_mat = ocv_converted;
}

EXPECT_LE(cv::norm(ocv_out_mat, out_mat, cv::NORM_INF), tolerance);

#if PERF_TEST
// iterate testing, and print performance
const auto type_str = depthToString(ocv_depth);
const auto in_type_str = depthToString(precision_to_depth(in_prec));
const auto out_type_str = depthToString(precision_to_depth(out_prec));
const auto interp_str = interp == RESIZE_AREA ? "AREA"
: interp == RESIZE_BILINEAR ? "BILINEAR" : "?";
const auto in_layout_str = layoutToString(in_layout);
const auto out_layout_str = layoutToString(out_layout);

test_ms([&]() { preprocess->execute(out_blob, info, false); },
300,
"Preproc %s %s %d %s %dx%d %d %s %dx%d %s->%s",
type_str.c_str(),
"Preproc %s %s %s %d %s %dx%d %d %s %dx%d %s->%s",
in_type_str.c_str(),
out_type_str.c_str(),
interp_str,
in_ocv_chan,
in_layout_str.c_str(), in_size.width, in_size.height,
Expand Down
9 changes: 8 additions & 1 deletion inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,16 @@ struct ColorConvertYUV420TestIE:
double>> // tolerance
{};

struct PrecisionConvertTestIE: public TestParams<std::tuple<cv::Size,
int, // input matrix depth
int, // output matrix depth
double>> // tolerance
{};

//------------------------------------------------------------------------------

using PreprocParams = std::tuple< InferenceEngine::Precision // input-output data type
using PreprocParams = std::tuple< std::pair<InferenceEngine::Precision // input data type
, InferenceEngine::Precision> // output data type
, InferenceEngine::ResizeAlgorithm // resize algorithm, if needed
, InferenceEngine::ColorFormat // input color format, if needed
, InferenceEngine::Layout // input tensor layout
Expand Down
Loading

0 comments on commit f3362fa

Please sign in to comment.