From faa055f8474f777ab0890d26a3fd1b339b7f3bc3 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 10 Aug 2021 18:08:14 +0300 Subject: [PATCH 01/22] Added RandomUniform reference implementation. --- .../op_reference/random_uniform.cpp | 140 +++++++++ .../core/include/ngraph/op/random_uniform.hpp | 3 + .../runtime/reference/random_uniform.hpp | 28 ++ .../src/runtime/reference/random_uniform.cpp | 268 ++++++++++++++++++ ngraph/core/src/op/random_uniform.cpp | 61 ++++ 5 files changed, 500 insertions(+) create mode 100644 docs/template_plugin/tests/functional/op_reference/random_uniform.cpp create mode 100644 ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp create mode 100644 ngraph/core/reference/src/runtime/reference/random_uniform.cpp diff --git a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp new file mode 100644 index 00000000000000..9e9e2cf428403d --- /dev/null +++ b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp @@ -0,0 +1,140 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include + +#include + +#include "base_reference_test.hpp" +#include "ngraph/opsets/opset8.hpp" +#include "ngraph/util.hpp" + +using namespace ngraph; + +namespace reference_tests { +namespace { + +struct RandomUniformParams { + RandomUniformParams(const std::vector& paramOutShape, const Tensor& paramMinValue, const Tensor& paramMaxValue, ngraph::element::Type paramOutType, + int64_t paramGlobalSeed, int64_t paramOpSeed, const Tensor& paramExpected, const std::string& test_name) + : out_shape(paramOutShape), + min_val(paramMinValue), + max_val(paramMaxValue), + out_type(paramOutType), + global_seed(paramGlobalSeed), + op_seed(paramOpSeed), + expected(paramExpected), + test_case_name(test_name) {} + std::vector out_shape; + Tensor min_val; + Tensor max_val; + ngraph::element::Type out_type; + int64_t global_seed; + int64_t op_seed; + Tensor expected; + std::string test_case_name; +}; + +class ReferenceRandomUniformLayerTest : public testing::TestWithParam, public CommonReferenceTest { +public: + void SetUp() override { + auto params = GetParam(); + function = CreateFunction(params.out_shape, params.min_val, params.max_val, params.out_type, params.global_seed, params.op_seed); + inputData = {params.min_val.data, params.max_val.data}; + refOutData = {params.expected.data}; + } + static std::string getTestCaseName(const testing::TestParamInfo& obj) { + auto param = obj.param; + return param.test_case_name; + } + +private: + static std::shared_ptr CreateFunction(const std::vector& out_shape, const Tensor& min_val, const Tensor& max_val, + ngraph::element::Type out_type, int64_t global_seed, int64_t op_seed) { + const auto min_val_param = std::make_shared(min_val.type, min_val.shape); + const auto max_val_param = std::make_shared(max_val.type, max_val.shape); + auto out_shape_ = std::make_shared(element::i64, Shape {out_shape.size()}, out_shape); + + return std::make_shared( + NodeVector {std::make_shared(out_shape_, min_val_param, max_val_param, out_type, global_seed, op_seed)}, + ParameterVector {min_val_param, max_val_param}); + } +}; + +TEST_P(ReferenceRandomUniformLayerTest, RandomUniformWithHardcodedRefs) { + Exec(); +} + +} // namespace + +INSTANTIATE_TEST_SUITE_P( + smoke_RandomUniform_With_Hardcoded_Refs, ReferenceRandomUniformLayerTest, + ::testing::Values( + RandomUniformParams( + std::vector {3, 2, 4}, Tensor {{1}, element::f32, std::vector {0}}, Tensor {{1}, element::f32, std::vector {1}}, + element::Type_t::f32, 150, 10, + Tensor {{3, 2, 4}, element::f32, std::vector {0.7011235952377319336, 0.3053963184356689453, 0.9393105506896972656, 0.9456034898757934570, + 0.1169477701187133789, 0.5077005624771118164, 0.5197197198867797852, 0.2272746562957763672, + 0.9913740158081054688, 0.3551903963088989258, 0.8269231319427490234, 0.5986485481262207031, + 0.3136410713195800781, 0.5748131275177001953, 0.4139908552169799805, 0.9630825519561767578, + 0.3714079856872558594, 0.8525316715240478516, 0.0935858488082885742, 0.0820095539093017578, + 0.2365508079528808594, 0.8105630874633789062, 0.7422660589218139648, 0.7610669136047363281}}, + "float32_default_min_max"), + RandomUniformParams(std::vector {3, 2, 4}, Tensor {{1}, element::f16, std::vector {0}}, + Tensor {{1}, element::f16, std::vector {1}}, element::Type_t::f16, 150, 10, + Tensor {{3, 2, 4}, element::f16, std::vector {0.6044921875, 0.8066406250, 0.8320312500, 0.3837890625, 0.0361328125, + 0.0830078125, 0.5439453125, 0.8339843750, 0.3359375000, 0.7197265625, + 0.1542968750, 0.1289062500, 0.3476562500, 0.8691406250, 0.4130859375, + 0.5722656250, 0.5742187500, 0.9394531250, 0.6552734375, 0.8222656250, + 0.8242187500, 0.1328125000, 0.6435546875, 0.6601562500}}, + "float16_default_min_max"), + RandomUniformParams(std::vector {3, 2, 4}, Tensor {{1}, element::f32, std::vector {-650}}, + Tensor {{1}, element::f32, std::vector {450}}, element::Type_t::f32, 150, 10, + Tensor {{3, 2, 4}, + element::f32, + std::vector {121.2359619140625000000, -314.0640563964843750000, 383.2415771484375000000, 390.1638183593750000000, + -521.3574218750000000000, -91.5293579101562500000, -78.3082885742187500000, -399.9978637695312500000, + 440.5114746093750000000, -259.2905578613281250000, 259.6154174804687500000, 8.5134277343750000000, + -304.9948120117187500000, -17.7055664062500000000, -194.6100463867187500000, 409.3907470703125000000, + -241.4512023925781250000, 287.7848510742187500000, -547.0555419921875000000, -559.7894897460937500000, + -389.7940979003906250000, 241.6193847656250000000, 166.4926757812500000000, 187.1735839843750000000}}, + "float32_non_default_min_max"), + RandomUniformParams(std::vector {3, 2, 4}, Tensor {{1}, element::f16, std::vector {-1.5}}, + Tensor {{1}, element::f16, std::vector {-1.0}}, element::Type_t::f16, 150, 10, + Tensor {{3, 2, 4}, element::f16, std::vector {-1.1972656250, -1.0966796875, -1.0839843750, -1.3085937500, -1.4824218750, + -1.4589843750, -1.2285156250, -1.0830078125, -1.3320312500, -1.1406250000, + -1.4228515625, -1.4355468750, -1.3261718750, -1.0654296875, -1.2929687500, + -1.2138671875, -1.2128906250, -1.0302734375, -1.1718750000, -1.0888671875, + -1.0878906250, -1.4335937500, -1.1777343750, -1.1699218750}}, + "float16_non_default_min_max"), + RandomUniformParams(std::vector {2, 3, 4}, Tensor {{1}, element::i32, std::vector {-100}}, + Tensor {{1}, element::i32, std::vector {50}}, element::Type_t::i32, 100, 350, + Tensor {{2, 3, 4}, + element::i32, + std::vector { + 22, -56, -33, -89, -98, -33, -3, -48, -82, 5, -66, 21, 29, -42, -73, -37, 3, 36, -35, 20, -11, -8, -78, 47, + }}, + "int32"), + RandomUniformParams(std::vector {5, 4, 3}, Tensor {{1}, element::i64, std::vector {-2600}}, + Tensor {{1}, element::i64, std::vector {3700}}, element::Type_t::i64, 755, 951, + Tensor {{5, 4, 3}, + element::i64, + std::vector {2116, -1581, 2559, -339, -1660, 519, 90, 2027, -210, 3330, 1831, -1737, 2683, 2661, 3473, + 1220, 3534, -2384, 2199, 1935, 499, 2861, 2743, 3223, -531, -836, -65, 3435, 632, 1765, + 2613, 1891, 1698, 3069, 169, -792, -32, 2976, -1552, -2588, 3327, -1756, 2637, -1084, 3567, + -778, -1465, 2967, 1242, 2672, -1585, -2271, 3536, -1502, 400, 2241, 3126, 908, 1073, -2110}}, + "int64"), + RandomUniformParams(std::vector {7, 3}, Tensor {{1}, element::bf16, std::vector {0}}, + Tensor {{1}, element::bf16, std::vector {1}}, element::Type_t::bf16, 4978, 5164, + Tensor {{7, 3}, element::bf16, std::vector {0.8984375, 0.84375, 0.1640625, 0.1875, 0.46875, 0.6875, 0.5234375, + 0.3046875, 0.9140625, 0.453125, 0.953125, 0.328125, 0.359375, 0.1875, + 0.9453125, 0.390625, 0.21875, 0.9921875, 0.8203125, 0.453125, 0.875}}, + "bfloat16_default_min_max"), + RandomUniformParams(std::vector {7, 3}, Tensor {{1}, element::bf16, std::vector {-150}}, + Tensor {{1}, element::bf16, std::vector {200}}, element::Type_t::bf16, 4978, 5164, + Tensor {{7, 3}, element::bf16, std::vector {164, 146, -92.5, -84.5, 14, 90, 33, -43.5, 170, 8, 182, + -35, -24, -84.5, 180, -14, -73.5, 198, 138, 8, 156}}, + "bfloat16_non_default_min_max")), + ReferenceRandomUniformLayerTest::getTestCaseName); +} // namespace reference_tests diff --git a/ngraph/core/include/ngraph/op/random_uniform.hpp b/ngraph/core/include/ngraph/op/random_uniform.hpp index a9da867dae6323..a7236b14a02793 100644 --- a/ngraph/core/include/ngraph/op/random_uniform.hpp +++ b/ngraph/core/include/ngraph/op/random_uniform.hpp @@ -59,6 +59,9 @@ namespace ngraph uint64_t get_op_seed() const { return m_op_seed; } void set_op_seed(uint64_t seed2) { m_op_seed = seed2; } + bool evaluate(const HostTensorVector& outputs, + const HostTensorVector& inputs) const override; + protected: ngraph::element::Type m_output_type; uint64_t m_global_seed; diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp new file mode 100644 index 00000000000000..4898afe552e5f8 --- /dev/null +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -0,0 +1,28 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include +#include "ngraph/shape.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + void random_uniform(const uint64_t* out_shape, + const char* min_val, + const char* max_val, + char* out, + const Shape& out_shape_shape, + ngraph::element::Type elem_type, + uint64_t seed, + uint64_t seed2); + + } // namespace reference + } // namespace runtime +} // namespace ngraph \ No newline at end of file diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp new file mode 100644 index 00000000000000..47874a0e0a917f --- /dev/null +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -0,0 +1,268 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include +#include "ngraph/shape.hpp" + +#include "ngraph/runtime/reference/random_uniform.hpp" + +namespace ngraph +{ + namespace runtime + { + namespace reference + { + std::pair split_high_low(uint64_t value) + { + uint32_t low = static_cast(value); + uint32_t high = static_cast(value >> 32); + return {low, high}; + } + + uint64_t unite_high_low(uint32_t high, uint32_t low) + { + return (static_cast(high) << 32) + low; + } + + void calculate_round(uint64_t key, uint64_t& counter, uint64_t& n) + { + auto counter_lr = split_high_low(counter); + auto key_lr = split_high_low(key); + auto n_lr = split_high_low(n); + auto prod0 = split_high_low(static_cast(0xD2511F53) * n_lr.first); + auto prod1 = split_high_low(static_cast(0xCD9E8D57) * counter_lr.first); + + n_lr.first = prod1.second ^ n_lr.second ^ key_lr.first; + n_lr.second = prod1.first; + counter_lr.first = prod0.second ^ counter_lr.second ^ key_lr.second; + counter_lr.second = prod0.first; + + counter = unite_high_low(counter_lr.second, counter_lr.first); + n = unite_high_low(n_lr.second, n_lr.first); + } + + void raise_key(uint64_t& key) + { + auto key_lr = split_high_low(key); + key_lr.first += 0x9E3779B9; + key_lr.second += 0xBB67AE85; + key = unite_high_low(key_lr.second, key_lr.first); + } + + float uint32_to_float(uint32_t x) + { + uint32_t x_uint32 = (static_cast(127) << 23) | (x & 0x7fffffu); + + float x_float; + memcpy(&x_float, &x_uint32, sizeof(x_uint32)); + return x_float - 1.0f; + } + + float16 uint32_to_float16(uint32_t x) + { + uint16_t x_uint16 = static_cast(x); + x_uint16 = (static_cast(15) << 10) | (x_uint16 & 0x3ffu); + + float16 x_float16; + memcpy(&x_float16, &x_uint16, sizeof(x_uint16)); + return x_float16 - static_cast(1); + } + + double uint32_to_double(uint32_t x1, uint32_t x2) + { + uint64_t significant = + ((static_cast(x1) & 0xfffffu) << 32) | static_cast(x2); + uint64_t x_uint64 = ((static_cast(1023) << 52) | significant); + + double x_double; + memcpy(&x_double, &x_uint64, sizeof(x_uint64)); + return x_double - 1.0; + } + + uint64_t uint32_to_uint64(uint32_t x1, uint32_t x2) + { + return (static_cast(x2) << 32) | static_cast(x1); + } + + bfloat16 uint32_to_bfloat16(uint32_t x) + { + uint16_t x_uint16 = static_cast(x); + x_uint16 = (static_cast(127) << 7) | (x_uint16 & 0x7fu); + + bfloat16 x_bfloat16; + memcpy(&x_bfloat16, &x_uint16, sizeof(x_uint16)); + return x_bfloat16 - static_cast(1); + } + + void run_philox(uint64_t key, + uint64_t counter, + uint64_t n, + size_t n_rounds, + std::vector& res) + { + for (size_t i = 0; i < n_rounds; i++) + { + calculate_round(key, counter, n); + if (i < n_rounds - 1) + raise_key(key); + } + auto res1 = split_high_low(n); + auto res2 = split_high_low(counter); + res[0] = res1.first; + res[1] = res1.second; + res[2] = res2.first; + res[3] = res2.second; + } + + void random_uniform(const uint64_t* out_shape, + const char* min_val, + const char* max_val, + char* out, + const Shape& out_shape_shape, + ngraph::element::Type elem_type, + uint64_t seed, + uint64_t seed2) + { + if (seed == 0 && seed2 == 0) + { + std::srand(std::time(nullptr)); + seed = std::rand(); + } + uint64_t key = seed; + uint64_t counter = seed2; + uint64_t n = 0; + size_t shape_count = shape_size(out_shape_shape); + size_t elem_count = 1; + for (size_t i = 0; i < shape_count; i++) + { + elem_count *= out_shape[i]; + } + size_t step = elem_type.size() > 4 ? 2 : 4; + for (size_t k = 0; k < elem_count; k += step) + { + // generate 4 random uint32 values using Philox algorithm + std::vector res(4); + run_philox(key, counter, n, 10, res); + switch (elem_type) + { + case ngraph::element::Type_t::f32: + { + float res_float[4]; + std::transform(res.data(), res.data() + 4, res_float, uint32_to_float); + float mn[1]; + float mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert uint32 values to float32 and normalize to range [min_val, + // max_val) + std::transform( + res.data(), res.data() + 4, res_float, [&mn, &mx](uint32_t elem) { + return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; + }); + + memcpy(out + k * elem_type.size(), + res_float, + std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::f16: + { + float16 res_float16[4]; + // convert uint32 values to float16 and normalize to range [min_val, + // max_val) + std::transform(res.data(), res.data() + 4, res_float16, uint32_to_float16); + float16 mn[1]; + float16 mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + std::transform( + res.data(), res.data() + 4, res_float16, [&mn, &mx](uint32_t elem) { + return uint32_to_float16(elem) * (mx[0] - mn[0]) + mn[0]; + }); + memcpy(out + k * elem_type.size(), + res_float16, + std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::bf16: + { + bfloat16 res_bfloat16[4]; + bfloat16 mn[1]; + bfloat16 mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert uint32 values to bfloat16 and normalize to range [min_val, + // max_val) + std::transform( + res.data(), res.data() + 4, res_bfloat16, [&mn, &mx](uint32_t elem) { + return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; + }); + memcpy(out + k * elem_type.size(), + res_bfloat16, + std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::f64: + { + double res_double[2]; + double mn[1]; + double mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + + // convert 2 pairs of uint32 values to 2 double values and normalize to + // range [min_val, max_val) + res_double[0] = uint32_to_double(res[0], res[1]) * (mx[0] - mn[0]) + mn[0]; + res_double[1] = uint32_to_double(res[2], res[3]) * (mx[0] - mn[0]) + mn[0]; + memcpy(out + k * elem_type.size(), + res_double, + std::min((size_t)2, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::i32: + { + int res_int[4]; + int mn[1]; + int mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert uint32 values to int32 values and normalize to range [min_val, + // max_val) + std::transform( + res.data(), res.data() + 4, res_int, [&mn, &mx](uint32_t elem) { + return elem % (mx[0] - mn[0]) + mn[0]; + }); + memcpy(out + k * elem_type.size(), + res_int, + std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::i64: + { + int64_t res_int64[2]; + int64_t mn[1]; + int64_t mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert 2 pairs of uint32 values to 2 double values and normalize to + // range [min_val, max_val) + res_int64[0] = uint32_to_uint64(res[0], res[1]) % (mx[0] - mn[0]) + mn[0]; + res_int64[1] = uint32_to_uint64(res[2], res[3]) % (mx[0] - mn[0]) + mn[0]; + memcpy(out + k * elem_type.size(), + res_int64, + std::min((size_t)2, elem_count - k) * elem_type.size()); + break; + } + default: + throw ngraph_error("Unsupported type of RandomUniform: " + + elem_type.get_type_name()); + } + if (++n == 0) + ++counter; + } + } + + } // namespace reference + } // namespace runtime +} // namespace ngraph \ No newline at end of file diff --git a/ngraph/core/src/op/random_uniform.cpp b/ngraph/core/src/op/random_uniform.cpp index 6c037071d43259..c36ee48bcbeac4 100644 --- a/ngraph/core/src/op/random_uniform.cpp +++ b/ngraph/core/src/op/random_uniform.cpp @@ -5,6 +5,7 @@ #include "ngraph/op/random_uniform.hpp" #include #include "itt.hpp" +#include "ngraph/runtime/reference/random_uniform.hpp" using namespace std; using namespace ngraph; @@ -142,3 +143,63 @@ shared_ptr op::v8::RandomUniform::clone_with_new_inputs(const OutputVector return make_shared( new_args[0], new_args[1], new_args[2], m_output_type, m_global_seed, m_op_seed); } + +bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, + const HostTensorVector& inputs) const +{ + NGRAPH_OP_SCOPE(v8_Roll_evaluate); + const uint64_t* out_shape; + std::vector out_shape_uint64(shape_size(inputs[0]->get_shape())); + + if (inputs[0]->get_element_type() == element::Type_t::u64) + { + out_shape = inputs[0]->get_data_ptr(); + } + else if (inputs[0]->get_element_type() == element::Type_t::i32) + { + auto out_shape_i32 = inputs[0]->get_data_ptr(); + std::transform(out_shape_i32, + out_shape_i32 + shape_size(inputs[0]->get_shape()), + out_shape_uint64.begin(), + [](const int32_t& elem) { return static_cast(elem); }); + out_shape = out_shape_uint64.data(); + } + else if (inputs[0]->get_element_type() == element::Type_t::i64) + { + auto out_shape_i64 = inputs[0]->get_data_ptr(); + std::transform(out_shape_i64, + out_shape_i64 + shape_size(inputs[0]->get_shape()), + out_shape_uint64.begin(), + [](const int64_t& elem) { return static_cast(elem); }); + out_shape = out_shape_uint64.data(); + } + else + { + throw ngraph_error("Unsupported type of out shape in RandomUniform operation: " + + inputs[0]->get_element_type().get_type_name()); + } + + element::Type_t t_out = get_out_type(); + char* out; + switch (t_out) + { + case element::Type_t::i32: out = (char*)outputs[0]->get_data_ptr(); break; + case element::Type_t::i64: out = (char*)outputs[0]->get_data_ptr(); break; + case element::Type_t::f16: out = (char*)outputs[0]->get_data_ptr(); break; + case element::Type_t::bf16: out = (char*)outputs[0]->get_data_ptr(); break; + case element::Type_t::f32: out = (char*)outputs[0]->get_data_ptr(); break; + case element::Type_t::f64: out = (char*)outputs[0]->get_data_ptr(); break; + default: + throw ngraph_error("Unsupported type of RandomUniform: " + get_out_type().get_type_name()); + } + + runtime::reference::random_uniform(out_shape, + inputs[1]->get_data_ptr(), + inputs[2]->get_data_ptr(), + out, + inputs[0]->get_shape(), + get_out_type(), + get_global_seed(), + get_op_seed()); + return true; +} \ No newline at end of file From ecd82fc1f37f1be98dff76158d415742f65f0160 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 10 Aug 2021 18:13:37 +0300 Subject: [PATCH 02/22] Corrected comments. --- .../src/runtime/reference/random_uniform.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 47874a0e0a917f..8191a45e3a4d5e 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -154,8 +154,8 @@ namespace ngraph float mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to float32 and normalize to range [min_val, - // max_val) + // convert uint32 values to float32 and normalize to range + // [min_val, max_val) std::transform( res.data(), res.data() + 4, res_float, [&mn, &mx](uint32_t elem) { return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; @@ -169,8 +169,8 @@ namespace ngraph case ngraph::element::Type_t::f16: { float16 res_float16[4]; - // convert uint32 values to float16 and normalize to range [min_val, - // max_val) + // convert uint32 values to float16 and normalize to range + // [min_val, max_val) std::transform(res.data(), res.data() + 4, res_float16, uint32_to_float16); float16 mn[1]; float16 mx[1]; @@ -192,8 +192,8 @@ namespace ngraph bfloat16 mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to bfloat16 and normalize to range [min_val, - // max_val) + // convert uint32 values to bfloat16 and normalize to range + // [min_val, max_val) std::transform( res.data(), res.data() + 4, res_bfloat16, [&mn, &mx](uint32_t elem) { return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; @@ -227,8 +227,8 @@ namespace ngraph int mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to int32 values and normalize to range [min_val, - // max_val) + // convert uint32 values to int32 values and normalize to range + // [min_val, max_val) std::transform( res.data(), res.data() + 4, res_int, [&mn, &mx](uint32_t elem) { return elem % (mx[0] - mn[0]) + mn[0]; From 0a4844d4aa0729b170a427e7cac42bba329e41b2 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 10 Aug 2021 18:17:28 +0300 Subject: [PATCH 03/22] Small correction. --- .../include/ngraph/runtime/reference/random_uniform.hpp | 2 +- ngraph/core/reference/src/runtime/reference/random_uniform.cpp | 2 +- ngraph/core/src/op/random_uniform.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index 4898afe552e5f8..51fdd175bfccfa 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -25,4 +25,4 @@ namespace ngraph } // namespace reference } // namespace runtime -} // namespace ngraph \ No newline at end of file +} // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 8191a45e3a4d5e..32b333e38d4f0a 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -265,4 +265,4 @@ namespace ngraph } // namespace reference } // namespace runtime -} // namespace ngraph \ No newline at end of file +} // namespace ngraph diff --git a/ngraph/core/src/op/random_uniform.cpp b/ngraph/core/src/op/random_uniform.cpp index c36ee48bcbeac4..a1a958861d9c94 100644 --- a/ngraph/core/src/op/random_uniform.cpp +++ b/ngraph/core/src/op/random_uniform.cpp @@ -202,4 +202,4 @@ bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, get_global_seed(), get_op_seed()); return true; -} \ No newline at end of file +} From db0fd64803cffcf6bf630c2bc818d8edc87047b9 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Mon, 16 Aug 2021 11:56:06 +0300 Subject: [PATCH 04/22] Code style correction. --- .../op_reference/random_uniform.cpp | 204 +++++--- .../runtime/reference/random_uniform.hpp | 32 +- .../src/runtime/reference/random_uniform.cpp | 474 ++++++++---------- ngraph/core/src/op/random_uniform.cpp | 54 +- 4 files changed, 397 insertions(+), 367 deletions(-) diff --git a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp index 9e9e2cf428403d..4a459e9aef3af5 100644 --- a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp +++ b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp @@ -16,8 +16,14 @@ namespace reference_tests { namespace { struct RandomUniformParams { - RandomUniformParams(const std::vector& paramOutShape, const Tensor& paramMinValue, const Tensor& paramMaxValue, ngraph::element::Type paramOutType, - int64_t paramGlobalSeed, int64_t paramOpSeed, const Tensor& paramExpected, const std::string& test_name) + RandomUniformParams(const std::vector& paramOutShape, + const Tensor& paramMinValue, + const Tensor& paramMaxValue, + ngraph::element::Type paramOutType, + int64_t paramGlobalSeed, + int64_t paramOpSeed, + const Tensor& paramExpected, + const std::string& test_name) : out_shape(paramOutShape), min_val(paramMinValue), max_val(paramMaxValue), @@ -40,7 +46,12 @@ class ReferenceRandomUniformLayerTest : public testing::TestWithParam CreateFunction(const std::vector& out_shape, const Tensor& min_val, const Tensor& max_val, - ngraph::element::Type out_type, int64_t global_seed, int64_t op_seed) { + static std::shared_ptr CreateFunction(const std::vector& out_shape, + const Tensor& min_val, + const Tensor& max_val, + ngraph::element::Type out_type, + int64_t global_seed, + int64_t op_seed) { const auto min_val_param = std::make_shared(min_val.type, min_val.shape); const auto max_val_param = std::make_shared(max_val.type, max_val.shape); - auto out_shape_ = std::make_shared(element::i64, Shape {out_shape.size()}, out_shape); + auto out_shape_ = std::make_shared(element::i64, Shape{out_shape.size()}, out_shape); - return std::make_shared( - NodeVector {std::make_shared(out_shape_, min_val_param, max_val_param, out_type, global_seed, op_seed)}, - ParameterVector {min_val_param, max_val_param}); + return std::make_shared(NodeVector{std::make_shared(out_shape_, + min_val_param, + max_val_param, + out_type, + global_seed, + op_seed)}, + ParameterVector{min_val_param, max_val_param}); } }; @@ -69,72 +88,125 @@ TEST_P(ReferenceRandomUniformLayerTest, RandomUniformWithHardcodedRefs) { } // namespace INSTANTIATE_TEST_SUITE_P( - smoke_RandomUniform_With_Hardcoded_Refs, ReferenceRandomUniformLayerTest, + smoke_RandomUniform_With_Hardcoded_Refs, + ReferenceRandomUniformLayerTest, ::testing::Values( RandomUniformParams( - std::vector {3, 2, 4}, Tensor {{1}, element::f32, std::vector {0}}, Tensor {{1}, element::f32, std::vector {1}}, - element::Type_t::f32, 150, 10, - Tensor {{3, 2, 4}, element::f32, std::vector {0.7011235952377319336, 0.3053963184356689453, 0.9393105506896972656, 0.9456034898757934570, - 0.1169477701187133789, 0.5077005624771118164, 0.5197197198867797852, 0.2272746562957763672, - 0.9913740158081054688, 0.3551903963088989258, 0.8269231319427490234, 0.5986485481262207031, - 0.3136410713195800781, 0.5748131275177001953, 0.4139908552169799805, 0.9630825519561767578, - 0.3714079856872558594, 0.8525316715240478516, 0.0935858488082885742, 0.0820095539093017578, - 0.2365508079528808594, 0.8105630874633789062, 0.7422660589218139648, 0.7610669136047363281}}, + std::vector{3, 2, 4}, + Tensor{{1}, element::f32, std::vector{0}}, + Tensor{{1}, element::f32, std::vector{1}}, + element::Type_t::f32, + 150, + 10, + Tensor{{3, 2, 4}, + element::f32, + std::vector{ + 0.7011235952377319336, 0.3053963184356689453, 0.9393105506896972656, 0.9456034898757934570, + 0.1169477701187133789, 0.5077005624771118164, 0.5197197198867797852, 0.2272746562957763672, + 0.9913740158081054688, 0.3551903963088989258, 0.8269231319427490234, 0.5986485481262207031, + 0.3136410713195800781, 0.5748131275177001953, 0.4139908552169799805, 0.9630825519561767578, + 0.3714079856872558594, 0.8525316715240478516, 0.0935858488082885742, 0.0820095539093017578, + 0.2365508079528808594, 0.8105630874633789062, 0.7422660589218139648, 0.7610669136047363281}}, "float32_default_min_max"), - RandomUniformParams(std::vector {3, 2, 4}, Tensor {{1}, element::f16, std::vector {0}}, - Tensor {{1}, element::f16, std::vector {1}}, element::Type_t::f16, 150, 10, - Tensor {{3, 2, 4}, element::f16, std::vector {0.6044921875, 0.8066406250, 0.8320312500, 0.3837890625, 0.0361328125, - 0.0830078125, 0.5439453125, 0.8339843750, 0.3359375000, 0.7197265625, - 0.1542968750, 0.1289062500, 0.3476562500, 0.8691406250, 0.4130859375, - 0.5722656250, 0.5742187500, 0.9394531250, 0.6552734375, 0.8222656250, - 0.8242187500, 0.1328125000, 0.6435546875, 0.6601562500}}, + RandomUniformParams(std::vector{3, 2, 4}, + Tensor{{1}, element::f16, std::vector{0}}, + Tensor{{1}, element::f16, std::vector{1}}, + element::Type_t::f16, + 150, + 10, + Tensor{{3, 2, 4}, + element::f16, + std::vector{0.6044921875, 0.8066406250, 0.8320312500, 0.3837890625, + 0.0361328125, 0.0830078125, 0.5439453125, 0.8339843750, + 0.3359375000, 0.7197265625, 0.1542968750, 0.1289062500, + 0.3476562500, 0.8691406250, 0.4130859375, 0.5722656250, + 0.5742187500, 0.9394531250, 0.6552734375, 0.8222656250, + 0.8242187500, 0.1328125000, 0.6435546875, 0.6601562500}}, "float16_default_min_max"), - RandomUniformParams(std::vector {3, 2, 4}, Tensor {{1}, element::f32, std::vector {-650}}, - Tensor {{1}, element::f32, std::vector {450}}, element::Type_t::f32, 150, 10, - Tensor {{3, 2, 4}, - element::f32, - std::vector {121.2359619140625000000, -314.0640563964843750000, 383.2415771484375000000, 390.1638183593750000000, - -521.3574218750000000000, -91.5293579101562500000, -78.3082885742187500000, -399.9978637695312500000, - 440.5114746093750000000, -259.2905578613281250000, 259.6154174804687500000, 8.5134277343750000000, - -304.9948120117187500000, -17.7055664062500000000, -194.6100463867187500000, 409.3907470703125000000, - -241.4512023925781250000, 287.7848510742187500000, -547.0555419921875000000, -559.7894897460937500000, - -389.7940979003906250000, 241.6193847656250000000, 166.4926757812500000000, 187.1735839843750000000}}, + RandomUniformParams(std::vector{3, 2, 4}, + Tensor{{1}, element::f32, std::vector{-650}}, + Tensor{{1}, element::f32, std::vector{450}}, + element::Type_t::f32, + 150, + 10, + Tensor{{3, 2, 4}, + element::f32, + std::vector{ + 121.2359619140625000000, -314.0640563964843750000, 383.2415771484375000000, + 390.1638183593750000000, -521.3574218750000000000, -91.5293579101562500000, + -78.3082885742187500000, -399.9978637695312500000, 440.5114746093750000000, + -259.2905578613281250000, 259.6154174804687500000, 8.5134277343750000000, + -304.9948120117187500000, -17.7055664062500000000, -194.6100463867187500000, + 409.3907470703125000000, -241.4512023925781250000, 287.7848510742187500000, + -547.0555419921875000000, -559.7894897460937500000, -389.7940979003906250000, + 241.6193847656250000000, 166.4926757812500000000, 187.1735839843750000000}}, "float32_non_default_min_max"), - RandomUniformParams(std::vector {3, 2, 4}, Tensor {{1}, element::f16, std::vector {-1.5}}, - Tensor {{1}, element::f16, std::vector {-1.0}}, element::Type_t::f16, 150, 10, - Tensor {{3, 2, 4}, element::f16, std::vector {-1.1972656250, -1.0966796875, -1.0839843750, -1.3085937500, -1.4824218750, - -1.4589843750, -1.2285156250, -1.0830078125, -1.3320312500, -1.1406250000, - -1.4228515625, -1.4355468750, -1.3261718750, -1.0654296875, -1.2929687500, - -1.2138671875, -1.2128906250, -1.0302734375, -1.1718750000, -1.0888671875, - -1.0878906250, -1.4335937500, -1.1777343750, -1.1699218750}}, + RandomUniformParams(std::vector{3, 2, 4}, + Tensor{{1}, element::f16, std::vector{-1.5}}, + Tensor{{1}, element::f16, std::vector{-1.0}}, + element::Type_t::f16, + 150, + 10, + Tensor{{3, 2, 4}, + element::f16, + std::vector{-1.1972656250, -1.0966796875, -1.0839843750, -1.3085937500, + -1.4824218750, -1.4589843750, -1.2285156250, -1.0830078125, + -1.3320312500, -1.1406250000, -1.4228515625, -1.4355468750, + -1.3261718750, -1.0654296875, -1.2929687500, -1.2138671875, + -1.2128906250, -1.0302734375, -1.1718750000, -1.0888671875, + -1.0878906250, -1.4335937500, -1.1777343750, -1.1699218750}}, "float16_non_default_min_max"), - RandomUniformParams(std::vector {2, 3, 4}, Tensor {{1}, element::i32, std::vector {-100}}, - Tensor {{1}, element::i32, std::vector {50}}, element::Type_t::i32, 100, 350, - Tensor {{2, 3, 4}, - element::i32, - std::vector { - 22, -56, -33, -89, -98, -33, -3, -48, -82, 5, -66, 21, 29, -42, -73, -37, 3, 36, -35, 20, -11, -8, -78, 47, - }}, + RandomUniformParams(std::vector{2, 3, 4}, + Tensor{{1}, element::i32, std::vector{-100}}, + Tensor{{1}, element::i32, std::vector{50}}, + element::Type_t::i32, + 100, + 350, + Tensor{{2, 3, 4}, + element::i32, + std::vector{ + 22, -56, -33, -89, -98, -33, -3, -48, -82, 5, -66, 21, + 29, -42, -73, -37, 3, 36, -35, 20, -11, -8, -78, 47, + }}, "int32"), - RandomUniformParams(std::vector {5, 4, 3}, Tensor {{1}, element::i64, std::vector {-2600}}, - Tensor {{1}, element::i64, std::vector {3700}}, element::Type_t::i64, 755, 951, - Tensor {{5, 4, 3}, - element::i64, - std::vector {2116, -1581, 2559, -339, -1660, 519, 90, 2027, -210, 3330, 1831, -1737, 2683, 2661, 3473, - 1220, 3534, -2384, 2199, 1935, 499, 2861, 2743, 3223, -531, -836, -65, 3435, 632, 1765, - 2613, 1891, 1698, 3069, 169, -792, -32, 2976, -1552, -2588, 3327, -1756, 2637, -1084, 3567, - -778, -1465, 2967, 1242, 2672, -1585, -2271, 3536, -1502, 400, 2241, 3126, 908, 1073, -2110}}, + RandomUniformParams(std::vector{5, 4, 3}, + Tensor{{1}, element::i64, std::vector{-2600}}, + Tensor{{1}, element::i64, std::vector{3700}}, + element::Type_t::i64, + 755, + 951, + Tensor{{5, 4, 3}, + element::i64, + std::vector{ + 2116, -1581, 2559, -339, -1660, 519, 90, 2027, -210, 3330, 1831, -1737, + 2683, 2661, 3473, 1220, 3534, -2384, 2199, 1935, 499, 2861, 2743, 3223, + -531, -836, -65, 3435, 632, 1765, 2613, 1891, 1698, 3069, 169, -792, + -32, 2976, -1552, -2588, 3327, -1756, 2637, -1084, 3567, -778, -1465, 2967, + 1242, 2672, -1585, -2271, 3536, -1502, 400, 2241, 3126, 908, 1073, -2110}}, "int64"), - RandomUniformParams(std::vector {7, 3}, Tensor {{1}, element::bf16, std::vector {0}}, - Tensor {{1}, element::bf16, std::vector {1}}, element::Type_t::bf16, 4978, 5164, - Tensor {{7, 3}, element::bf16, std::vector {0.8984375, 0.84375, 0.1640625, 0.1875, 0.46875, 0.6875, 0.5234375, - 0.3046875, 0.9140625, 0.453125, 0.953125, 0.328125, 0.359375, 0.1875, - 0.9453125, 0.390625, 0.21875, 0.9921875, 0.8203125, 0.453125, 0.875}}, + RandomUniformParams(std::vector{7, 3}, + Tensor{{1}, element::bf16, std::vector{0}}, + Tensor{{1}, element::bf16, std::vector{1}}, + element::Type_t::bf16, + 4978, + 5164, + Tensor{{7, 3}, + element::bf16, + std::vector{0.8984375, 0.84375, 0.1640625, 0.1875, 0.46875, 0.6875, + 0.5234375, 0.3046875, 0.9140625, 0.453125, 0.953125, 0.328125, + 0.359375, 0.1875, 0.9453125, 0.390625, 0.21875, 0.9921875, + 0.8203125, 0.453125, 0.875}}, "bfloat16_default_min_max"), - RandomUniformParams(std::vector {7, 3}, Tensor {{1}, element::bf16, std::vector {-150}}, - Tensor {{1}, element::bf16, std::vector {200}}, element::Type_t::bf16, 4978, 5164, - Tensor {{7, 3}, element::bf16, std::vector {164, 146, -92.5, -84.5, 14, 90, 33, -43.5, 170, 8, 182, - -35, -24, -84.5, 180, -14, -73.5, 198, 138, 8, 156}}, + RandomUniformParams(std::vector{7, 3}, + Tensor{{1}, element::bf16, std::vector{-150}}, + Tensor{{1}, element::bf16, std::vector{200}}, + element::Type_t::bf16, + 4978, + 5164, + Tensor{{7, 3}, + element::bf16, + std::vector{164, 146, -92.5, -84.5, 14, 90, 33, -43.5, 170, 8, 182, + -35, -24, -84.5, 180, -14, -73.5, 198, 138, 8, 156}}, "bfloat16_non_default_min_max")), ReferenceRandomUniformLayerTest::getTestCaseName); } // namespace reference_tests diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index 51fdd175bfccfa..b85d705c70e678 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -6,23 +6,21 @@ #include #include + #include "ngraph/shape.hpp" -namespace ngraph -{ - namespace runtime - { - namespace reference - { - void random_uniform(const uint64_t* out_shape, - const char* min_val, - const char* max_val, - char* out, - const Shape& out_shape_shape, - ngraph::element::Type elem_type, - uint64_t seed, - uint64_t seed2); +namespace ngraph { +namespace runtime { +namespace reference { +void random_uniform(const uint64_t* out_shape, + const char* min_val, + const char* max_val, + char* out, + const Shape& out_shape_shape, + ngraph::element::Type elem_type, + uint64_t seed, + uint64_t seed2); - } // namespace reference - } // namespace runtime -} // namespace ngraph +} // namespace reference +} // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 32b333e38d4f0a..3121eb0d6146c5 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -2,267 +2,221 @@ // SPDX-License-Identifier: Apache-2.0 // -#include -#include "ngraph/shape.hpp" - #include "ngraph/runtime/reference/random_uniform.hpp" -namespace ngraph -{ - namespace runtime - { - namespace reference - { - std::pair split_high_low(uint64_t value) - { - uint32_t low = static_cast(value); - uint32_t high = static_cast(value >> 32); - return {low, high}; - } - - uint64_t unite_high_low(uint32_t high, uint32_t low) - { - return (static_cast(high) << 32) + low; - } - - void calculate_round(uint64_t key, uint64_t& counter, uint64_t& n) - { - auto counter_lr = split_high_low(counter); - auto key_lr = split_high_low(key); - auto n_lr = split_high_low(n); - auto prod0 = split_high_low(static_cast(0xD2511F53) * n_lr.first); - auto prod1 = split_high_low(static_cast(0xCD9E8D57) * counter_lr.first); - - n_lr.first = prod1.second ^ n_lr.second ^ key_lr.first; - n_lr.second = prod1.first; - counter_lr.first = prod0.second ^ counter_lr.second ^ key_lr.second; - counter_lr.second = prod0.first; - - counter = unite_high_low(counter_lr.second, counter_lr.first); - n = unite_high_low(n_lr.second, n_lr.first); - } - - void raise_key(uint64_t& key) - { - auto key_lr = split_high_low(key); - key_lr.first += 0x9E3779B9; - key_lr.second += 0xBB67AE85; - key = unite_high_low(key_lr.second, key_lr.first); - } - - float uint32_to_float(uint32_t x) - { - uint32_t x_uint32 = (static_cast(127) << 23) | (x & 0x7fffffu); - - float x_float; - memcpy(&x_float, &x_uint32, sizeof(x_uint32)); - return x_float - 1.0f; - } - - float16 uint32_to_float16(uint32_t x) - { - uint16_t x_uint16 = static_cast(x); - x_uint16 = (static_cast(15) << 10) | (x_uint16 & 0x3ffu); - - float16 x_float16; - memcpy(&x_float16, &x_uint16, sizeof(x_uint16)); - return x_float16 - static_cast(1); - } - - double uint32_to_double(uint32_t x1, uint32_t x2) - { - uint64_t significant = - ((static_cast(x1) & 0xfffffu) << 32) | static_cast(x2); - uint64_t x_uint64 = ((static_cast(1023) << 52) | significant); - - double x_double; - memcpy(&x_double, &x_uint64, sizeof(x_uint64)); - return x_double - 1.0; - } - - uint64_t uint32_to_uint64(uint32_t x1, uint32_t x2) - { - return (static_cast(x2) << 32) | static_cast(x1); - } - - bfloat16 uint32_to_bfloat16(uint32_t x) - { - uint16_t x_uint16 = static_cast(x); - x_uint16 = (static_cast(127) << 7) | (x_uint16 & 0x7fu); - - bfloat16 x_bfloat16; - memcpy(&x_bfloat16, &x_uint16, sizeof(x_uint16)); - return x_bfloat16 - static_cast(1); - } - - void run_philox(uint64_t key, - uint64_t counter, - uint64_t n, - size_t n_rounds, - std::vector& res) - { - for (size_t i = 0; i < n_rounds; i++) - { - calculate_round(key, counter, n); - if (i < n_rounds - 1) - raise_key(key); - } - auto res1 = split_high_low(n); - auto res2 = split_high_low(counter); - res[0] = res1.first; - res[1] = res1.second; - res[2] = res2.first; - res[3] = res2.second; - } - - void random_uniform(const uint64_t* out_shape, - const char* min_val, - const char* max_val, - char* out, - const Shape& out_shape_shape, - ngraph::element::Type elem_type, - uint64_t seed, - uint64_t seed2) - { - if (seed == 0 && seed2 == 0) - { - std::srand(std::time(nullptr)); - seed = std::rand(); - } - uint64_t key = seed; - uint64_t counter = seed2; - uint64_t n = 0; - size_t shape_count = shape_size(out_shape_shape); - size_t elem_count = 1; - for (size_t i = 0; i < shape_count; i++) - { - elem_count *= out_shape[i]; - } - size_t step = elem_type.size() > 4 ? 2 : 4; - for (size_t k = 0; k < elem_count; k += step) - { - // generate 4 random uint32 values using Philox algorithm - std::vector res(4); - run_philox(key, counter, n, 10, res); - switch (elem_type) - { - case ngraph::element::Type_t::f32: - { - float res_float[4]; - std::transform(res.data(), res.data() + 4, res_float, uint32_to_float); - float mn[1]; - float mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to float32 and normalize to range - // [min_val, max_val) - std::transform( - res.data(), res.data() + 4, res_float, [&mn, &mx](uint32_t elem) { - return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; - }); - - memcpy(out + k * elem_type.size(), - res_float, - std::min((size_t)4, elem_count - k) * elem_type.size()); - break; - } - case ngraph::element::Type_t::f16: - { - float16 res_float16[4]; - // convert uint32 values to float16 and normalize to range - // [min_val, max_val) - std::transform(res.data(), res.data() + 4, res_float16, uint32_to_float16); - float16 mn[1]; - float16 mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - std::transform( - res.data(), res.data() + 4, res_float16, [&mn, &mx](uint32_t elem) { - return uint32_to_float16(elem) * (mx[0] - mn[0]) + mn[0]; - }); - memcpy(out + k * elem_type.size(), - res_float16, - std::min((size_t)4, elem_count - k) * elem_type.size()); - break; - } - case ngraph::element::Type_t::bf16: - { - bfloat16 res_bfloat16[4]; - bfloat16 mn[1]; - bfloat16 mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to bfloat16 and normalize to range - // [min_val, max_val) - std::transform( - res.data(), res.data() + 4, res_bfloat16, [&mn, &mx](uint32_t elem) { - return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; - }); - memcpy(out + k * elem_type.size(), - res_bfloat16, - std::min((size_t)4, elem_count - k) * elem_type.size()); - break; - } - case ngraph::element::Type_t::f64: - { - double res_double[2]; - double mn[1]; - double mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); +#include - // convert 2 pairs of uint32 values to 2 double values and normalize to - // range [min_val, max_val) - res_double[0] = uint32_to_double(res[0], res[1]) * (mx[0] - mn[0]) + mn[0]; - res_double[1] = uint32_to_double(res[2], res[3]) * (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), - res_double, - std::min((size_t)2, elem_count - k) * elem_type.size()); - break; - } - case ngraph::element::Type_t::i32: - { - int res_int[4]; - int mn[1]; - int mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to int32 values and normalize to range - // [min_val, max_val) - std::transform( - res.data(), res.data() + 4, res_int, [&mn, &mx](uint32_t elem) { - return elem % (mx[0] - mn[0]) + mn[0]; - }); - memcpy(out + k * elem_type.size(), - res_int, - std::min((size_t)4, elem_count - k) * elem_type.size()); - break; - } - case ngraph::element::Type_t::i64: - { - int64_t res_int64[2]; - int64_t mn[1]; - int64_t mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert 2 pairs of uint32 values to 2 double values and normalize to - // range [min_val, max_val) - res_int64[0] = uint32_to_uint64(res[0], res[1]) % (mx[0] - mn[0]) + mn[0]; - res_int64[1] = uint32_to_uint64(res[2], res[3]) % (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), - res_int64, - std::min((size_t)2, elem_count - k) * elem_type.size()); - break; - } - default: - throw ngraph_error("Unsupported type of RandomUniform: " + - elem_type.get_type_name()); - } - if (++n == 0) - ++counter; - } - } +#include "ngraph/shape.hpp" - } // namespace reference - } // namespace runtime -} // namespace ngraph +namespace ngraph { +namespace runtime { +namespace reference { +std::pair split_high_low(uint64_t value) { + uint32_t low = static_cast(value); + uint32_t high = static_cast(value >> 32); + return {low, high}; +} + +uint64_t unite_high_low(uint32_t high, uint32_t low) { + return (static_cast(high) << 32) + low; +} + +void calculate_round(uint64_t key, uint64_t& counter, uint64_t& n) { + auto counter_lr = split_high_low(counter); + auto key_lr = split_high_low(key); + auto n_lr = split_high_low(n); + auto prod0 = split_high_low(static_cast(0xD2511F53) * n_lr.first); + auto prod1 = split_high_low(static_cast(0xCD9E8D57) * counter_lr.first); + + n_lr.first = prod1.second ^ n_lr.second ^ key_lr.first; + n_lr.second = prod1.first; + counter_lr.first = prod0.second ^ counter_lr.second ^ key_lr.second; + counter_lr.second = prod0.first; + + counter = unite_high_low(counter_lr.second, counter_lr.first); + n = unite_high_low(n_lr.second, n_lr.first); +} + +void raise_key(uint64_t& key) { + auto key_lr = split_high_low(key); + key_lr.first += 0x9E3779B9; + key_lr.second += 0xBB67AE85; + key = unite_high_low(key_lr.second, key_lr.first); +} + +float uint32_to_float(uint32_t x) { + uint32_t x_uint32 = (static_cast(127) << 23) | (x & 0x7fffffu); + + float x_float; + memcpy(&x_float, &x_uint32, sizeof(x_uint32)); + return x_float - 1.0f; +} + +float16 uint32_to_float16(uint32_t x) { + uint16_t x_uint16 = static_cast(x); + x_uint16 = (static_cast(15) << 10) | (x_uint16 & 0x3ffu); + + float16 x_float16; + memcpy(&x_float16, &x_uint16, sizeof(x_uint16)); + return x_float16 - static_cast(1); +} + +double uint32_to_double(uint32_t x1, uint32_t x2) { + uint64_t significant = ((static_cast(x1) & 0xfffffu) << 32) | static_cast(x2); + uint64_t x_uint64 = ((static_cast(1023) << 52) | significant); + + double x_double; + memcpy(&x_double, &x_uint64, sizeof(x_uint64)); + return x_double - 1.0; +} + +uint64_t uint32_to_uint64(uint32_t x1, uint32_t x2) { + return (static_cast(x2) << 32) | static_cast(x1); +} + +bfloat16 uint32_to_bfloat16(uint32_t x) { + uint16_t x_uint16 = static_cast(x); + x_uint16 = (static_cast(127) << 7) | (x_uint16 & 0x7fu); + + bfloat16 x_bfloat16; + memcpy(&x_bfloat16, &x_uint16, sizeof(x_uint16)); + return x_bfloat16 - static_cast(1); +} + +void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std::vector& res) { + for (size_t i = 0; i < n_rounds; i++) { + calculate_round(key, counter, n); + if (i < n_rounds - 1) + raise_key(key); + } + auto res1 = split_high_low(n); + auto res2 = split_high_low(counter); + res[0] = res1.first; + res[1] = res1.second; + res[2] = res2.first; + res[3] = res2.second; +} + +void random_uniform(const uint64_t* out_shape, + const char* min_val, + const char* max_val, + char* out, + const Shape& out_shape_shape, + ngraph::element::Type elem_type, + uint64_t seed, + uint64_t seed2) { + if (seed == 0 && seed2 == 0) { + std::srand(std::time(nullptr)); + seed = std::rand(); + } + uint64_t key = seed; + uint64_t counter = seed2; + uint64_t n = 0; + size_t shape_count = shape_size(out_shape_shape); + size_t elem_count = 1; + for (size_t i = 0; i < shape_count; i++) { + elem_count *= out_shape[i]; + } + size_t step = elem_type.size() > 4 ? 2 : 4; + for (size_t k = 0; k < elem_count; k += step) { + // generate 4 random uint32 values using Philox algorithm + std::vector res(4); + run_philox(key, counter, n, 10, res); + switch (elem_type) { + case ngraph::element::Type_t::f32: { + float res_float[4]; + std::transform(res.data(), res.data() + 4, res_float, uint32_to_float); + float mn[1]; + float mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert uint32 values to float32 and normalize to range + // [min_val, max_val) + std::transform(res.data(), res.data() + 4, res_float, [&mn, &mx](uint32_t elem) { + return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; + }); + + memcpy(out + k * elem_type.size(), res_float, std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::f16: { + float16 res_float16[4]; + // convert uint32 values to float16 and normalize to range + // [min_val, max_val) + std::transform(res.data(), res.data() + 4, res_float16, uint32_to_float16); + float16 mn[1]; + float16 mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + std::transform(res.data(), res.data() + 4, res_float16, [&mn, &mx](uint32_t elem) { + return uint32_to_float16(elem) * (mx[0] - mn[0]) + mn[0]; + }); + memcpy(out + k * elem_type.size(), res_float16, std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::bf16: { + bfloat16 res_bfloat16[4]; + bfloat16 mn[1]; + bfloat16 mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert uint32 values to bfloat16 and normalize to range + // [min_val, max_val) + std::transform(res.data(), res.data() + 4, res_bfloat16, [&mn, &mx](uint32_t elem) { + return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; + }); + memcpy(out + k * elem_type.size(), res_bfloat16, std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::f64: { + double res_double[2]; + double mn[1]; + double mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + + // convert 2 pairs of uint32 values to 2 double values and normalize to + // range [min_val, max_val) + res_double[0] = uint32_to_double(res[0], res[1]) * (mx[0] - mn[0]) + mn[0]; + res_double[1] = uint32_to_double(res[2], res[3]) * (mx[0] - mn[0]) + mn[0]; + memcpy(out + k * elem_type.size(), res_double, std::min((size_t)2, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::i32: { + int res_int[4]; + int mn[1]; + int mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert uint32 values to int32 values and normalize to range + // [min_val, max_val) + std::transform(res.data(), res.data() + 4, res_int, [&mn, &mx](uint32_t elem) { + return elem % (mx[0] - mn[0]) + mn[0]; + }); + memcpy(out + k * elem_type.size(), res_int, std::min((size_t)4, elem_count - k) * elem_type.size()); + break; + } + case ngraph::element::Type_t::i64: { + int64_t res_int64[2]; + int64_t mn[1]; + int64_t mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + // convert 2 pairs of uint32 values to 2 double values and normalize to + // range [min_val, max_val) + res_int64[0] = uint32_to_uint64(res[0], res[1]) % (mx[0] - mn[0]) + mn[0]; + res_int64[1] = uint32_to_uint64(res[2], res[3]) % (mx[0] - mn[0]) + mn[0]; + memcpy(out + k * elem_type.size(), res_int64, std::min((size_t)2, elem_count - k) * elem_type.size()); + break; + } + default: + throw ngraph_error("Unsupported type of RandomUniform: " + elem_type.get_type_name()); + } + if (++n == 0) + ++counter; + } +} + +} // namespace reference +} // namespace runtime +} // namespace ngraph diff --git a/ngraph/core/src/op/random_uniform.cpp b/ngraph/core/src/op/random_uniform.cpp index c65f58318dcfbd..b6a6e4b226c27d 100644 --- a/ngraph/core/src/op/random_uniform.cpp +++ b/ngraph/core/src/op/random_uniform.cpp @@ -127,51 +127,57 @@ shared_ptr op::v8::RandomUniform::clone_with_new_inputs(const OutputVector m_op_seed); } -bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, - const HostTensorVector& inputs) const -{ +bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { NGRAPH_OP_SCOPE(v8_Roll_evaluate); const uint64_t* out_shape; std::vector out_shape_uint64(shape_size(inputs[0]->get_shape())); - if (inputs[0]->get_element_type() == element::Type_t::u64) - { + if (inputs[0]->get_element_type() == element::Type_t::u64) { out_shape = inputs[0]->get_data_ptr(); - } - else if (inputs[0]->get_element_type() == element::Type_t::i32) - { + } else if (inputs[0]->get_element_type() == element::Type_t::i32) { auto out_shape_i32 = inputs[0]->get_data_ptr(); std::transform(out_shape_i32, out_shape_i32 + shape_size(inputs[0]->get_shape()), out_shape_uint64.begin(), - [](const int32_t& elem) { return static_cast(elem); }); + [](const int32_t& elem) { + return static_cast(elem); + }); out_shape = out_shape_uint64.data(); - } - else if (inputs[0]->get_element_type() == element::Type_t::i64) - { + } else if (inputs[0]->get_element_type() == element::Type_t::i64) { auto out_shape_i64 = inputs[0]->get_data_ptr(); std::transform(out_shape_i64, out_shape_i64 + shape_size(inputs[0]->get_shape()), out_shape_uint64.begin(), - [](const int64_t& elem) { return static_cast(elem); }); + [](const int64_t& elem) { + return static_cast(elem); + }); out_shape = out_shape_uint64.data(); - } - else - { + } else { throw ngraph_error("Unsupported type of out shape in RandomUniform operation: " + inputs[0]->get_element_type().get_type_name()); } element::Type_t t_out = get_out_type(); char* out; - switch (t_out) - { - case element::Type_t::i32: out = (char*)outputs[0]->get_data_ptr(); break; - case element::Type_t::i64: out = (char*)outputs[0]->get_data_ptr(); break; - case element::Type_t::f16: out = (char*)outputs[0]->get_data_ptr(); break; - case element::Type_t::bf16: out = (char*)outputs[0]->get_data_ptr(); break; - case element::Type_t::f32: out = (char*)outputs[0]->get_data_ptr(); break; - case element::Type_t::f64: out = (char*)outputs[0]->get_data_ptr(); break; + switch (t_out) { + case element::Type_t::i32: + out = (char*)outputs[0]->get_data_ptr(); + break; + case element::Type_t::i64: + out = (char*)outputs[0]->get_data_ptr(); + break; + case element::Type_t::f16: + out = (char*)outputs[0]->get_data_ptr(); + break; + case element::Type_t::bf16: + out = (char*)outputs[0]->get_data_ptr(); + break; + case element::Type_t::f32: + out = (char*)outputs[0]->get_data_ptr(); + break; + case element::Type_t::f64: + out = (char*)outputs[0]->get_data_ptr(); + break; default: throw ngraph_error("Unsupported type of RandomUniform: " + get_out_type().get_type_name()); } From 1ac3c165226232893e88fbddb9021d95989d526e Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 17 Aug 2021 10:42:17 +0300 Subject: [PATCH 05/22] Added has_evaluate() method. --- .../core/include/ngraph/op/random_uniform.hpp | 2 ++ ngraph/core/src/op/random_uniform.cpp | 24 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ngraph/core/include/ngraph/op/random_uniform.hpp b/ngraph/core/include/ngraph/op/random_uniform.hpp index cf66cc5015bdb6..8586fe75796b92 100644 --- a/ngraph/core/include/ngraph/op/random_uniform.hpp +++ b/ngraph/core/include/ngraph/op/random_uniform.hpp @@ -65,6 +65,8 @@ class NGRAPH_API RandomUniform : public Op { bool evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const override; + bool has_evaluate() const override; + protected: ngraph::element::Type m_output_type; uint64_t m_global_seed; diff --git a/ngraph/core/src/op/random_uniform.cpp b/ngraph/core/src/op/random_uniform.cpp index b6a6e4b226c27d..67c7e9e37e3e33 100644 --- a/ngraph/core/src/op/random_uniform.cpp +++ b/ngraph/core/src/op/random_uniform.cpp @@ -117,7 +117,7 @@ bool op::v8::RandomUniform::visit_attributes(AttributeVisitor& visitor) { } shared_ptr op::v8::RandomUniform::clone_with_new_inputs(const OutputVector& new_args) const { - NGRAPH_OP_SCOPE(v8_Roll_clone_with_new_inputs); + NGRAPH_OP_SCOPE(v8_RandomUniform_clone_with_new_inputs); check_new_args_count(this, new_args); return make_shared(new_args[0], new_args[1], @@ -128,7 +128,7 @@ shared_ptr op::v8::RandomUniform::clone_with_new_inputs(const OutputVector } bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, const HostTensorVector& inputs) const { - NGRAPH_OP_SCOPE(v8_Roll_evaluate); + NGRAPH_OP_SCOPE(v8_RandomUniform_evaluate); const uint64_t* out_shape; std::vector out_shape_uint64(shape_size(inputs[0]->get_shape())); @@ -192,3 +192,23 @@ bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, const Host get_op_seed()); return true; } + +bool op::v8::RandomUniform::has_evaluate() const { + NGRAPH_OP_SCOPE(v8_RandomUniform_has_evaluate); + if (get_input_element_type(0) != ngraph::element::i32 && get_input_element_type(0) != ngraph::element::i64) { + return false; + } + + switch (get_out_type()) { + case ngraph::element::i32: + case ngraph::element::i64: + case ngraph::element::f16: + case ngraph::element::bf16: + case ngraph::element::f32: + case ngraph::element::f64: + return true; + default: + break; + } + return false; +} From 6ac585f88c2f893abf5c3b2faf75ba1c5f4632db Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 17 Aug 2021 19:30:51 +0300 Subject: [PATCH 06/22] Added comments, added names to consts. --- .../runtime/reference/random_uniform.hpp | 5 + .../src/runtime/reference/random_uniform.cpp | 105 ++++++++++++++---- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index b85d705c70e678..825d770fdba623 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -21,6 +21,11 @@ void random_uniform(const uint64_t* out_shape, uint64_t seed, uint64_t seed2); +const uint32_t crush_resistance_const_lower_value = 0x9E3779B9; +const uint32_t crush_resistance_const_upper_value = 0xBB67AE85; +const uint64_t statistic_maximizing_multiplier_n = 0xD2511F53; +const uint64_t statistic_maximizing_multiplier_counter = 0xCD9E8D57; + } // namespace reference } // namespace runtime } // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 3121eb0d6146c5..fda93c420882d0 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -11,40 +11,62 @@ namespace ngraph { namespace runtime { namespace reference { + +// Splits uint64 value into two uint32 values with right and left part of original value. std::pair split_high_low(uint64_t value) { uint32_t low = static_cast(value); uint32_t high = static_cast(value >> 32); return {low, high}; } +// Concatenates two uint32 values into single uint64 values. uint64_t unite_high_low(uint32_t high, uint32_t low) { return (static_cast(high) << 32) + low; } +// Runs single "round" of Philox algorithm. void calculate_round(uint64_t key, uint64_t& counter, uint64_t& n) { + // Split key, counter and n into two uint32 values. auto counter_lr = split_high_low(counter); auto key_lr = split_high_low(key); auto n_lr = split_high_low(n); - auto prod0 = split_high_low(static_cast(0xD2511F53) * n_lr.first); - auto prod1 = split_high_low(static_cast(0xCD9E8D57) * counter_lr.first); + // Each round performs following updating for n and counter: + // left uint32 part = mullo(R, M) + // right uint32 part = mulhi(R, M) xor k xor L + // mulhi(a, b) = floor((a * b) / 2^32) + // mullo(a, b) = (a * b) mod 2^32, + // where M - statistic_maximizing_multiplier const + auto prod0 = split_high_low(statistic_maximizing_multiplier_n * n_lr.first); + auto prod1 = split_high_low(statistic_maximizing_multiplier_counter * counter_lr.first); n_lr.first = prod1.second ^ n_lr.second ^ key_lr.first; n_lr.second = prod1.first; counter_lr.first = prod0.second ^ counter_lr.second ^ key_lr.second; counter_lr.second = prod0.first; + // Unite counter and n into uint64 values. counter = unite_high_low(counter_lr.second, counter_lr.first); n = unite_high_low(n_lr.second, n_lr.first); } +// Increases key value. void raise_key(uint64_t& key) { auto key_lr = split_high_low(key); - key_lr.first += 0x9E3779B9; - key_lr.second += 0xBB67AE85; + key_lr.first += crush_resistance_const_lower_value; + key_lr.second += crush_resistance_const_upper_value; key = unite_high_low(key_lr.second, key_lr.first); } +// Helper function for converting uint32 values to float32. float uint32_to_float(uint32_t x) { + // float32 is formatted as follows: sign(1 bit) exponent(8 bits) mantissa(23 bits). The value is interpreted + // The value is interpreted using following formula: + // (-1)^sign * 1, mantissa * 2 ^ (exponent - 127) + // Here we set the following values: + // sign = 0 + // exponent = 127, for obtaining a zero exponent. + // mantissa = 23 right bits from generated uint32 random value. + uint32_t x_uint32 = (static_cast(127) << 23) | (x & 0x7fffffu); float x_float; @@ -52,7 +74,16 @@ float uint32_to_float(uint32_t x) { return x_float - 1.0f; } +// Helper function for converting uint32 values to float16. float16 uint32_to_float16(uint32_t x) { + // float16 is formatted as follows: sign(1 bit) exponent(5 bits) mantissa(10 bits). The value is interpreted + // The value is interpreted using following formula: + // (-1)^sign * 1, mantissa * 2 ^ (exponent - 15) + // Here we set the following values: + // sign = 0 + // exponent = 15, for obtaining a zero exponent. + // mantissa = 10 right bits from generated uint32 random value. + uint16_t x_uint16 = static_cast(x); x_uint16 = (static_cast(15) << 10) | (x_uint16 & 0x3ffu); @@ -61,7 +92,16 @@ float16 uint32_to_float16(uint32_t x) { return x_float16 - static_cast(1); } +// Helper function for converting uint32 values to double. double uint32_to_double(uint32_t x1, uint32_t x2) { + // float64 is formatted as follows: sign(1 bit) exponent(11 bits) mantissa(52 bits). The value is interpreted + // The value is interpreted using following formula: + // (-1)^sign * 1, mantissa * 2 ^ (exponent - 1023) + // Here we set the following values: + // sign = 0 + // exponent = 1023, for obtaining a zero exponent. + // mantissa = 52 right bits from two concatenated uint32 values from random integer generator. + uint64_t significant = ((static_cast(x1) & 0xfffffu) << 32) | static_cast(x2); uint64_t x_uint64 = ((static_cast(1023) << 52) | significant); @@ -70,11 +110,22 @@ double uint32_to_double(uint32_t x1, uint32_t x2) { return x_double - 1.0; } +// Helper function for converting uint32 values to uint64. uint64_t uint32_to_uint64(uint32_t x1, uint32_t x2) { + // Concatenates x1 and x2 and casts to uint64_t. return (static_cast(x2) << 32) | static_cast(x1); } +// Helper function for converting uint32 values to bfloat16. bfloat16 uint32_to_bfloat16(uint32_t x) { + // bfloat16 is formatted as follows: sign(1 bit) exponent(8 bits) mantissa(7 bits). The value is interpreted + // The value is interpreted using following formula: + // (-1)^sign * 1, mantissa * 2 ^ (exponent - 127) + // Here we set the following values: + // sign = 0 + // exponent = 127, for obtaining a zero exponent. + // mantissa = 7 right bits from generated uint32 random value. + uint16_t x_uint16 = static_cast(x); x_uint16 = (static_cast(127) << 7) | (x_uint16 & 0x7fu); @@ -83,6 +134,7 @@ bfloat16 uint32_to_bfloat16(uint32_t x) { return x_bfloat16 - static_cast(1); } +// Runs Philox algorithm. void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std::vector& res) { for (size_t i = 0; i < n_rounds; i++) { calculate_round(key, counter, n); @@ -97,6 +149,7 @@ void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std res[3] = res2.second; } +// Implementation of RandomUniform that uses Philox algorithm as inner random unsigned integer generator. void random_uniform(const uint64_t* out_shape, const char* min_val, const char* max_val, @@ -117,59 +170,67 @@ void random_uniform(const uint64_t* out_shape, for (size_t i = 0; i < shape_count; i++) { elem_count *= out_shape[i]; } + // Each run of Philox algorithm generates 4 uint32 values. + // If output_type is int32, f32, bf16, or f16 each value is converted to + // corresponding type so we have 4 result values. For f64 and i64 we use + // a pair of values for conversion, so we have 2 result values. + // Step indicates how many values we generate in one iteration. size_t step = elem_type.size() > 4 ? 2 : 4; + for (size_t k = 0; k < elem_count; k += step) { // generate 4 random uint32 values using Philox algorithm std::vector res(4); run_philox(key, counter, n, 10, res); + + // convert values to corresponding output_type switch (elem_type) { case ngraph::element::Type_t::f32: { - float res_float[4]; - std::transform(res.data(), res.data() + 4, res_float, uint32_to_float); + float res_float[step]; + std::transform(res.data(), res.data() + step, res_float, uint32_to_float); float mn[1]; float mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); // convert uint32 values to float32 and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + 4, res_float, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_float, [&mn, &mx](uint32_t elem) { return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_float, std::min((size_t)4, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_float, std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::f16: { - float16 res_float16[4]; + float16 res_float16[step]; // convert uint32 values to float16 and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + 4, res_float16, uint32_to_float16); + std::transform(res.data(), res.data() + step, res_float16, uint32_to_float16); float16 mn[1]; float16 mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); - std::transform(res.data(), res.data() + 4, res_float16, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_float16, [&mn, &mx](uint32_t elem) { return uint32_to_float16(elem) * (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_float16, std::min((size_t)4, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_float16, std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::bf16: { - bfloat16 res_bfloat16[4]; + bfloat16 res_bfloat16[step]; bfloat16 mn[1]; bfloat16 mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); // convert uint32 values to bfloat16 and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + 4, res_bfloat16, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_bfloat16, [&mn, &mx](uint32_t elem) { return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_bfloat16, std::min((size_t)4, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_bfloat16, std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::f64: { - double res_double[2]; + double res_double[step]; double mn[1]; double mx[1]; memcpy(mn, min_val, elem_type.size()); @@ -179,25 +240,25 @@ void random_uniform(const uint64_t* out_shape, // range [min_val, max_val) res_double[0] = uint32_to_double(res[0], res[1]) * (mx[0] - mn[0]) + mn[0]; res_double[1] = uint32_to_double(res[2], res[3]) * (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), res_double, std::min((size_t)2, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_double, std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::i32: { - int res_int[4]; + int res_int[step]; int mn[1]; int mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); // convert uint32 values to int32 values and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + 4, res_int, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_int, [&mn, &mx](uint32_t elem) { return elem % (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_int, std::min((size_t)4, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_int, std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::i64: { - int64_t res_int64[2]; + int64_t res_int64[step]; int64_t mn[1]; int64_t mx[1]; memcpy(mn, min_val, elem_type.size()); @@ -206,7 +267,7 @@ void random_uniform(const uint64_t* out_shape, // range [min_val, max_val) res_int64[0] = uint32_to_uint64(res[0], res[1]) % (mx[0] - mn[0]) + mn[0]; res_int64[1] = uint32_to_uint64(res[2], res[3]) % (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), res_int64, std::min((size_t)2, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_int64, std::min(step, elem_count - k) * elem_type.size()); break; } default: From 49401d8bdb8e8d9101b51eb6ad1da558d5239847 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 17 Aug 2021 19:45:48 +0300 Subject: [PATCH 07/22] Small fix. --- ngraph/core/reference/src/runtime/reference/random_uniform.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index fda93c420882d0..e3c940b39aa422 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -175,7 +175,7 @@ void random_uniform(const uint64_t* out_shape, // corresponding type so we have 4 result values. For f64 and i64 we use // a pair of values for conversion, so we have 2 result values. // Step indicates how many values we generate in one iteration. - size_t step = elem_type.size() > 4 ? 2 : 4; + const size_t step = elem_type.size() > 4 ? 2 : 4; for (size_t k = 0; k < elem_count; k += step) { // generate 4 random uint32 values using Philox algorithm From 13866bc4274a49b0a7cfb29dd5977b1512d8dbb6 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Wed, 18 Aug 2021 10:44:11 +0300 Subject: [PATCH 08/22] Replaced arrays with vectors. --- .../runtime/reference/random_uniform.hpp | 1 + .../src/runtime/reference/random_uniform.cpp | 40 +++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index 825d770fdba623..0b3e14172a1e4e 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -25,6 +25,7 @@ const uint32_t crush_resistance_const_lower_value = 0x9E3779B9; const uint32_t crush_resistance_const_upper_value = 0xBB67AE85; const uint64_t statistic_maximizing_multiplier_n = 0xD2511F53; const uint64_t statistic_maximizing_multiplier_counter = 0xCD9E8D57; +const size_t rounds_number = 10; } // namespace reference } // namespace runtime diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index e3c940b39aa422..27abc9281eaeaf 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -179,58 +179,58 @@ void random_uniform(const uint64_t* out_shape, for (size_t k = 0; k < elem_count; k += step) { // generate 4 random uint32 values using Philox algorithm - std::vector res(4); - run_philox(key, counter, n, 10, res); + std::vector res(step); + run_philox(key, counter, n, rounds_number, res); // convert values to corresponding output_type switch (elem_type) { case ngraph::element::Type_t::f32: { - float res_float[step]; - std::transform(res.data(), res.data() + step, res_float, uint32_to_float); + std::vector res_float(step); + std::transform(res.data(), res.data() + step, res_float.data(), uint32_to_float); float mn[1]; float mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); // convert uint32 values to float32 and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_float, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_float.data(), [&mn, &mx](uint32_t elem) { return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_float, std::min(step, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_float.data(), std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::f16: { - float16 res_float16[step]; + std::vector res_float16(step); // convert uint32 values to float16 and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_float16, uint32_to_float16); + std::transform(res.data(), res.data() + step, res_float16.data(), uint32_to_float16); float16 mn[1]; float16 mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); - std::transform(res.data(), res.data() + step, res_float16, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_float16.data(), [&mn, &mx](uint32_t elem) { return uint32_to_float16(elem) * (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_float16, std::min(step, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_float16.data(), std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::bf16: { - bfloat16 res_bfloat16[step]; + std::vector res_bfloat16(step); bfloat16 mn[1]; bfloat16 mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); // convert uint32 values to bfloat16 and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_bfloat16, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_bfloat16.data(), [&mn, &mx](uint32_t elem) { return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_bfloat16, std::min(step, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_bfloat16.data(), std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::f64: { - double res_double[step]; + std::vector res_double(step); double mn[1]; double mx[1]; memcpy(mn, min_val, elem_type.size()); @@ -240,25 +240,25 @@ void random_uniform(const uint64_t* out_shape, // range [min_val, max_val) res_double[0] = uint32_to_double(res[0], res[1]) * (mx[0] - mn[0]) + mn[0]; res_double[1] = uint32_to_double(res[2], res[3]) * (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), res_double, std::min(step, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_double.data(), std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::i32: { - int res_int[step]; + std::vector res_int(step); int mn[1]; int mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); // convert uint32 values to int32 values and normalize to range // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_int, [&mn, &mx](uint32_t elem) { + std::transform(res.data(), res.data() + step, res_int.data(), [&mn, &mx](uint32_t elem) { return elem % (mx[0] - mn[0]) + mn[0]; }); - memcpy(out + k * elem_type.size(), res_int, std::min(step, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_int.data(), std::min(step, elem_count - k) * elem_type.size()); break; } case ngraph::element::Type_t::i64: { - int64_t res_int64[step]; + std::vector res_int64(step); int64_t mn[1]; int64_t mx[1]; memcpy(mn, min_val, elem_type.size()); @@ -267,7 +267,7 @@ void random_uniform(const uint64_t* out_shape, // range [min_val, max_val) res_int64[0] = uint32_to_uint64(res[0], res[1]) % (mx[0] - mn[0]) + mn[0]; res_int64[1] = uint32_to_uint64(res[2], res[3]) % (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), res_int64, std::min(step, elem_count - k) * elem_type.size()); + memcpy(out + k * elem_type.size(), res_int64.data(), std::min(step, elem_count - k) * elem_type.size()); break; } default: From c370a7c7cf91ebdeb8f843c80a6e04ac4bab201e Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Tue, 24 Aug 2021 14:01:42 +0300 Subject: [PATCH 09/22] Apply suggestions from code review Co-authored-by: Ilya Churaev --- .../tests/functional/op_reference/random_uniform.cpp | 2 +- .../include/ngraph/runtime/reference/random_uniform.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp index 4a459e9aef3af5..c288d03e24f406 100644 --- a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp +++ b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp @@ -64,7 +64,7 @@ class ReferenceRandomUniformLayerTest : public testing::TestWithParam CreateFunction(const std::vector& out_shape, const Tensor& min_val, const Tensor& max_val, - ngraph::element::Type out_type, + const ngraph::element::Type& out_type, int64_t global_seed, int64_t op_seed) { const auto min_val_param = std::make_shared(min_val.type, min_val.shape); diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index 0b3e14172a1e4e..fb6b343de1fd75 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -17,7 +17,7 @@ void random_uniform(const uint64_t* out_shape, const char* max_val, char* out, const Shape& out_shape_shape, - ngraph::element::Type elem_type, + const ngraph::element::Type& elem_type, uint64_t seed, uint64_t seed2); From bd37f4e8c7a30766c7d37de76d843126d6c0c1a5 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Thu, 26 Aug 2021 18:11:04 +0300 Subject: [PATCH 10/22] Code refactoring. --- .../src/runtime/reference/random_uniform.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 27abc9281eaeaf..18afefd02adfb4 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -110,11 +110,6 @@ double uint32_to_double(uint32_t x1, uint32_t x2) { return x_double - 1.0; } -// Helper function for converting uint32 values to uint64. -uint64_t uint32_to_uint64(uint32_t x1, uint32_t x2) { - // Concatenates x1 and x2 and casts to uint64_t. - return (static_cast(x2) << 32) | static_cast(x1); -} // Helper function for converting uint32 values to bfloat16. bfloat16 uint32_to_bfloat16(uint32_t x) { @@ -155,7 +150,7 @@ void random_uniform(const uint64_t* out_shape, const char* max_val, char* out, const Shape& out_shape_shape, - ngraph::element::Type elem_type, + const ngraph::element::Type& elem_type, uint64_t seed, uint64_t seed2) { if (seed == 0 && seed2 == 0) { @@ -265,8 +260,11 @@ void random_uniform(const uint64_t* out_shape, memcpy(mx, max_val, elem_type.size()); // convert 2 pairs of uint32 values to 2 double values and normalize to // range [min_val, max_val) - res_int64[0] = uint32_to_uint64(res[0], res[1]) % (mx[0] - mn[0]) + mn[0]; - res_int64[1] = uint32_to_uint64(res[2], res[3]) % (mx[0] - mn[0]) + mn[0]; + auto v1 = static_cast(unite_high_low(res[1], res[0]) % (mx[0] - mn[0])); + auto v2 = static_cast(unite_high_low(res[3], res[2]) % (mx[0] - mn[0])); + + res_int64[0] = v1 + mn[0]; + res_int64[1] = v2 + mn[0]; memcpy(out + k * elem_type.size(), res_int64.data(), std::min(step, elem_count - k) * elem_type.size()); break; } From 9a85bebbb30518aee074f1474d379d6934e48b19 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Thu, 26 Aug 2021 18:26:38 +0300 Subject: [PATCH 11/22] Corrected tests, code style. --- .../op_reference/random_uniform.cpp | 68 ++++++++----------- .../src/runtime/reference/random_uniform.cpp | 1 - 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp index c288d03e24f406..1f7784b8bbce75 100644 --- a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp +++ b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp @@ -91,23 +91,20 @@ INSTANTIATE_TEST_SUITE_P( smoke_RandomUniform_With_Hardcoded_Refs, ReferenceRandomUniformLayerTest, ::testing::Values( - RandomUniformParams( - std::vector{3, 2, 4}, - Tensor{{1}, element::f32, std::vector{0}}, - Tensor{{1}, element::f32, std::vector{1}}, - element::Type_t::f32, - 150, - 10, - Tensor{{3, 2, 4}, - element::f32, - std::vector{ - 0.7011235952377319336, 0.3053963184356689453, 0.9393105506896972656, 0.9456034898757934570, - 0.1169477701187133789, 0.5077005624771118164, 0.5197197198867797852, 0.2272746562957763672, - 0.9913740158081054688, 0.3551903963088989258, 0.8269231319427490234, 0.5986485481262207031, - 0.3136410713195800781, 0.5748131275177001953, 0.4139908552169799805, 0.9630825519561767578, - 0.3714079856872558594, 0.8525316715240478516, 0.0935858488082885742, 0.0820095539093017578, - 0.2365508079528808594, 0.8105630874633789062, 0.7422660589218139648, 0.7610669136047363281}}, - "float32_default_min_max"), + RandomUniformParams(std::vector{3, 2, 4}, + Tensor{{1}, element::f32, std::vector{0}}, + Tensor{{1}, element::f32, std::vector{1}}, + element::Type_t::f32, + 150, + 10, + Tensor{{3, 2, 4}, + element::f32, + std::vector{0.70112360, 0.30539632, 0.93931055, 0.94560349, 0.11694777, + 0.50770056, 0.51971972, 0.22727466, 0.99137402, 0.35519040, + 0.82692313, 0.59864855, 0.31364107, 0.57481313, 0.41399086, + 0.96308255, 0.37140799, 0.85253167, 0.09358585, 0.08200955, + 0.23655081, 0.81056309, 0.74226606, 0.76106691}}, + "float32_default_min_max"), RandomUniformParams(std::vector{3, 2, 4}, Tensor{{1}, element::f16, std::vector{0}}, Tensor{{1}, element::f16, std::vector{1}}, @@ -116,12 +113,11 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f16, - std::vector{0.6044921875, 0.8066406250, 0.8320312500, 0.3837890625, - 0.0361328125, 0.0830078125, 0.5439453125, 0.8339843750, - 0.3359375000, 0.7197265625, 0.1542968750, 0.1289062500, - 0.3476562500, 0.8691406250, 0.4130859375, 0.5722656250, - 0.5742187500, 0.9394531250, 0.6552734375, 0.8222656250, - 0.8242187500, 0.1328125000, 0.6435546875, 0.6601562500}}, + std::vector{0.60449219, 0.80664062, 0.83203125, 0.38378906, 0.03613281, + 0.08300781, 0.54394531, 0.83398438, 0.33593750, 0.71972656, + 0.15429688, 0.12890625, 0.34765625, 0.86914062, 0.41308594, + 0.57226562, 0.57421875, 0.93945312, 0.65527344, 0.82226562, + 0.82421875, 0.13281250, 0.64355469, 0.66015625}}, "float16_default_min_max"), RandomUniformParams(std::vector{3, 2, 4}, Tensor{{1}, element::f32, std::vector{-650}}, @@ -131,15 +127,12 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f32, - std::vector{ - 121.2359619140625000000, -314.0640563964843750000, 383.2415771484375000000, - 390.1638183593750000000, -521.3574218750000000000, -91.5293579101562500000, - -78.3082885742187500000, -399.9978637695312500000, 440.5114746093750000000, - -259.2905578613281250000, 259.6154174804687500000, 8.5134277343750000000, - -304.9948120117187500000, -17.7055664062500000000, -194.6100463867187500000, - 409.3907470703125000000, -241.4512023925781250000, 287.7848510742187500000, - -547.0555419921875000000, -559.7894897460937500000, -389.7940979003906250000, - 241.6193847656250000000, 166.4926757812500000000, 187.1735839843750000000}}, + std::vector{121.23596191, -314.06405640, 383.24157715, 390.16381836, + -521.35742188, -91.52935791, -78.30828857, -399.99786377, + 440.51147461, -259.29055786, 259.61541748, 8.51342773, + -304.99481201, -17.70556641, -194.61004639, 409.39074707, + -241.45120239, 287.78485107, -547.05554199, -559.78948975, + -389.79409790, 241.61938477, 166.49267578, 187.17358398}}, "float32_non_default_min_max"), RandomUniformParams(std::vector{3, 2, 4}, Tensor{{1}, element::f16, std::vector{-1.5}}, @@ -149,12 +142,11 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f16, - std::vector{-1.1972656250, -1.0966796875, -1.0839843750, -1.3085937500, - -1.4824218750, -1.4589843750, -1.2285156250, -1.0830078125, - -1.3320312500, -1.1406250000, -1.4228515625, -1.4355468750, - -1.3261718750, -1.0654296875, -1.2929687500, -1.2138671875, - -1.2128906250, -1.0302734375, -1.1718750000, -1.0888671875, - -1.0878906250, -1.4335937500, -1.1777343750, -1.1699218750}}, + std::vector{-1.19726562, -1.09667969, -1.08398438, -1.30859375, -1.48242188, + -1.45898438, -1.22851562, -1.08300781, -1.33203125, -1.14062500, + -1.42285156, -1.43554688, -1.32617188, -1.06542969, -1.29296875, + -1.21386719, -1.21289062, -1.03027344, -1.17187500, -1.08886719, + -1.08789062, -1.43359375, -1.17773438, -1.16992188}}, "float16_non_default_min_max"), RandomUniformParams(std::vector{2, 3, 4}, Tensor{{1}, element::i32, std::vector{-100}}, diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 18afefd02adfb4..bb2ac6912ce0a2 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -110,7 +110,6 @@ double uint32_to_double(uint32_t x1, uint32_t x2) { return x_double - 1.0; } - // Helper function for converting uint32 values to bfloat16. bfloat16 uint32_to_bfloat16(uint32_t x) { // bfloat16 is formatted as follows: sign(1 bit) exponent(8 bits) mantissa(7 bits). The value is interpreted From 13a000cc3339ecaa02e2f9ab66e6e9229ed93f1d Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Thu, 26 Aug 2021 18:36:12 +0300 Subject: [PATCH 12/22] Added comment. --- .../include/ngraph/runtime/reference/random_uniform.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index fb6b343de1fd75..aca4bc8e9470de 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -21,6 +21,8 @@ void random_uniform(const uint64_t* out_shape, uint64_t seed, uint64_t seed2); +// Following const values are taken from the original paper: +// https://www.thesalmons.org/john/random123/papers/random123sc11.pdf const uint32_t crush_resistance_const_lower_value = 0x9E3779B9; const uint32_t crush_resistance_const_upper_value = 0xBB67AE85; const uint64_t statistic_maximizing_multiplier_n = 0xD2511F53; From bf087ef972a1a30bb078e8cca08f4680a60a5e62 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Thu, 26 Aug 2021 18:45:23 +0300 Subject: [PATCH 13/22] Added comments. --- .../src/runtime/reference/random_uniform.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index bb2ac6912ce0a2..91d0ab3d48806f 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -57,7 +57,8 @@ void raise_key(uint64_t& key) { key = unite_high_low(key_lr.second, key_lr.first); } -// Helper function for converting uint32 values to float32. +// Helper function for converting uint32 values to float32. Sets fractional part of +// floating value with bits from uint32 value. Resulting value is in interval [0,1). float uint32_to_float(uint32_t x) { // float32 is formatted as follows: sign(1 bit) exponent(8 bits) mantissa(23 bits). The value is interpreted // The value is interpreted using following formula: @@ -74,7 +75,8 @@ float uint32_to_float(uint32_t x) { return x_float - 1.0f; } -// Helper function for converting uint32 values to float16. +// Helper function for converting uint32 values to float16.Sets fractional part of +// floating value with bits from uint32 value. Resulting value is in interval [0,1). float16 uint32_to_float16(uint32_t x) { // float16 is formatted as follows: sign(1 bit) exponent(5 bits) mantissa(10 bits). The value is interpreted // The value is interpreted using following formula: @@ -92,7 +94,8 @@ float16 uint32_to_float16(uint32_t x) { return x_float16 - static_cast(1); } -// Helper function for converting uint32 values to double. +// Helper function for converting uint32 values to double. Sets fractional part of +// floating double with bits from uint32 values. Resulting value is in interval [0,1). double uint32_to_double(uint32_t x1, uint32_t x2) { // float64 is formatted as follows: sign(1 bit) exponent(11 bits) mantissa(52 bits). The value is interpreted // The value is interpreted using following formula: @@ -110,7 +113,8 @@ double uint32_to_double(uint32_t x1, uint32_t x2) { return x_double - 1.0; } -// Helper function for converting uint32 values to bfloat16. +// Helper function for converting uint32 values to bfloat16. Sets fractional part of +// floating value with bits from uint32 value. Resulting value is in interval [0,1). bfloat16 uint32_to_bfloat16(uint32_t x) { // bfloat16 is formatted as follows: sign(1 bit) exponent(8 bits) mantissa(7 bits). The value is interpreted // The value is interpreted using following formula: From d12d03239d9af5a8f436c13b2a8956e27ea58b83 Mon Sep 17 00:00:00 2001 From: Popova Date: Fri, 27 Aug 2021 10:59:46 +0300 Subject: [PATCH 14/22] Temporarily added debug output. --- .../src/runtime/reference/random_uniform.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 91d0ab3d48806f..6bb7d7af73783e 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -257,18 +257,37 @@ void random_uniform(const uint64_t* out_shape, } case ngraph::element::Type_t::i64: { std::vector res_int64(step); + std::cout << "random_uniform iteration start." << std::endl; + std::cout << step << std::endl; int64_t mn[1]; int64_t mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); + std::cout << "min max:" << std::endl; + std::cout << mn[0] << std::endl; + std::cout << mx[0] << std::endl; // convert 2 pairs of uint32 values to 2 double values and normalize to // range [min_val, max_val) + std::cout << "philox output:" << std::endl; + std::cout << res[0] << std::endl; + std::cout << res[1] << std::endl; + std::cout << res[2] << std::endl; + std::cout << res[3] << std::endl; + std::cout << "unite_high_low:" << std::endl; + std::cout << unite_high_low(res[1], res[0]) << std::endl; + std::cout << unite_high_low(res[3], res[2]) << std::endl; auto v1 = static_cast(unite_high_low(res[1], res[0]) % (mx[0] - mn[0])); auto v2 = static_cast(unite_high_low(res[3], res[2]) % (mx[0] - mn[0])); + std::cout << "v1 v2:" << std::endl; + std::cout << v1 << std::endl; + std::cout << v2 << std::endl; res_int64[0] = v1 + mn[0]; res_int64[1] = v2 + mn[0]; + std::cout << "memcopy start:" << std::endl; memcpy(out + k * elem_type.size(), res_int64.data(), std::min(step, elem_count - k) * elem_type.size()); + std::cout << "memcopy end." << std::endl; + std::cout << "random_uniform iteration end." << std::endl; break; } default: From d38b31b5d4c7349dbb6e9ece98631b6052bb9c86 Mon Sep 17 00:00:00 2001 From: Popova Date: Fri, 27 Aug 2021 12:25:56 +0300 Subject: [PATCH 15/22] Temporarily added debug output. --- .../src/runtime/reference/random_uniform.cpp | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 6bb7d7af73783e..f0313657120d42 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -135,7 +135,9 @@ bfloat16 uint32_to_bfloat16(uint32_t x) { // Runs Philox algorithm. void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std::vector& res) { for (size_t i = 0; i < n_rounds; i++) { + std::cout << "round start." << std::endl; calculate_round(key, counter, n); + std::cout << "round end." << std::endl; if (i < n_rounds - 1) raise_key(key); } @@ -156,6 +158,7 @@ void random_uniform(const uint64_t* out_shape, const ngraph::element::Type& elem_type, uint64_t seed, uint64_t seed2) { + std::cout << "random_uniform start." << std::endl; if (seed == 0 && seed2 == 0) { std::srand(std::time(nullptr)); seed = std::rand(); @@ -168,17 +171,24 @@ void random_uniform(const uint64_t* out_shape, for (size_t i = 0; i < shape_count; i++) { elem_count *= out_shape[i]; } + std::cout << "elem count = " << elem_count << std::endl; + std::cout << "shape_count = " << shape_count << std::endl; // Each run of Philox algorithm generates 4 uint32 values. // If output_type is int32, f32, bf16, or f16 each value is converted to // corresponding type so we have 4 result values. For f64 and i64 we use // a pair of values for conversion, so we have 2 result values. // Step indicates how many values we generate in one iteration. const size_t step = elem_type.size() > 4 ? 2 : 4; + std::cout << "step = " << step << std::endl; + std::cout << "elem_type.size() = " << elem_type.size() << std::endl; for (size_t k = 0; k < elem_count; k += step) { // generate 4 random uint32 values using Philox algorithm + std::cout << "k = " << k << std::endl; std::vector res(step); + std::cout << "philox start." << std::endl; run_philox(key, counter, n, rounds_number, res); + std::cout << "philox end." << std::endl; // convert values to corresponding output_type switch (elem_type) { @@ -256,38 +266,21 @@ void random_uniform(const uint64_t* out_shape, break; } case ngraph::element::Type_t::i64: { + std::cout << "convert to int64 start." << std::endl; std::vector res_int64(step); - std::cout << "random_uniform iteration start." << std::endl; - std::cout << step << std::endl; int64_t mn[1]; int64_t mx[1]; memcpy(mn, min_val, elem_type.size()); memcpy(mx, max_val, elem_type.size()); - std::cout << "min max:" << std::endl; - std::cout << mn[0] << std::endl; - std::cout << mx[0] << std::endl; // convert 2 pairs of uint32 values to 2 double values and normalize to // range [min_val, max_val) - std::cout << "philox output:" << std::endl; - std::cout << res[0] << std::endl; - std::cout << res[1] << std::endl; - std::cout << res[2] << std::endl; - std::cout << res[3] << std::endl; - std::cout << "unite_high_low:" << std::endl; - std::cout << unite_high_low(res[1], res[0]) << std::endl; - std::cout << unite_high_low(res[3], res[2]) << std::endl; auto v1 = static_cast(unite_high_low(res[1], res[0]) % (mx[0] - mn[0])); auto v2 = static_cast(unite_high_low(res[3], res[2]) % (mx[0] - mn[0])); - std::cout << "v1 v2:" << std::endl; - std::cout << v1 << std::endl; - std::cout << v2 << std::endl; res_int64[0] = v1 + mn[0]; res_int64[1] = v2 + mn[0]; - std::cout << "memcopy start:" << std::endl; memcpy(out + k * elem_type.size(), res_int64.data(), std::min(step, elem_count - k) * elem_type.size()); - std::cout << "memcopy end." << std::endl; - std::cout << "random_uniform iteration end." << std::endl; + std::cout << "convert to int64 end." << std::endl; break; } default: @@ -296,6 +289,7 @@ void random_uniform(const uint64_t* out_shape, if (++n == 0) ++counter; } + std::cout << "random_uniform end." << std::endl; } } // namespace reference From 80fb9a0af684d93f73f45db3daa04555ceee277f Mon Sep 17 00:00:00 2001 From: Popova Date: Fri, 27 Aug 2021 12:54:59 +0300 Subject: [PATCH 16/22] Removed debug output. --- .../src/runtime/reference/random_uniform.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index f0313657120d42..3e86f9d0c8ad12 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -135,9 +135,7 @@ bfloat16 uint32_to_bfloat16(uint32_t x) { // Runs Philox algorithm. void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std::vector& res) { for (size_t i = 0; i < n_rounds; i++) { - std::cout << "round start." << std::endl; calculate_round(key, counter, n); - std::cout << "round end." << std::endl; if (i < n_rounds - 1) raise_key(key); } @@ -158,7 +156,6 @@ void random_uniform(const uint64_t* out_shape, const ngraph::element::Type& elem_type, uint64_t seed, uint64_t seed2) { - std::cout << "random_uniform start." << std::endl; if (seed == 0 && seed2 == 0) { std::srand(std::time(nullptr)); seed = std::rand(); @@ -168,27 +165,21 @@ void random_uniform(const uint64_t* out_shape, uint64_t n = 0; size_t shape_count = shape_size(out_shape_shape); size_t elem_count = 1; + const size_t philox_output_size = 4; for (size_t i = 0; i < shape_count; i++) { elem_count *= out_shape[i]; } - std::cout << "elem count = " << elem_count << std::endl; - std::cout << "shape_count = " << shape_count << std::endl; // Each run of Philox algorithm generates 4 uint32 values. // If output_type is int32, f32, bf16, or f16 each value is converted to // corresponding type so we have 4 result values. For f64 and i64 we use // a pair of values for conversion, so we have 2 result values. // Step indicates how many values we generate in one iteration. const size_t step = elem_type.size() > 4 ? 2 : 4; - std::cout << "step = " << step << std::endl; - std::cout << "elem_type.size() = " << elem_type.size() << std::endl; for (size_t k = 0; k < elem_count; k += step) { // generate 4 random uint32 values using Philox algorithm - std::cout << "k = " << k << std::endl; - std::vector res(step); - std::cout << "philox start." << std::endl; + std::vector res(philox_output_size); run_philox(key, counter, n, rounds_number, res); - std::cout << "philox end." << std::endl; // convert values to corresponding output_type switch (elem_type) { @@ -266,7 +257,6 @@ void random_uniform(const uint64_t* out_shape, break; } case ngraph::element::Type_t::i64: { - std::cout << "convert to int64 start." << std::endl; std::vector res_int64(step); int64_t mn[1]; int64_t mx[1]; @@ -280,7 +270,6 @@ void random_uniform(const uint64_t* out_shape, res_int64[0] = v1 + mn[0]; res_int64[1] = v2 + mn[0]; memcpy(out + k * elem_type.size(), res_int64.data(), std::min(step, elem_count - k) * elem_type.size()); - std::cout << "convert to int64 end." << std::endl; break; } default: @@ -289,7 +278,6 @@ void random_uniform(const uint64_t* out_shape, if (++n == 0) ++counter; } - std::cout << "random_uniform end." << std::endl; } } // namespace reference From 63e8ebbf17ef3506889d73c6bff7d38fbb9b88d0 Mon Sep 17 00:00:00 2001 From: Popova Date: Fri, 27 Aug 2021 13:40:32 +0300 Subject: [PATCH 17/22] Added comment. --- .../tests/functional/op_reference/random_uniform.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp index 1f7784b8bbce75..ae467e272d3703 100644 --- a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp +++ b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp @@ -87,6 +87,7 @@ TEST_P(ReferenceRandomUniformLayerTest, RandomUniformWithHardcodedRefs) { } // namespace +// Reference values for the following tests are obtained from single layer TensorFlow model with tf.random.uniform(). INSTANTIATE_TEST_SUITE_P( smoke_RandomUniform_With_Hardcoded_Refs, ReferenceRandomUniformLayerTest, @@ -127,6 +128,7 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f32, +#The following numbers are obtained from tf.random.uniform(). std::vector{121.23596191, -314.06405640, 383.24157715, 390.16381836, -521.35742188, -91.52935791, -78.30828857, -399.99786377, 440.51147461, -259.29055786, 259.61541748, 8.51342773, @@ -142,6 +144,7 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f16, +#The following numbers are obtained from tf.random.uniform(). std::vector{-1.19726562, -1.09667969, -1.08398438, -1.30859375, -1.48242188, -1.45898438, -1.22851562, -1.08300781, -1.33203125, -1.14062500, -1.42285156, -1.43554688, -1.32617188, -1.06542969, -1.29296875, From d2a110ac46c74a5726c12e017da50a8911bde7be Mon Sep 17 00:00:00 2001 From: Popova Date: Fri, 27 Aug 2021 14:15:56 +0300 Subject: [PATCH 18/22] Added comment. --- .../tests/functional/op_reference/random_uniform.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp index ae467e272d3703..2e454e4a1453ee 100644 --- a/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp +++ b/docs/template_plugin/tests/functional/op_reference/random_uniform.cpp @@ -87,7 +87,7 @@ TEST_P(ReferenceRandomUniformLayerTest, RandomUniformWithHardcodedRefs) { } // namespace -// Reference values for the following tests are obtained from single layer TensorFlow model with tf.random.uniform(). +// Reference values for the following tests are obtained from single layer TensorFlow model with tf.random.uniform(). INSTANTIATE_TEST_SUITE_P( smoke_RandomUniform_With_Hardcoded_Refs, ReferenceRandomUniformLayerTest, @@ -128,7 +128,6 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f32, -#The following numbers are obtained from tf.random.uniform(). std::vector{121.23596191, -314.06405640, 383.24157715, 390.16381836, -521.35742188, -91.52935791, -78.30828857, -399.99786377, 440.51147461, -259.29055786, 259.61541748, 8.51342773, @@ -144,7 +143,6 @@ INSTANTIATE_TEST_SUITE_P( 10, Tensor{{3, 2, 4}, element::f16, -#The following numbers are obtained from tf.random.uniform(). std::vector{-1.19726562, -1.09667969, -1.08398438, -1.30859375, -1.48242188, -1.45898438, -1.22851562, -1.08300781, -1.33203125, -1.14062500, -1.42285156, -1.43554688, -1.32617188, -1.06542969, -1.29296875, From 1e259e3f3fdb62c3241c49c787129248d18b48c6 Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Fri, 3 Sep 2021 23:22:26 +0300 Subject: [PATCH 19/22] Enabled state saving for RandomUniform. --- ...isable_random_uniform_constant_folding.hpp | 27 ++++++++++++++++++ .../common_optimizations.cpp | 2 ++ ...isable_random_uniform_constant_folding.cpp | 24 ++++++++++++++++ .../core/include/ngraph/op/random_uniform.hpp | 12 ++++++-- .../runtime/reference/random_uniform.hpp | 9 ++++-- .../src/runtime/reference/random_uniform.cpp | 28 +++++++++++++++---- ngraph/core/src/op/random_uniform.cpp | 9 ++++-- 7 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 inference-engine/src/transformations/include/transformations/common_optimizations/disable_random_uniform_constant_folding.hpp create mode 100644 inference-engine/src/transformations/src/transformations/common_optimizations/disable_random_uniform_constant_folding.cpp diff --git a/inference-engine/src/transformations/include/transformations/common_optimizations/disable_random_uniform_constant_folding.hpp b/inference-engine/src/transformations/include/transformations/common_optimizations/disable_random_uniform_constant_folding.hpp new file mode 100644 index 00000000000000..e9c59587eb67de --- /dev/null +++ b/inference-engine/src/transformations/include/transformations/common_optimizations/disable_random_uniform_constant_folding.hpp @@ -0,0 +1,27 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include +#include + +namespace ngraph { +namespace pass { + +class DisableRandomUniformConstantFolding; + +} // namespace pass +} // namespace ngraph + +/** + * @ingroup ie_transformation_common_api + * @brief Disables ConstantFolding for RandomUniform operation. It is required as RandomUniform + * should generate new sequence each run. + */ +class ngraph::pass::DisableRandomUniformConstantFolding : public ngraph::pass::MatcherPass { +public: + NGRAPH_RTTI_DECLARATION; + DisableRandomUniformConstantFolding(); +}; diff --git a/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp b/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp index 253c4f113ab073..4e176543504b20 100644 --- a/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp +++ b/inference-engine/src/transformations/src/transformations/common_optimizations/common_optimizations.cpp @@ -27,6 +27,7 @@ #include "transformations/common_optimizations/hswish_fusion.hpp" #include "transformations/common_optimizations/convert_quantize_dequantize.hpp" #include "transformations/common_optimizations/relu_fake_quantize_fusion.hpp" +#include "transformations/common_optimizations/disable_random_uniform_constant_folding.hpp" #include "transformations/common_optimizations/add_fake_quantize_fusion.hpp" #include "transformations/common_optimizations/mul_fake_quantize_fusion.hpp" #include "transformations/common_optimizations/clamp_fusion.hpp" @@ -88,6 +89,7 @@ bool ngraph::pass::CommonOptimizations::run_on_function(std::shared_ptr(); + manager.register_pass(); manager.register_pass(); manager.register_pass(); manager.register_pass(); // Resolves dynamism (replaces NonZero), CF needed diff --git a/inference-engine/src/transformations/src/transformations/common_optimizations/disable_random_uniform_constant_folding.cpp b/inference-engine/src/transformations/src/transformations/common_optimizations/disable_random_uniform_constant_folding.cpp new file mode 100644 index 00000000000000..7c93745d3fa449 --- /dev/null +++ b/inference-engine/src/transformations/src/transformations/common_optimizations/disable_random_uniform_constant_folding.cpp @@ -0,0 +1,24 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "transformations/common_optimizations/disable_random_uniform_constant_folding.hpp" + +#include +#include +#include +#include + +NGRAPH_RTTI_DEFINITION(ngraph::pass::DisableRandomUniformConstantFolding, "DisableRandomUniformConstantFolding", 0); + +ngraph::pass::DisableRandomUniformConstantFolding::DisableRandomUniformConstantFolding() { + auto random_uniform = pattern::wrap_type(); + + ngraph::matcher_pass_callback callback = [=](pattern::Matcher& m) { + disable_constant_folding(m.get_match_root()); + return true; + }; + + auto m = std::make_shared(random_uniform, "DisableRandomUniformConstantFolding"); + this->register_matcher(m, callback); +} diff --git a/ngraph/core/include/ngraph/op/random_uniform.hpp b/ngraph/core/include/ngraph/op/random_uniform.hpp index 8586fe75796b92..f20ddecccab431 100644 --- a/ngraph/core/include/ngraph/op/random_uniform.hpp +++ b/ngraph/core/include/ngraph/op/random_uniform.hpp @@ -30,8 +30,8 @@ class NGRAPH_API RandomUniform : public Op { const Output& min_val, const Output& max_val, const ngraph::element::Type& out_type, - uint64_t global_seed, - uint64_t op_seed); + uint64_t global_seed = 0, + uint64_t op_seed = 0); void validate_and_infer_types() override; @@ -39,6 +39,11 @@ class NGRAPH_API RandomUniform : public Op { std::shared_ptr clone_with_new_inputs(const OutputVector& new_args) const override; + /// \return Turns off constant folding for RandomUniform operation. + bool constant_fold(OutputVector& output_values, const OutputVector& inputs_values) override { + return false; + } + /// \return The output tensor type. const ngraph::element::Type& get_out_type() const { return m_output_type; @@ -71,6 +76,9 @@ class NGRAPH_API RandomUniform : public Op { ngraph::element::Type m_output_type; uint64_t m_global_seed; uint64_t m_op_seed; + + mutable std::mutex m_state_mutex; + mutable std::pair m_state; }; } // namespace v8 } // namespace op diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index aca4bc8e9470de..b17fadabd1c287 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -12,14 +12,15 @@ namespace ngraph { namespace runtime { namespace reference { -void random_uniform(const uint64_t* out_shape, + std::pair random_uniform(const uint64_t* out_shape, const char* min_val, const char* max_val, char* out, const Shape& out_shape_shape, const ngraph::element::Type& elem_type, uint64_t seed, - uint64_t seed2); + uint64_t seed2, + std::pair prev_state); // Following const values are taken from the original paper: // https://www.thesalmons.org/john/random123/papers/random123sc11.pdf @@ -29,6 +30,10 @@ const uint64_t statistic_maximizing_multiplier_n = 0xD2511F53; const uint64_t statistic_maximizing_multiplier_counter = 0xCD9E8D57; const size_t rounds_number = 10; +// Determines how many sequence elements of RNG sequence are skipped between runs. +// Can be any positive value, 256 is chosen for parity with Tensorflow. +const uint64_t skip_const = 256; + } // namespace reference } // namespace runtime } // namespace ngraph diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 3e86f9d0c8ad12..051ef5288bebab 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -148,27 +148,37 @@ void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std } // Implementation of RandomUniform that uses Philox algorithm as inner random unsigned integer generator. -void random_uniform(const uint64_t* out_shape, +std::pair random_uniform(const uint64_t* out_shape, const char* min_val, const char* max_val, char* out, const Shape& out_shape_shape, const ngraph::element::Type& elem_type, uint64_t seed, - uint64_t seed2) { + uint64_t seed2, + std::pair prev_state) { if (seed == 0 && seed2 == 0) { std::srand(std::time(nullptr)); seed = std::rand(); } + uint64_t n_state = prev_state.first; + uint64_t counter_state = prev_state.second; + + // Init Philox key and counters uint64_t key = seed; - uint64_t counter = seed2; - uint64_t n = 0; + uint64_t counter = counter_state > 0 ? counter_state : seed2; + uint64_t n = n_state; + + // Calculate total element count for generation size_t shape_count = shape_size(out_shape_shape); size_t elem_count = 1; - const size_t philox_output_size = 4; for (size_t i = 0; i < shape_count; i++) { elem_count *= out_shape[i]; } + + // Philox algorithm returns 4 elements of RNG sequence per each invocation + const size_t philox_output_size = 4; + // Each run of Philox algorithm generates 4 uint32 values. // If output_type is int32, f32, bf16, or f16 each value is converted to // corresponding type so we have 4 result values. For f64 and i64 we use @@ -278,6 +288,14 @@ void random_uniform(const uint64_t* out_shape, if (++n == 0) ++counter; } + + // Calculate counter values for next RandomUniform run + uint64_t skip_count = elem_count * skip_const; + n_state += skip_count; + if (n_state < skip_count) + counter_state++; + + return {n_state, counter_state}; } } // namespace reference diff --git a/ngraph/core/src/op/random_uniform.cpp b/ngraph/core/src/op/random_uniform.cpp index 67c7e9e37e3e33..7d6db6a8827f64 100644 --- a/ngraph/core/src/op/random_uniform.cpp +++ b/ngraph/core/src/op/random_uniform.cpp @@ -182,14 +182,19 @@ bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, const Host throw ngraph_error("Unsupported type of RandomUniform: " + get_out_type().get_type_name()); } - runtime::reference::random_uniform(out_shape, + auto state = runtime::reference::random_uniform(out_shape, inputs[1]->get_data_ptr(), inputs[2]->get_data_ptr(), out, inputs[0]->get_shape(), get_out_type(), get_global_seed(), - get_op_seed()); + get_op_seed(), + m_state); + + // Update RandomUniform state + std::lock_guard guard(m_state_mutex); + m_state = state; return true; } From 201a4e782076bc1fe89c03886bfecb82606eb48f Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Sun, 5 Sep 2021 11:34:29 +0300 Subject: [PATCH 20/22] Code style. --- .../runtime/reference/random_uniform.hpp | 18 +++++++++--------- .../src/runtime/reference/random_uniform.cpp | 16 ++++++++-------- ngraph/core/src/op/random_uniform.cpp | 16 ++++++++-------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp index b17fadabd1c287..43df6529de33a6 100644 --- a/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp +++ b/ngraph/core/reference/include/ngraph/runtime/reference/random_uniform.hpp @@ -12,15 +12,15 @@ namespace ngraph { namespace runtime { namespace reference { - std::pair random_uniform(const uint64_t* out_shape, - const char* min_val, - const char* max_val, - char* out, - const Shape& out_shape_shape, - const ngraph::element::Type& elem_type, - uint64_t seed, - uint64_t seed2, - std::pair prev_state); +std::pair random_uniform(const uint64_t* out_shape, + const char* min_val, + const char* max_val, + char* out, + const Shape& out_shape_shape, + const ngraph::element::Type& elem_type, + uint64_t seed, + uint64_t seed2, + std::pair prev_state); // Following const values are taken from the original paper: // https://www.thesalmons.org/john/random123/papers/random123sc11.pdf diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 051ef5288bebab..393dc5c1a52836 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -149,14 +149,14 @@ void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std // Implementation of RandomUniform that uses Philox algorithm as inner random unsigned integer generator. std::pair random_uniform(const uint64_t* out_shape, - const char* min_val, - const char* max_val, - char* out, - const Shape& out_shape_shape, - const ngraph::element::Type& elem_type, - uint64_t seed, - uint64_t seed2, - std::pair prev_state) { + const char* min_val, + const char* max_val, + char* out, + const Shape& out_shape_shape, + const ngraph::element::Type& elem_type, + uint64_t seed, + uint64_t seed2, + std::pair prev_state) { if (seed == 0 && seed2 == 0) { std::srand(std::time(nullptr)); seed = std::rand(); diff --git a/ngraph/core/src/op/random_uniform.cpp b/ngraph/core/src/op/random_uniform.cpp index 7d6db6a8827f64..90a356cdef350b 100644 --- a/ngraph/core/src/op/random_uniform.cpp +++ b/ngraph/core/src/op/random_uniform.cpp @@ -183,14 +183,14 @@ bool op::v8::RandomUniform::evaluate(const HostTensorVector& outputs, const Host } auto state = runtime::reference::random_uniform(out_shape, - inputs[1]->get_data_ptr(), - inputs[2]->get_data_ptr(), - out, - inputs[0]->get_shape(), - get_out_type(), - get_global_seed(), - get_op_seed(), - m_state); + inputs[1]->get_data_ptr(), + inputs[2]->get_data_ptr(), + out, + inputs[0]->get_shape(), + get_out_type(), + get_global_seed(), + get_op_seed(), + m_state); // Update RandomUniform state std::lock_guard guard(m_state_mutex); From 597441a213f50a31e4ed6fe3cb438fd79835d9ae Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Sun, 5 Sep 2021 14:14:10 +0300 Subject: [PATCH 21/22] Used to template to convert types. --- .../src/runtime/reference/random_uniform.cpp | 168 ++++++++++-------- 1 file changed, 97 insertions(+), 71 deletions(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 393dc5c1a52836..56856cdca38091 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -147,6 +147,47 @@ void run_philox(uint64_t key, uint64_t counter, uint64_t n, size_t n_rounds, std res[3] = res2.second; } +// Converts uint32 values to destination type and normalizes to required range +template +void convert_to_output_type(const std::vector& res, + size_t step, + const ngraph::element::Type& elem_type, + const char* min_val, + const char* max_val, + char* out, + size_t k, + size_t elem_count, + T (*convert_single_input)(uint32_t) = nullptr, + T (*convert_two_inputs)(uint32_t, uint32_t, T, T) = nullptr, + T (*mod_func)(uint32_t, T, T) = nullptr) { + // Get min and max values + T mn[1]; + T mx[1]; + memcpy(mn, min_val, elem_type.size()); + memcpy(mx, max_val, elem_type.size()); + + std::vector res_out_type(step); + if (elem_type.size() > 4) { + // Each element of resulting sequence is formed using two uint32 values + res_out_type[0] = convert_two_inputs(res[0], res[1], mn[0], mx[0]); + res_out_type[1] = convert_two_inputs(res[2], res[3], mn[0], mx[0]); + } else { + // Each element of resulting sequence is formed using single uint32 value + std::transform(res.data(), + res.data() + step, + res_out_type.data(), + [&mn, &mx, &convert_single_input, &mod_func](uint32_t elem) { + if (convert_single_input != nullptr) { + return convert_single_input(elem) * (mx[0] - mn[0]) + mn[0]; + } else { + return mod_func(elem, mn[0], mx[0]); + } + }); + } + + memcpy(out + k * elem_type.size(), res_out_type.data(), std::min(step, elem_count - k) * elem_type.size()); +} + // Implementation of RandomUniform that uses Philox algorithm as inner random unsigned integer generator. std::pair random_uniform(const uint64_t* out_shape, const char* min_val, @@ -194,92 +235,77 @@ std::pair random_uniform(const uint64_t* out_shape, // convert values to corresponding output_type switch (elem_type) { case ngraph::element::Type_t::f32: { - std::vector res_float(step); - std::transform(res.data(), res.data() + step, res_float.data(), uint32_to_float); - float mn[1]; - float mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to float32 and normalize to range - // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_float.data(), [&mn, &mx](uint32_t elem) { - return uint32_to_float(elem) * (mx[0] - mn[0]) + mn[0]; - }); - - memcpy(out + k * elem_type.size(), res_float.data(), std::min(step, elem_count - k) * elem_type.size()); + convert_to_output_type(res, step, elem_type, min_val, max_val, out, k, elem_count, uint32_to_float); break; } case ngraph::element::Type_t::f16: { - std::vector res_float16(step); - // convert uint32 values to float16 and normalize to range - // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_float16.data(), uint32_to_float16); - float16 mn[1]; - float16 mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - std::transform(res.data(), res.data() + step, res_float16.data(), [&mn, &mx](uint32_t elem) { - return uint32_to_float16(elem) * (mx[0] - mn[0]) + mn[0]; - }); - memcpy(out + k * elem_type.size(), res_float16.data(), std::min(step, elem_count - k) * elem_type.size()); + convert_to_output_type(res, + step, + elem_type, + min_val, + max_val, + out, + k, + elem_count, + uint32_to_float16); break; } case ngraph::element::Type_t::bf16: { - std::vector res_bfloat16(step); - bfloat16 mn[1]; - bfloat16 mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to bfloat16 and normalize to range - // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_bfloat16.data(), [&mn, &mx](uint32_t elem) { - return uint32_to_bfloat16(elem) * (mx[0] - mn[0]) + mn[0]; - }); - memcpy(out + k * elem_type.size(), res_bfloat16.data(), std::min(step, elem_count - k) * elem_type.size()); + convert_to_output_type(res, + step, + elem_type, + min_val, + max_val, + out, + k, + elem_count, + uint32_to_bfloat16); break; } case ngraph::element::Type_t::f64: { - std::vector res_double(step); - double mn[1]; - double mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - - // convert 2 pairs of uint32 values to 2 double values and normalize to - // range [min_val, max_val) - res_double[0] = uint32_to_double(res[0], res[1]) * (mx[0] - mn[0]) + mn[0]; - res_double[1] = uint32_to_double(res[2], res[3]) * (mx[0] - mn[0]) + mn[0]; - memcpy(out + k * elem_type.size(), res_double.data(), std::min(step, elem_count - k) * elem_type.size()); + convert_to_output_type(res, + step, + elem_type, + min_val, + max_val, + out, + k, + elem_count, + nullptr, + [](uint32_t a, uint32_t b, double mn, double mx) { + return uint32_to_double(a, b) * (mx - mn) + mn; + }); break; } case ngraph::element::Type_t::i32: { - std::vector res_int(step); - int mn[1]; - int mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert uint32 values to int32 values and normalize to range - // [min_val, max_val) - std::transform(res.data(), res.data() + step, res_int.data(), [&mn, &mx](uint32_t elem) { - return elem % (mx[0] - mn[0]) + mn[0]; - }); - memcpy(out + k * elem_type.size(), res_int.data(), std::min(step, elem_count - k) * elem_type.size()); + convert_to_output_type(res, + step, + elem_type, + min_val, + max_val, + out, + k, + elem_count, + nullptr, + nullptr, + [](uint32_t x, int mn, int mx) { + return static_cast(x % (mx - mn) + mn); + }); break; } case ngraph::element::Type_t::i64: { - std::vector res_int64(step); - int64_t mn[1]; - int64_t mx[1]; - memcpy(mn, min_val, elem_type.size()); - memcpy(mx, max_val, elem_type.size()); - // convert 2 pairs of uint32 values to 2 double values and normalize to - // range [min_val, max_val) - auto v1 = static_cast(unite_high_low(res[1], res[0]) % (mx[0] - mn[0])); - auto v2 = static_cast(unite_high_low(res[3], res[2]) % (mx[0] - mn[0])); - - res_int64[0] = v1 + mn[0]; - res_int64[1] = v2 + mn[0]; - memcpy(out + k * elem_type.size(), res_int64.data(), std::min(step, elem_count - k) * elem_type.size()); + convert_to_output_type(res, + step, + elem_type, + min_val, + max_val, + out, + k, + elem_count, + nullptr, + [](uint32_t a, uint32_t b, int64_t mn, int64_t mx) { + return static_cast(unite_high_low(b, a) % (mx - mn) + mn); + }); break; } default: From 04226e53370e6d67211c055c7c60d6e84efaf3ba Mon Sep 17 00:00:00 2001 From: Anastasia Popova Date: Sun, 5 Sep 2021 16:29:01 +0300 Subject: [PATCH 22/22] Added comments. --- .../core/reference/src/runtime/reference/random_uniform.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp index 56856cdca38091..6e6f1f7c95d06a 100644 --- a/ngraph/core/reference/src/runtime/reference/random_uniform.cpp +++ b/ngraph/core/reference/src/runtime/reference/random_uniform.cpp @@ -198,14 +198,18 @@ std::pair random_uniform(const uint64_t* out_shape, uint64_t seed, uint64_t seed2, std::pair prev_state) { + // When both seed values are equal to zero RandomUniform should generate non-deterministic sequence. + // Implementation in plugins may differ for this case. if (seed == 0 && seed2 == 0) { std::srand(std::time(nullptr)); seed = std::rand(); } + + // Get previous counter state uint64_t n_state = prev_state.first; uint64_t counter_state = prev_state.second; - // Init Philox key and counters + // Initialize Philox key and counters uint64_t key = seed; uint64_t counter = counter_state > 0 ? counter_state : seed2; uint64_t n = n_state;