Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Op Conformance] Update compare accuracy function #21783

Merged
merged 1 commit into from
Jan 23, 2024
Merged

[Op Conformance] Update compare accuracy function #21783

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 49 additions & 5 deletions src/tests/test_utils/common_test_utils/src/ov_tensor_utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,11 +305,24 @@ ov::runtime::Tensor create_and_fill_tensor_consistently(const ov::element::Type
constexpr double eps = std::numeric_limits<double>::epsilon();

inline double less(double a, double b) {
return (b - a) > (std::fmax(std::fabs(a), std::fabs(b)) * eps);
return std::fabs(a - b) > eps && a < b;
}

inline double less_or_equal(double a, double b) {
return (b - a) >= (std::fmax(std::fabs(a), std::fabs(b)) * eps);
bool res = true;
if (std::isnan(a) || std::isnan(b)) {
res = false;
} else if (std::isinf(b) && b > 0) {
// b is grater than any number or eq the +Inf
res = true;
} else if (std::isinf(a) && a > 0) {
res = false;
} else {
res = (std::fabs(b - a) <= (std::fmax(std::fabs(a), std::fabs(b)) * eps) || a < b);
}
double eq_midle_res = std::fabs(b - a);
bool eq_res = (std::fabs(b - a) <= (std::fmax(std::fabs(a), std::fabs(b)) * eps));
return res;
}

struct Error {
Expand Down Expand Up @@ -369,13 +382,33 @@ void compare(const ov::Tensor& expected,
if (abs_threshold == std::numeric_limits<double>::max() && rel_threshold == std::numeric_limits<double>::max()) {
if (sizeof(ExpectedT) == 1 || sizeof(ActualT) == 1) {
abs_threshold = 1.;
rel_threshold = 1.;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sbalandi Please check the changes on all platforms and skip tests per platform (in case some test is failed only on one platform)

if (expected.get_element_type() == ov::element::Type_t::boolean) {
abs_threshold = 0.;
rel_threshold = 0.;
}
} else {
std::vector<double> abs_values(shape_size_cnt);
for (size_t i = 0; i < shape_size_cnt; i++) {
abs_values[i] = std::fabs(static_cast<double>(expected_data[i]));
}
auto abs_median = calculate_median(abs_values);
auto elem_type = expected.get_element_type();

abs_threshold = abs_median * 0.05 < 1e-5 ? 1e-5 : 0.05 * abs_median;

if (elem_type == ov::element::Type_t::boolean) {
abs_threshold = 0.;
} else if (elem_type.is_integral_number()) {
abs_threshold = 1.0;
} else if (elem_type == ov::element::Type_t::f32 || elem_type == ov::element::Type_t::f64) {
abs_threshold = abs_median * 0.05 < 1e-5 ? 1e-5 : 0.05 * abs_median;
} else if (elem_type == ov::element::Type_t::bf16 || elem_type == ov::element::Type_t::f16) {
abs_threshold = abs_median * 0.05 < 1e-3 ? 1e-3 : 0.05 * abs_median;
}

rel_threshold = abs_threshold;

if (std::is_integral<ExpectedT>::value) {
abs_threshold = std::ceil(abs_threshold);
}
Expand All @@ -388,10 +421,21 @@ void compare(const ov::Tensor& expected,
std::cout << "[ COMPARATION ] abs_threshold: " << abs_threshold << std::endl;
}

auto max_type_expected = std::numeric_limits<ExpectedT>::max();
auto max_type_actual = std::numeric_limits<ActualT>::max();
auto min_type_expected = std::numeric_limits<ExpectedT>::min();
auto min_type_actual = std::numeric_limits<ActualT>::min();
Error abs_error(abs_threshold), rel_error(rel_threshold);
for (size_t i = 0; i < shape_size_cnt; ++i) {
double expected_value = expected_data[i];
double actual_value = actual_data[i];
if ((std::isinf(expected_value) || expected_value >= max_type_expected) &&
(std::isinf(actual_value) || actual_value >= max_type_actual)) {
continue;
} else if ((std::isinf(expected_value) || expected_value <= min_type_expected) &&
(std::isinf(actual_value) || actual_value <= min_type_actual)) {
continue;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

std::numeric_limits<float>::min() is smallest positive number representable by float, here all negative values can pass the check, which is not correct behavior, maybe you mean std::numeric_limits<float>::lowest() ?

}
if (std::isnan(expected_value) && std::isnan(actual_value))
continue;
if (std::isnan(expected_value)) {
Expand All @@ -406,15 +450,15 @@ void compare(const ov::Tensor& expected,
}

double abs = std::fabs(expected_value - actual_value);
double rel = expected_value ? (abs / std::fabs(expected_value)) : abs;

double rel =
expected_value && actual_value && !std::isinf(expected_value) ? (abs / std::fabs(expected_value)) : 0;
abs_error.update(abs, i);
rel_error.update(rel, i);
}
abs_error.mean /= shape_size_cnt;
rel_error.mean /= shape_size_cnt;

if (!(less_or_equal(abs_error.max, abs_threshold) && less_or_equal(rel_error.max, rel_threshold))) {
if (!(less_or_equal(abs_error.max, abs_threshold) || less_or_equal(rel_error.mean, rel_threshold))) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@iefode This PR leads to some failures of plugin unit test pass. Would you please check again?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ceciliapeng2011 In your case, please update your branch to latest master. Commit was reverted

std::ostringstream out_stream;
out_stream << "abs_max < abs_threshold && rel_max < rel_threshold"
<< "\n\t abs_max: " << abs_error.max << "\n\t\t coordinate " << abs_error.max_coordinate
Expand Down
Loading