[FP8] Implementation of FP8 element types (ov::element::f8e4m3 and ov…

…::element::f8e5m2) (openvinotoolkit#21608) * FP8 element types init * Remove redundant union * Replace using ov * Update fundamental types * Update class name check * Update tests * Remove redundant sign * Expose f8 types in Python * Add to py to dtype map * Style alignment * Align python style * Update test values * Remove constexpr from_bits to fix warning * Add trivially construcitble asserts and common constrexpr * Align python tests opset * Update f8e4m3 <-> float conversion * f8e5m2 class update * Add f8e5m2 unit test * Add to string conversion tests * Rename class f8e4m3 -> float8_e4m3 * Rename f8e5m2 -> float8_e5m2 * Remove size() and to_string from float8 - size() can be replaced by compile time sizeof - to_string can be replaced by std::to_string() * float8 E5M2 remove unused constexpr value * Fix union initialization and ncc style rules * Fix test issues * Use NaN from std::numeric_limits instead macro - minor refactor of float8_e4m3 * Update nf4 usage in element_type.cpp * Sync openvino.style with master * Update f8e5m2 test --------- Co-authored-by: Raasz, Pawel <[email protected]>
akashchi · Jan 16, 2024 · b3c2c38 · b3c2c38
1 parent 0ad801a
commit b3c2c38
Show file tree

Hide file tree

Showing 27 changed files with 1,426 additions and 75 deletions.
diff --git a/cmake/developer_package/ncc_naming_style/openvino.style b/cmake/developer_package/ncc_naming_style/openvino.style
@@ -1,6 +1,6 @@
 # custom OpenVINO values
 CppMethod: '^(operator\W+|[a-z_\d]+|signaling_NaN|quiet_NaN|OPENVINO_OP)$'
-ClassName: '^([A-Z][\w]+|b?float16|numeric_limits|ngraph_error|stopwatch|unsupported_op)$'
+ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|numeric_limits|ngraph_error|stopwatch|unsupported_op)$'
 StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair|stat)$'
 FunctionName: '^(operator\W+|[a-z_\d]+)|PrintTo$'
 Namespace: '^([a-z\d_]*|InferenceEngine)$'
@@ -18,7 +18,7 @@ VariableReference: '^\w+$'
 
 EnumName: '^[A-Z][\w]+$'
 # excepts element_type
-EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u4|u8|u16|u32|u64|nf4|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$'
+EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u4|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$'
 # TODO: align
 UsingDeclaration: '^.*$'
 TypedefName: '^.*$'

diff --git a/src/bindings/c/include/openvino/c/ov_common.h b/src/bindings/c/include/openvino/c/ov_common.h
@@ -187,6 +187,8 @@ typedef enum {
     U32,             //!< u32 element type
     U64,             //!< u64 element type
     NF4,             //!< nf4 element type
+    F8E4M3,          //!< f8e4m3 element type
+    F8E5M3,          //!< f8e5m2 element type
 } ov_element_type_e;
 
 /**
@@ -210,4 +212,4 @@ ov_free(const char* content);
  * @ingroup ov_base_c_api
  */
 OPENVINO_C_API(const char*)
-ov_get_last_err_msg();
+ov_get_last_err_msg();
diff --git a/src/bindings/c/src/ov_tensor.cpp b/src/bindings/c/src/ov_tensor.cpp
@@ -24,7 +24,9 @@ const std::map<ov_element_type_e, ov::element::Type> element_type_map = {
     {ov_element_type_e::U16, ov::element::u16},
     {ov_element_type_e::U32, ov::element::u32},
     {ov_element_type_e::U64, ov::element::u64},
-    {ov_element_type_e::NF4, ov::element::nf4}};
+    {ov_element_type_e::NF4, ov::element::nf4},
+    {ov_element_type_e::F8E4M3, ov::element::f8e4m3},
+    {ov_element_type_e::F8E5M3, ov::element::f8e5m2}};
 
 inline ov_element_type_e find_ov_element_type_e(ov::element::Type type) {
     for (auto iter = element_type_map.begin(); iter != element_type_map.end(); iter++) {

diff --git a/src/bindings/python/src/pyopenvino/core/common.cpp b/src/bindings/python/src/pyopenvino/core/common.cpp
@@ -19,24 +19,16 @@ namespace type_helpers {
 
 const std::map<ov::element::Type, py::dtype>& ov_type_to_dtype() {
     static const std::map<ov::element::Type, py::dtype> ov_type_to_dtype_mapping = {
-        {ov::element::f16, py::dtype("float16")},
-        {ov::element::bf16, py::dtype("float16")},
-        {ov::element::f32, py::dtype("float32")},
-        {ov::element::f64, py::dtype("float64")},
-        {ov::element::i8, py::dtype("int8")},
-        {ov::element::i16, py::dtype("int16")},
-        {ov::element::i32, py::dtype("int32")},
-        {ov::element::i64, py::dtype("int64")},
-        {ov::element::u8, py::dtype("uint8")},
-        {ov::element::u16, py::dtype("uint16")},
-        {ov::element::u32, py::dtype("uint32")},
-        {ov::element::u64, py::dtype("uint64")},
-        {ov::element::boolean, py::dtype("bool")},
-        {ov::element::u1, py::dtype("uint8")},
-        {ov::element::u4, py::dtype("uint8")},
-        {ov::element::nf4, py::dtype("uint8")},
-        {ov::element::i4, py::dtype("int8")},
-        {ov::element::string, py::dtype("bytes_")},
+        {ov::element::f16, py::dtype("float16")},  {ov::element::bf16, py::dtype("float16")},
+        {ov::element::f32, py::dtype("float32")},  {ov::element::f64, py::dtype("float64")},
+        {ov::element::i8, py::dtype("int8")},      {ov::element::i16, py::dtype("int16")},
+        {ov::element::i32, py::dtype("int32")},    {ov::element::i64, py::dtype("int64")},
+        {ov::element::u8, py::dtype("uint8")},     {ov::element::u16, py::dtype("uint16")},
+        {ov::element::u32, py::dtype("uint32")},   {ov::element::u64, py::dtype("uint64")},
+        {ov::element::boolean, py::dtype("bool")}, {ov::element::u1, py::dtype("uint8")},
+        {ov::element::u4, py::dtype("uint8")},     {ov::element::nf4, py::dtype("uint8")},
+        {ov::element::i4, py::dtype("int8")},      {ov::element::f8e4m3, py::dtype("uint8")},
+        {ov::element::f8e5m2, py::dtype("uint8")}, {ov::element::string, py::dtype("bytes_")},
     };
     return ov_type_to_dtype_mapping;
 }

diff --git a/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp b/src/bindings/python/src/pyopenvino/graph/types/element_type.cpp
@@ -50,6 +50,8 @@ void regclass_graph_Type(py::module m) {
     type.attr("u64") = ov::element::u64;
     type.attr("bf16") = ov::element::bf16;
     type.attr("nf4") = ov::element::nf4;
+    type.attr("f8e4m3") = ov::element::f8e4m3;
+    type.attr("f8e5m2") = ov::element::f8e5m2;
     type.attr("string") = ov::element::string;
 
     type.def("__hash__", &ov::element::Type::hash);

diff --git a/src/bindings/python/tests/test_graph/test_constant.py b/src/bindings/python/tests/test_graph/test_constant.py
@@ -411,3 +411,125 @@ def test_memory_sharing(shared_flag):
     else:
         assert not np.array_equal(ov_const.data, arr)
         assert not np.shares_memory(arr, ov_const.data)
+
+
+@pytest.mark.parametrize(("ov_type", "numpy_dtype"), [
+    (Type.f32, np.float32),
+    (Type.f16, np.float16),
+])
+def test_float_to_f8e5m2_constant(ov_type, numpy_dtype):
+    from openvino.runtime import opset12 as opset
+    import openvino as ov
+    data = np.array([4.75, 4.5, -5.25, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5,
+                     0.6, 0.7, 0.8, 0.9, 1, -0.0, -0.1, -0.2, -0.3,
+                     -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, 0.0000152587890625, 448, 500, 512, 57344], dtype=numpy_dtype)
+
+    compressed_const = opset.constant(data, dtype=ov.Type.f8e5m2, name="f8e5m2_constant")
+    convert = opset.convert(compressed_const, data.dtype)
+    parameter = opset.parameter(ov.PartialShape([-1]), ov_type)
+    add_op = opset.add(parameter, convert)
+    model = ov.Model([add_op], [parameter])
+
+    compiled = ov.compile_model(model)
+    tensor = np.zeros(data.shape, dtype=numpy_dtype)
+    result = compiled(tensor)[0]
+
+    target = [5.0, 4.0, -5.0, 0.0, 0.09375, 0.1875, 0.3125, 0.375, 0.5, 0.625, 0.75,
+              0.75, 0.875, 1.0, -0.0, -0.09375, -0.1875, -0.3125, -0.375,
+              -0.5, -0.625, -0.75, -0.75, -0.875, -1.0, 0.0000152587890625,
+              448, 512, 512, 57344]
+    target = np.array(target, dtype=numpy_dtype)
+
+    assert np.allclose(result, target)
+
+
+@pytest.mark.parametrize(("ov_type", "numpy_dtype"), [
+    (Type.f32, np.float32),
+    (Type.f16, np.float16),
+])
+def test_float_to_f8e4m3_constant(ov_type, numpy_dtype):
+    from openvino.runtime import opset12 as opset
+    import openvino as ov
+    data = np.array([4.75, 4.5, -5.25, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5,
+                     0.6, 0.7, 0.8, 0.9, 1, -0.0, -0.1, -0.2, -0.3,
+                     -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1, 448, 512], dtype=numpy_dtype)
+
+    compressed_const = opset.constant(data, dtype=ov.Type.f8e4m3, name="f8e4m3_constant")
+    convert = opset.convert(compressed_const, data.dtype)
+    parameter = opset.parameter(ov.PartialShape([-1]), ov_type)
+    add_op = opset.add(parameter, convert)
+    model = ov.Model([add_op], [parameter])
+
+    compiled = ov.compile_model(model)
+    tensor = np.zeros(data.shape, dtype=numpy_dtype)
+    result = compiled(tensor)[0]
+
+    target = [5.0, 4.5, -5.0, 0.0, 0.1015625, 0.203125, 0.3125,
+              0.40625, 0.5, 0.625, 0.6875, 0.8125, 0.875, 1,
+              -0, -0.1015625, -0.203125, -0.3125, -0.40625, -0.5, -0.625,
+              -0.6875, -0.8125, -0.875, -1, 448, np.nan]
+    target = np.array(target, dtype=numpy_dtype)
+
+    assert np.allclose(result, target, equal_nan=True)
+
+
+@pytest.mark.parametrize(("ov_type", "numpy_dtype"), [
+    (Type.f32, np.float32),
+    (Type.f16, np.float16),
+])
+def test_float_to_f8e5m2_convert(ov_type, numpy_dtype):
+    from openvino.runtime import opset12 as opset
+    import openvino as ov
+    data = np.array([4.75, 4.5, -5.25, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5,
+                     0.6, 0.7, 0.8, 0.9, 1, -0.0, -0.1, -0.2, -0.3,
+                     -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, 0.0000152587890625, 448, 500, 512, 57344], dtype=numpy_dtype)
+
+    compressed_const = opset.constant(data, dtype=ov_type, name="fx_constant")
+    convert_to_fp8 = opset.convert(compressed_const, Type.f8e5m2)
+    convert_back = opset.convert(convert_to_fp8, ov_type)
+    parameter = opset.parameter(ov.PartialShape([-1]), ov_type)
+    add_op = opset.add(parameter, convert_back)
+    model = ov.Model([add_op], [parameter])
+
+    compiled = ov.compile_model(model)
+    tensor = np.zeros(data.shape, dtype=numpy_dtype)
+    result = compiled(tensor)[0]
+
+    target = [5.0, 4.0, -5.0, 0.0, 0.09375, 0.1875, 0.3125, 0.375, 0.5, 0.625, 0.75,
+              0.75, 0.875, 1.0, -0.0, -0.09375, -0.1875, -0.3125, -0.375,
+              -0.5, -0.625, -0.75, -0.75, -0.875, -1.0, 0.0000152587890625,
+              448, 512, 512, 57344]
+    target = np.array(target, dtype=numpy_dtype)
+
+    assert np.allclose(result, target)
+
+
+@pytest.mark.parametrize(("ov_type", "numpy_dtype"), [
+    (Type.f32, np.float32),
+    (Type.f16, np.float16),
+])
+def test_float_to_f8e4m3_convert(ov_type, numpy_dtype):
+    from openvino.runtime import opset12 as opset
+    import openvino as ov
+    data = np.array([4.75, 4.5, -5.25, 0.0, 0.1, 0.2, 0.3, 0.4, 0.5,
+                     0.6, 0.7, 0.8, 0.9, 1, -0.0, -0.1, -0.2, -0.3,
+                     -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1, 448, 512], dtype=numpy_dtype)
+
+    compressed_const = opset.constant(data, dtype=ov_type, name="fx_constant")
+    convert_to_fp8 = opset.convert(compressed_const, Type.f8e4m3)
+    convert_back = opset.convert(convert_to_fp8, ov_type)
+    parameter = opset.parameter(ov.PartialShape([-1]), ov_type)
+    add_op = opset.add(parameter, convert_back)
+    model = ov.Model([add_op], [parameter])
+
+    compiled = ov.compile_model(model)
+    tensor = np.zeros(data.shape, dtype=numpy_dtype)
+    result = compiled(tensor)[0]
+
+    target = [5.0, 4.5, -5.0, 0.0, 0.1015625, 0.203125, 0.3125,
+              0.40625, 0.5, 0.625, 0.6875, 0.8125, 0.875, 1,
+              -0, -0.1015625, -0.203125, -0.3125, -0.40625, -0.5, -0.625,
+              -0.6875, -0.8125, -0.875, -1, 448, np.nan]
+    target = np.array(target, dtype=numpy_dtype)
+
+    assert np.allclose(result, target, equal_nan=True)
diff --git a/src/core/include/ngraph/type/element_type.hpp b/src/core/include/ngraph/type/element_type.hpp
@@ -35,6 +35,8 @@ using ov::element::dynamic;
 using ov::element::f16;
 using ov::element::f32;
 using ov::element::f64;
+using ov::element::f8e4m3;
+using ov::element::f8e5m2;
 using ov::element::i16;
 using ov::element::i32;
 using ov::element::i4;

diff --git a/src/core/include/openvino/core/type/element_type.hpp b/src/core/include/openvino/core/type/element_type.hpp
@@ -20,6 +20,8 @@
 #include "openvino/core/rtti.hpp"
 #include "openvino/core/type/bfloat16.hpp"
 #include "openvino/core/type/float16.hpp"
+#include "openvino/core/type/float8_e4m3.hpp"
+#include "openvino/core/type/float8_e5m2.hpp"
 
 /**
  * @defgroup ov_element_cpp_api Element types
@@ -52,6 +54,8 @@ enum class Type_t {
     u32,        //!< u32 element type
     u64,        //!< u64 element type
     nf4,        //!< nf4 element type
+    f8e4m3,     //!< f8e4m3 element type
+    f8e5m2,     //!< f8e5m2 element type
     string      //!< string element type
 };
 
@@ -182,6 +186,12 @@ constexpr Type u64(Type_t::u64);
 /// \brief nf4 element type
 /// \ingroup ov_element_cpp_api
 constexpr Type nf4(Type_t::nf4);
+/// \brief f8e4m3 element type
+/// \ingroup ov_element_cpp_api
+constexpr Type f8e4m3(Type_t::f8e4m3);
+/// \brief f8e4m3 element type
+/// \ingroup ov_element_cpp_api
+constexpr Type f8e5m2(Type_t::f8e5m2);
 /// \brief string element type
 /// \ingroup ov_element_cpp_api
 constexpr Type string(Type_t::string);
@@ -219,6 +229,10 @@ OPENVINO_API Type from<ov::bfloat16>();
 template <>
 OPENVINO_API Type from<ov::float16>();
 template <>
+OPENVINO_API Type from<ov::float8_e4m3>();
+template <>
+OPENVINO_API Type from<ov::float8_e5m2>();
+template <>
 OPENVINO_API Type from<std::string>();
 
 OPENVINO_API Type fundamental_type_for(const Type& type);

diff --git a/src/core/include/openvino/core/type/element_type_traits.hpp b/src/core/include/openvino/core/type/element_type_traits.hpp
@@ -98,6 +98,16 @@ struct element_type_traits<element::Type_t::nf4> {
     using value_type = int8_t;
 };
 
+template <>
+struct element_type_traits<element::Type_t::f8e4m3> {
+    using value_type = ov::float8_e4m3;
+};
+
+template <>
+struct element_type_traits<element::Type_t::f8e5m2> {
+    using value_type = ov::float8_e5m2;
+};
+
 template <>
 struct element_type_traits<element::Type_t::string> {
     using value_type = std::string;