XNNPACK backend supports subgraph API

DEPS

@@ -62,7 +62,7 @@ deps = {
     'url': '{github_git}/oneapi-src/oneDNN.git@4a129541fd4e67e6897072186ea2817a3154eddd',
   },
   'third_party/XNNPACK': {
-    'url': '{github_git}/google/XNNPACK.git@60fc61373f21f0ad3164cc719de464f4b787dc04'
+    'url': '{github_git}/google/XNNPACK.git@a9c0465458e185d8189f483b1bdd8965a3341838'
   },
   'third_party/onnxruntime': {
     'url': '{github_git}/microsoft/onnxruntime.git@0d9030e79888d1d5828730b254fedc53c7b640c1',

README.md

@@ -104,7 +104,7 @@ Currently "cpu", "gpu" and "default" are supported, more devices are to be suppo
 
 **Notes**:
  * For OpenVINO backend, please [install 2021.4 version](https://docs.openvinotoolkit.org/2021.4/openvino_docs_install_guides_installing_openvino_linux.html#install-openvino) and [set the environment variables](https://docs.openvinotoolkit.org/2021.4/openvino_docs_install_guides_installing_openvino_linux.html#set-the-environment-variables) before running the end2end tests.
- * The current implementation of XNNPACK, oneDNN and MLAS backends is mainly for the investigation of WebNN [Operation Level Execution
+ * The current implementation of oneDNN and MLAS backends is mainly for the investigation of WebNN [Operation Level Execution
 ](https://webmachinelearning.github.io/webnn/#usecase-op-level-exec) use case. So only a limited set of tests (such as of conv2d) is expected to pass.
 
 ### Run examples

src/tests/end2end/GemmTests.cpp

@@ -31,10 +31,16 @@ class GemmTests : public WebnnTest {
                   const std::vector<float>& bData,
                   const std::vector<int32_t>& expectedShape,
                   const std::vector<float>& expectedValue,
-                  const Options* options = nullptr) {
+                  const Options* options = nullptr,
+                  bool constantWeight = false) {
         const wnn::GraphBuilder builder = wnn::CreateGraphBuilder(GetContext());
         const wnn::Operand a = utils::BuildInput(builder, "a", aShape);
-        const wnn::Operand b = utils::BuildInput(builder, "b", bShape);
+        wnn::Operand b;
+        if (constantWeight) {
+            b = utils::BuildConstant(builder, bShape, bData.data(), bData.size() * sizeof(float));
+        } else {
+            b = utils::BuildInput(builder, "b", bShape);
+        }
         wnn::GemmOptions gemmOptions = {};
         if (options != nullptr) {
             if (!options->cData.empty()) {
@@ -52,7 +58,11 @@ class GemmTests : public WebnnTest {
         const wnn::Graph graph = utils::Build(builder, {{"c", gemm}});
         ASSERT_TRUE(graph);
         std::vector<float> result(utils::SizeOfShape(expectedShape));
-        utils::Compute(graph, {{"a", aData}, {"b", bData}}, {{"c", result}});
+        if (constantWeight) {
+            utils::Compute(graph, {{"a", aData}}, {{"c", result}});
+        } else {
+            utils::Compute(graph, {{"a", aData}, {"b", bData}}, {{"c", result}});
+        }
         EXPECT_TRUE(utils::CheckValue(result, expectedValue));
     }
 };
@@ -192,6 +202,29 @@ TEST_F(GemmTests, NoBias) {
     TestGemm(inputAShape, inputAData, inputBShape, inputBData, expectedShape, expectedValue);
 }
 
+TEST_F(GemmTests, NoBiasWithConstantWeight) {
+    const std::vector<int32_t> inputAShape = {2, 10};
+    const std::vector<float> inputAData = {
+        0.97596496, 0.47531518, 0.7147315, 0.14236908, 0.06151228, 0.05889508, 0.3534669,
+        0.31915423, 0.61336106, 0.5946216, 0.21969128, 0.7347848,  0.4087221,  0.00412959,
+        0.77303815, 0.6495765,  0.3174799, 0.62841094, 0.7002717,  0.63384914,
+    };
+    const std::vector<int32_t> inputBShape = {10, 3};
+    const std::vector<float> inputBData = {
+        0.51739925, 0.25108355, 0.31373033, 0.6488124,  0.9777175,  0.13308926,
+        0.47903556, 0.23692878, 0.0822504,  0.3080891,  0.51966125, 0.969734,
+        0.6691261,  0.59346807, 0.7651862,  0.48655444, 0.48373327, 0.2799068,
+        0.35760838, 0.19906454, 0.3612888,  0.11448191, 0.19188708, 0.00769753,
+        0.3161914,  0.323555,   0.17573832, 0.79587144, 0.91238266, 0.5517277,
+    };
+    const std::vector<int32_t> expectedShape = {2, 3};
+    const std::vector<float> expectedValue = {
+        2.0995352, 1.8906747, 1.1958704, 2.5321422, 2.6342242, 1.5699927,
+    };
+    TestGemm(inputAShape, inputAData, inputBShape, inputBData, expectedShape, expectedValue,
+             nullptr, true);
+}
+
 TEST_F(GemmTests, ScalarBias) {
     const std::vector<int32_t> inputAShape = {2, 3};
     const std::vector<float> inputAData = {
@@ -213,6 +246,27 @@ TEST_F(GemmTests, ScalarBias) {
              &options);
 }
 
+TEST_F(GemmTests, BiasWithConstantWeight) {
+    const std::vector<int32_t> inputAShape = {2, 3};
+    const std::vector<float> inputAData = {
+        0.41595492, 0.7063231, 0.3784654, 0.3524597, 0.41936764, 0.08190536,
+    };
+    const std::vector<int32_t> inputBShape = {3, 4};
+    const std::vector<float> inputBData = {
+        0.38356313, 0.92939967, 0.06164686, 0.09034675, 0.34704673, 0.9492532,
+        0.7738587,  0.93576515, 0.49937814, 0.38543963, 0.02364575, 0.80216527,
+    };
+    const std::vector<int32_t> expectedShape = {2, 4};
+    const std::vector<float> expectedValue = {
+        3.7336695, 4.3429437, 3.7211857, 4.1421247, 3.4616325, 3.8972316, 3.4881961, 3.6299748,
+    };
+    Options options;
+    options.cShape = {4};
+    options.cData = {3.14, 3.14, 3.14, 3.14};
+    TestGemm(inputAShape, inputAData, inputBShape, inputBData, expectedShape, expectedValue,
+             &options, true);
+}
+
 TEST_F(GemmTests, BroadcastingBias) {
     const std::vector<int32_t> inputAShape = {3, 7};
     const std::vector<float> inputAData = {
@@ -287,3 +341,28 @@ TEST_F(GemmTests, bTranspose) {
     TestGemm(inputAShape, inputAData, inputBShape, inputBData, expectedShape, expectedValue,
              &options);
 }
+
+TEST_F(GemmTests, bTransposeWithConstantWeight) {
+    const std::vector<int32_t> inputAShape = {3, 6};
+    const std::vector<float> inputAData = {
+        0.4520783,  0.25709572, 0.28996432, 0.03766193, 0.0546827,  0.46305302,
+        0.91171485, 0.48380807, 0.09058774, 0.6646215,  0.35773644, 0.03604647,
+        0.21229707, 0.18758385, 0.01589681, 0.9606218,  0.08803706, 0.18099776,
+    };
+    const std::vector<int32_t> inputBShape = {4, 6};
+    const std::vector<float> inputBData = {
+        0.1482661,  0.27676222, 0.10893039, 0.8347901, 0.7146212, 0.7316929,
+        0.97991717, 0.97123116, 0.69798464, 0.8436566, 0.9630883, 0.23252074,
+        0.09898344, 0.08882044, 0.90780985, 0.7116153, 0.5819304, 0.6742051,
+        0.5233705,  0.5594687,  0.963364,   0.1351259, 0.8119938, 0.13756031,
+    };
+    const std::vector<int32_t> expectedShape = {3, 4};
+    const std::vector<float> expectedValue = {
+        0.57909805, 1.0871967, 0.7016311, 0.77297145, 1.1157843, 2.340149,
+        0.92088836, 1.2203549, 1.0823897, 1.3386247,  0.9089607, 0.45756027,
+    };
+    Options options;
+    options.bTranspose = true;
+    TestGemm(inputAShape, inputAData, inputBShape, inputBData, expectedShape, expectedValue,
+             &options, true);
+}

src/tests/end2end/SplitTests.cpp

@@ -54,24 +54,29 @@ class SplitTests : public WebnnTest {
     }
 };
 
-TEST_F(SplitTests, SplitByDefault) {
+TEST_F(SplitTests, SplitEvenByDefault) {
     testSplit({6}, {1, 2, 3, 4, 5, 6}, {3},
               {
                   {{2}, {1, 2}},
                   {{2}, {3, 4}},
                   {{2}, {5, 6}},
               });
+}
 
+TEST_F(SplitTests, SplitByDefault) {
     testSplit({6}, {1, 2, 3, 4, 5, 6}, {2, 4}, {{{2}, {1, 2}}, {{4}, {3, 4, 5, 6}}});
 }
 
-TEST_F(SplitTests, SplitOneDimension) {
+TEST_F(SplitTests, SplitEvenOneDimension) {
     testSplit({2, 6}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, {2},
               {
                   {{2, 3}, {1, 2, 3, 7, 8, 9}},
                   {{2, 3}, {4, 5, 6, 10, 11, 12}},
               },
               1);
+}
+
+TEST_F(SplitTests, SplitOneDimension) {
     testSplit({2, 6}, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, {2, 4},
               {
                   {{2, 2}, {1, 2, 7, 8}},

src/webnn_native/BUILD.gn

@@ -303,6 +303,8 @@ source_set("webnn_native_sources") {
 
   if (webnn_enable_xnnpack) {
     sources += [
+      "xnnpack/BackendXNN.cpp",
+      "xnnpack/BackendXNN.h",
       "xnnpack/ContextXNN.cpp",
       "xnnpack/ContextXNN.h",
       "xnnpack/GraphXNN.cpp",

src/webnn_native/Instance.cpp

@@ -53,6 +53,11 @@ namespace webnn_native {
         BackendConnection* Connect(InstanceBase* instance);
     }
 #endif  // defined(WEBNN_ENABLE_BACKEND_MLAS)
+#if defined(WEBNN_ENABLE_BACKEND_XNNPACK)
+    namespace xnnpack {
+        BackendConnection* Connect(InstanceBase* instance);
+    }
+#endif  // defined(WEBNN_ENABLE_BACKEND_XNNPACK)
 
     namespace {
 
@@ -73,6 +78,9 @@ namespace webnn_native {
 #if defined(WEBNN_ENABLE_BACKEND_MLAS)
             enabledBackends.set(wnn::BackendType::MLAS);
 #endif  // defined(WEBNN_ENABLE_BACKEND_MLAS)
+#if defined(WEBNN_ENABLE_BACKEND_XNNPACK)
+            enabledBackends.set(wnn::BackendType::XNNPACK);
+#endif  // defined(WEBNN_ENABLE_BACKEND_XNNPACK)
             return enabledBackends;
         }
 
@@ -137,6 +145,12 @@ namespace webnn_native {
                 break;
 #endif  // defined(WEBNN_ENABLE_BACKEND_MLAS)
 
+#if defined(WEBNN_ENABLE_BACKEND_XNNPACK)
+            case wnn::BackendType::XNNPACK:
+                Register(xnnpack::Connect(this), wnn::BackendType::XNNPACK);
+                break;
+#endif  // defined(WEBNN_ENABLE_BACKEND_XNNPACK)
+
             default:
                 UNREACHABLE();
         }
@@ -156,6 +170,8 @@ namespace webnn_native {
             return mBackends[wnn::BackendType::OneDNN]->CreateContext(options);
         } else if (mBackends.find(wnn::BackendType::MLAS) != mBackends.end()) {
             return mBackends[wnn::BackendType::MLAS]->CreateContext(options);
+        } else if (mBackends.find(wnn::BackendType::XNNPACK) != mBackends.end()) {
+            return mBackends[wnn::BackendType::XNNPACK]->CreateContext(options);
         }
         UNREACHABLE();
         return nullptr;

src/webnn_native/WebnnNative.cpp

@@ -31,7 +31,7 @@ namespace webnn_native {
     namespace {
 
         void DumpMemoryLeaks() {
-#if defined(_WIN32) && defined(_DEBUG)
+#if 0
             // Send all reports to STDOUT.
             _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE);
             _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDOUT);

src/webnn_native/xnnpack/BackendXNN.cpp

@@ -0,0 +1,56 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "webnn_native/xnnpack/BackendXNN.h"
+
+#include "common/Log.h"
+#include "webnn_native/Instance.h"
+#include "webnn_native/xnnpack/ContextXNN.h"
+
+namespace webnn_native::xnnpack {
+
+    Backend::Backend(InstanceBase* instance)
+        : BackendConnection(instance, wnn::BackendType::XNNPACK) {
+    }
+
+    MaybeError Backend::Initialize() {
+        return {};
+    }
+
+    ContextBase* Backend::CreateContext(ContextOptions const* options) {
+        if (options->devicePreference == wnn::DevicePreference::Gpu) {
+            dawn::ErrorLog() << "XNNPACK backend only supports CPU device.";
+            return nullptr;
+        }
+        Ref<ContextBase> context = AcquireRef(new Context(options));
+        xnn_status status = reinterpret_cast<Context*>(context.Get())->Init();
+        if (status != xnn_status_success) {
+            dawn::ErrorLog() << "Failed to init XNNPACK:" << status;
+            return nullptr;
+        }
+        return context.Detach();
+    }
+
+    BackendConnection* Connect(InstanceBase* instance) {
+        Backend* backend = new Backend(instance);
+
+        if (instance->ConsumedError(backend->Initialize())) {
+            delete backend;
+            return nullptr;
+        }
+
+        return backend;
+    }
+
+}  // namespace webnn_native::xnnpack

src/webnn_native/xnnpack/BackendXNN.h

@@ -0,0 +1,36 @@
+// Copyright 2022 The Dawn Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef WEBNN_NATIVE_XNNPACK_BACKENDXNN_H_
+#define WEBNN_NATIVE_XNNPACK_BACKENDXNN_H_
+
+#include "webnn_native/BackendConnection.h"
+#include "webnn_native/Context.h"
+#include "webnn_native/Error.h"
+
+#include <memory>
+
+namespace webnn_native::xnnpack {
+
+    class Backend : public BackendConnection {
+      public:
+        Backend(InstanceBase* instance);
+
+        MaybeError Initialize();
+        ContextBase* CreateContext(ContextOptions const* options = nullptr) override;
+    };
+
+}  // namespace webnn_native::xnnpack
+
+#endif  // WEBNN_NATIVE_XNNPACK_BACKENDXNN_H_

src/webnn_native/xnnpack/ContextXNN.cpp

@@ -22,17 +22,7 @@
 
 namespace webnn_native::xnnpack {
 
-    ContextBase* Create() {
-        Ref<ContextBase> context = AcquireRef(new Context());
-        xnn_status status = reinterpret_cast<Context*>(context.Get())->Init();
-        if (status != xnn_status_success) {
-            dawn::ErrorLog() << "Failed to init XNNPack:" << status;
-            return nullptr;
-        }
-        return context.Detach();
-    }
-
-    Context::Context() {
+    Context::Context(ContextOptions const* options) {
     }
 
     Context::~Context() {

src/webnn_native/xnnpack/ContextXNN.h

@@ -23,7 +23,7 @@ namespace webnn_native::xnnpack {
 
     class Context : public ContextBase {
       public:
-        Context();
+        explicit Context(ContextOptions const* options);
         ~Context() override;
 
         xnn_status Init();