【Hackathon 6th Fundable Projects 3 No.13】barrier

paddle/phi/kernels/cpu/barrier_kernel.cc

@@ -0,0 +1,44 @@
+// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/core/kernel_registry.h"
+
+#if defined(PADDLE_WITH_GLOO)
+#include <gloo/barrier.h>
+#include "paddle/phi/core/distributed/gloo_comm_context.h"
+#endif
+
+namespace phi {
+
+template <typename T, typename Context>
+void BarrierKernel(const Context& dev_ctx,
+                   const DenseTensor& x_in,
+                   DenseTensor* out) {
+#if defined(PADDLE_WITH_GLOO)
+  auto comm_ctx =
+      static_cast<distributed::GlooCommContext*>(dev_ctx.GetCommContext());
+  PADDLE_ENFORCE_NE(
+      comm_ctx,
+      nullptr,
+      errors::Unavailable("GlooCommContext is nullptr, collective op should "
+                          "has ring_id attr."));
+#else
+  PADDLE_THROW(phi::errors::Unavailable(
+      "PaddlePaddle should compile with GLOO by setting WITH_GLOO=ON"));
+#endif
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(barrier, CPU, ALL_LAYOUT, phi::BarrierKernel, int) {}

paddle/phi/kernels/gpu/barrier_kernel.cu

@@ -0,0 +1,57 @@
+// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "glog/logging.h"
+
+#include "paddle/phi/backends/gpu/gpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+
+#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
+#include "paddle/common/flags.h"
+#include "paddle/phi/backends/gpu/gpu_helper.h"
+#include "paddle/phi/core/distributed/nccl_comm_context.h"
+#include "paddle/phi/core/platform/collective_helper.h"
+COMMON_DECLARE_bool(dynamic_static_unified_comm);
+#endif
+
+namespace phi {
+
+template <typename T, typename Context>
+void BarrierKernel(const Context& dev_ctx,
+                   const DenseTensor& x_in,
+                   DenseTensor* out) {
+#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
+
+  if (FLAGS_dynamic_static_unified_comm) {
+    auto comm_ctx =
+        static_cast<distributed::NCCLCommContext*>(dev_ctx.GetCommContext());
+    PADDLE_ENFORCE_NOT_NULL(
+        comm_ctx,
+        phi::errors::Unavailable(
+            "NCCLCommContext is nullptr, collective op should "
+            "has ring_id attr."));
+    auto stream = comm_ctx->GetStream();
+    ncclRedOp_t nccl_red_type = ncclSum;
+    comm_ctx->AllReduce(out, x_in, nccl_red_type, stream);
+    phi::backends::gpu::GpuStreamSync(stream);
+  }
+#else
+  PADDLE_THROW(
+      phi::errors::Unavailable("PaddlePaddle should compile with NCCL."));
+#endif
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(barrier, GPU, ALL_LAYOUT, phi::BarrierKernel, int) {}

paddle/phi/ops/yaml/inconsistent/static_ops.yaml

@@ -76,12 +76,6 @@
   interfaces : paddle::dialect::InferSymbolicShapeInterface
   traits : paddle::dialect::ForwardOnlyTrait
 
-- op : barrier
-  args : (Tensor x, int ring_id=0)
-  output : Tensor(out)
-  kernel :
-    func : barrier
-
 - op : batch_norm
   args : (Tensor x, Tensor mean, Tensor variance, Tensor scale, Tensor bias, bool is_test, float momentum, float epsilon, str data_format, bool use_global_stats, bool trainable_statistics)
   output : Tensor(out), Tensor(mean_out), Tensor(variance_out), Tensor(saved_mean), Tensor(saved_variance), Tensor(reserve_space)

paddle/phi/ops/yaml/ops.yaml

@@ -457,6 +457,17 @@
     data_type : param
   inplace : (in_sum_1 -> out_sum_1), (in_sum_2 -> out_sum_2), (in_sum_3 -> out_sum_3), (in_num_accumulates -> out_num_accumulates), (in_old_num_accumulates -> out_old_num_accumulates), (in_num_updates -> out_num_updates)
 
+- op : barrier
+  args : (Tensor x, int ring_id = 0)
+  output : Tensor(out)
+  infer_meta :
+    func : UnchangedInferMeta
+    param : [x]
+  kernel :
+    func : barrier
+    data_type : x
+  traits : paddle::dialect::ForwardOnlyTrait
+
 - op : batch_fc
   args : (Tensor input, Tensor w, Tensor bias)
   output : Tensor(out)