merge

.gitignore

@@ -107,3 +107,4 @@ paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen/*
 paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen_tmp/*
 paddle/fluid/pybind/static_op_function.*
 paddle/fluid/pybind/ops_api.cc
+paddle/fluid/pir/drr/src/*_op_factory_generated.*

cmake/external/warpctc.cmake

@@ -26,12 +26,23 @@ set(WARPCTC_TAG bdc2b4550453e0ef2d3b5190f9c6103a84eff184)
 set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warpctc)
 set(WARPCTC_PATCH_COMMAND "")
 set(WARPCTC_CCBIN_OPTION "")
+if(WIN32)
+  set(WARPCTC_PATCH_CUDA_COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      ${PADDLE_SOURCE_DIR}/patches/warpctc/CMakeLists.txt.cuda.patch
+      "<SOURCE_DIR>/")
+else()
+  set(WARPCTC_PATCH_CUDA_COMMAND
+      patch -d ${SOURCE_DIR} <
+      ${PADDLE_SOURCE_DIR}/patches/warpctc/CMakeLists.txt.cuda.patch)
+endif()
+
 if(NOT WIN32 AND WITH_GPU)
   if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
                                                   VERSION_GREATER 12.0)
     file(TO_NATIVE_PATH
          ${PADDLE_SOURCE_DIR}/patches/warpctc/CMakeLists.txt.patch native_src)
-    set(WARPCTC_PATCH_COMMAND patch -d ${SOURCE_DIR} < ${native_src})
+    set(WARPCTC_PATCH_COMMAND patch -d ${SOURCE_DIR} < ${native_src} &&)
     set(WARPCTC_CCBIN_OPTION -DCCBIN_COMPILER=${CCBIN_COMPILER})
   endif()
 endif()
@@ -87,7 +98,7 @@ ExternalProject_Add(
   SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPCTC_PREFIX_DIR}
   UPDATE_COMMAND ""
-  PATCH_COMMAND ${WARPCTC_PATCH_COMMAND}
+  PATCH_COMMAND ${WARPCTC_PATCH_COMMAND} ${WARPCTC_PATCH_CUDA_COMMAND}
   #BUILD_ALWAYS    1
   CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
              -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
@@ -101,6 +112,7 @@ ExternalProject_Add(
              -DWITH_GPU=${WITH_GPU}
              -DWITH_ROCM=${WITH_ROCM}
              -DWITH_OMP=${USE_OMP}
+             -DNVCC_FLAGS_EXTRA=${NVCC_FLAGS_EXTRA}
              -DWITH_TORCH=OFF
              -DCMAKE_DISABLE_FIND_PACKAGE_Torch=ON
              -DBUILD_SHARED=ON

cmake/external/warprnnt.cmake

@@ -24,6 +24,17 @@ set(WARPRNNT_TAG 7ea6bfe748779c245a0fcaa5dd9383826273eff2)
 set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warprnnt)
 set(WARPRNNT_PATCH_COMMAND "")
 set(WARPRNNT_CCBIN_OPTION "")
+if(WIN32)
+  set(WARPCTC_PATCH_CUDA_COMMAND
+      ${CMAKE_COMMAND} -E copy_if_different
+      ${PADDLE_SOURCE_DIR}/patches/warprnnt/CMakeLists.txt.cuda.patch
+      "<SOURCE_DIR>/")
+else()
+  set(WARPCTC_PATCH_CUDA_COMMAND
+      patch -d ${SOURCE_DIR} <
+      ${PADDLE_SOURCE_DIR}/patches/warprnnt/CMakeLists.txt.cuda.patch)
+endif()
+
 if(NOT WIN32 AND WITH_GPU)
   if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
                                                   VERSION_GREATER 12.0)
@@ -85,7 +96,7 @@ ExternalProject_Add(
   SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPRNNT_PREFIX_DIR}
   UPDATE_COMMAND ""
-  PATCH_COMMAND ""
+  PATCH_COMMAND ${WARPCTC_PATCH_CUDA_COMMAND}
   #BUILD_ALWAYS    1
   CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
              -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
@@ -99,6 +110,7 @@ ExternalProject_Add(
              -DWITH_GPU=${WITH_GPU}
              -DWITH_ROCM=${WITH_ROCM}
              -DWITH_OMP=${USE_OMP}
+             -DNVCC_FLAGS_EXTRA=${NVCC_FLAGS_EXTRA}
              -DBUILD_SHARED=ON
              -DBUILD_TESTS=OFF
              -DCMAKE_POSITION_INDEPENDENT_CODE=ON

paddle/cinn/adt/adapter_tensor.h

@@ -15,7 +15,7 @@
 #pragma once
 
 #include "paddle/cinn/adt/adt.h"
-#include "paddle/pir/core/value.h"
+#include "paddle/pir/include/core/value.h"
 
 namespace cinn::adt::adapter {
 

paddle/cinn/adt/dim_expr.h

@@ -19,7 +19,7 @@
 #include "paddle/cinn/adt/arithmetic.h"
 #include "paddle/cinn/adt/logical.h"
 #include "paddle/cinn/adt/symbolic_dim.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr.h"
 
 namespace cinn::adt {
 

paddle/cinn/adt/generate_map_expr.cc

@@ -30,8 +30,8 @@
 #include "paddle/cinn/hlir/framework/pir/group.h"
 #include "paddle/cinn/hlir/framework/pir/utils.h"
 #include "paddle/cinn/runtime/flags.h"
-#include "paddle/pir/core/operation.h"
-#include "paddle/pir/core/value.h"
+#include "paddle/pir/include/core/operation.h"
+#include "paddle/pir/include/core/value.h"
 
 #include "glog/logging.h"
 

paddle/cinn/adt/map_expr_ctx.h

@@ -20,8 +20,8 @@
 #include "paddle/cinn/adt/map_expr.h"
 #include "paddle/cinn/ir/lowered_func.h"
 #include "paddle/cinn/ir/utils/ir_copy.h"
-#include "paddle/pir/core/operation.h"
-#include "paddle/pir/dialect/shape/ir/shape_op.h"
+#include "paddle/pir/include/core/operation.h"
+#include "paddle/pir/include/dialect/shape/ir/shape_op.h"
 
 namespace cinn::adt {
 

paddle/cinn/adt/print_utils/print_equations.cc

@@ -21,7 +21,7 @@
 #include "paddle/cinn/adt/equation_graph.h"
 #include "paddle/cinn/adt/print_utils/print_dim_expr.h"
 #include "paddle/cinn/hlir/framework/pir/utils.h"
-#include "paddle/pir/core/operation.h"
+#include "paddle/pir/include/core/operation.h"
 
 namespace cinn::adt {
 

paddle/cinn/adt/simplify_value.cc

@@ -21,7 +21,7 @@
 #include "paddle/cinn/adt/index_expr_infer_context.h"
 #include "paddle/cinn/adt/match.h"
 #include "paddle/cinn/adt/simplify_value.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr_simplify.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr_simplify.h"
 
 namespace cinn::adt {
 

paddle/cinn/backends/codegen_c.cc

@@ -300,12 +300,13 @@ void CodeGenC::Visit(const ir::Block *op) {
 
   IncIndent();
 
-  for (int i = 0; i < op->stmts.size() - 1; i++) {
-    DoIndent();
-    IrPrinter::Visit(op->stmts[i]);
-    str_ += ";\n";
-  }
+  // Note: size_t (0 - 1) = 18446744073709551615
   if (op->stmts.size() >= 1) {
+    for (int i = 0; i < op->stmts.size() - 1; i++) {
+      DoIndent();
+      IrPrinter::Visit(op->stmts[i]);
+      str_ += ";\n";
+    }
     DoIndent();
     IrPrinter::Visit(op->stmts.back());
     str_ += ";";

paddle/cinn/common/broadcast_tree.cc

@@ -18,7 +18,7 @@
 #include <unordered_map>
 
 #include "paddle/cinn/common/dim_expr_util.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr_simplify.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr_simplify.h"
 
 namespace cinn::common {
 

paddle/cinn/common/broadcast_tree.h

@@ -15,7 +15,7 @@
 #pragma once
 
 #include "paddle/cinn/adt/tree.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr.h"
 
 namespace cinn::common {
 

paddle/cinn/common/dim_expr_converter.h

@@ -15,7 +15,7 @@
 #pragma once
 
 #include "paddle/cinn/ir/ir.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr.h"
 
 namespace cinn::common {
 

paddle/cinn/common/dim_expr_util.h

@@ -16,8 +16,8 @@
 
 #include <optional>
 #include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h"
-#include "paddle/pir/core/builder.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr.h"
+#include "paddle/pir/include/core/builder.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr.h"
 
 namespace cinn::common {
 

paddle/cinn/hlir/dialect/operator/ir/attribute_storage.h

@@ -20,8 +20,8 @@
 #include <vector>
 #include "paddle/cinn/hlir/framework/op.h"
 #include "paddle/cinn/hlir/framework/pir/utils.h"
-#include "paddle/pir/core/attribute_base.h"
-#include "paddle/pir/core/operation.h"
+#include "paddle/pir/include/core/attribute_base.h"
+#include "paddle/pir/include/core/operation.h"
 
 namespace cinn {
 namespace dialect {
@@ -47,6 +47,11 @@ struct GroupInfo {
   std::vector<::pir::Operation*> ops;
   std::vector<std::string> input_names;
   std::vector<std::string> output_names;
+  std::unordered_map<::pir::Operation*,
+                     std::vector<cinn::hlir::framework::pir::ScheduleInfoNode>>
+      alignment_schedule_info;
+  std::vector<int64_t> reduce_axis;
+  std::vector<int64_t> loop_ranges;
 
  private:
   void Initialize() {
@@ -64,7 +69,18 @@ struct GroupInfoAttributeStorage : public pir::AttributeStorage {
   }
 
   static std::size_t HashValue(const ParamKey& key) {
-    return std::hash<std::string>{}(key.group_id);
+    size_t hash_value = std::hash<std::string>{}(key.group_id);
+
+    for (auto d : key.loop_ranges) {
+      hash_value =
+          pir::detail::hash_combine(hash_value, std::hash<int64_t>()(d));
+    }
+
+    for (auto d : key.reduce_axis) {
+      hash_value =
+          pir::detail::hash_combine(hash_value, std::hash<int64_t>()(d));
+    }
+    return hash_value;
   }
 
   bool operator==(const ParamKey& key) const {

paddle/cinn/hlir/dialect/operator/ir/generate_shape_util.cc

@@ -14,8 +14,8 @@
 
 #include "paddle/cinn/hlir/dialect/operator/ir/generate_shape_util.h"
 #include <unordered_set>
-#include "paddle/pir/core/builder.h"
-#include "paddle/pir/core/builtin_attribute.h"
+#include "paddle/pir/include/core/builder.h"
+#include "paddle/pir/include/core/builtin_attribute.h"
 
 namespace cinn::dialect {
 using namespace symbol;  // NOLINT

paddle/cinn/hlir/dialect/operator/ir/generate_shape_util.h

@@ -18,8 +18,8 @@
 #include <optional>
 #include <vector>
 #include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h"
-#include "paddle/pir/core/builder.h"
-#include "paddle/pir/dialect/shape/utils/shape_or_data_expr.h"
+#include "paddle/pir/include/core/builder.h"
+#include "paddle/pir/include/dialect/shape/utils/shape_or_data_expr.h"
 
 namespace cinn::dialect {
 

paddle/cinn/hlir/dialect/operator/ir/manual_op.cc

@@ -17,21 +17,23 @@
 #include <vector>
 #include "glog/logging.h"
 #include "paddle/cinn/hlir/dialect/operator/ir/generate_shape_util.h"
+#include "paddle/cinn/hlir/dialect/operator/ir/op_attribute.h"
 #include "paddle/common/ddim.h"
 #include "paddle/common/enforce.h"
 #include "paddle/fluid/pir/dialect/operator/ir/ir_meta_tensor.h"
 #include "paddle/fluid/pir/dialect/operator/ir/ir_tensor.h"
 #include "paddle/fluid/pir/dialect/operator/ir/op_type.h"
 #include "paddle/fluid/pir/dialect/operator/utils/utils.h"
-#include "paddle/pir/core/builtin_type.h"
-#include "paddle/pir/core/op_base.h"
-#include "paddle/pir/dialect/control_flow/ir/cf_op.h"
-#include "paddle/pir/dialect/shape/utils/dim_expr_simplify.h"
+#include "paddle/pir/include/core/builtin_type.h"
+#include "paddle/pir/include/core/op_base.h"
+#include "paddle/pir/include/dialect/control_flow/ir/cf_op.h"
+#include "paddle/pir/include/dialect/shape/utils/dim_expr_simplify.h"
 
 namespace cinn {
 namespace dialect {
 
 const char* GroupOp::attributes_name[GroupOp::attributes_num] = {"group_info"};
+const char* FusionOp::attributes_name[GroupOp::attributes_num] = {"group_info"};
 const char* ConcatOp::attributes_name[ConcatOp::attributes_num] = {"axis"};
 const char* SplitOp::attributes_name[SplitOp::attributes_num] = {
     "num_or_sections", "axis"};
@@ -43,12 +45,25 @@ void GroupOp::Build(pir::Builder& builder,
   argument.output_types = output_types;
 }
 
+void GroupOp::Build(pir::Builder& builder,             // NOLINT
+                    pir::OperationArgument& argument,  // NOLINT
+                    const std::vector<pir::Type>& output_types,
+                    const cinn::dialect::GroupInfo& group_info) {
+  argument.AddRegion(nullptr);
+  argument.output_types = output_types;
+
+  argument.AddAttribute("group_info",
+                        cinn::dialect::GroupInfoAttribute::get(
+                            pir::IrContext::Instance(), group_info));
+}
+
 void GroupOp::Build(pir::Builder& builder,             // NOLINT
                     pir::OperationArgument& argument,  // NOLINT
                     std::unique_ptr<pir::Block>&& block) {
   VLOG(4) << "Start build GroupOp";
   if (block && !block->empty()) {
-    IR_ENFORCE(block->back().isa<pir::YieldOp>());
+    // IR_ENFORCE(block->back().isa<pir::YieldOp>());
+    PADDLE_ENFORCE_EQ(block->back().isa<pir::YieldOp>(), true);
     auto& op = block->back();
     for (size_t i = 0; i < op.num_operands(); ++i) {
       argument.AddOutput(op.operand(i).type());
@@ -96,6 +111,18 @@ void FusionOp::Build(pir::Builder& builder,
   argument.output_types = output_types;
 }
 
+void FusionOp::Build(pir::Builder& builder,             // NOLINT
+                     pir::OperationArgument& argument,  // NOLINT
+                     const std::vector<pir::Type>& output_types,
+                     const cinn::dialect::GroupInfo& group_info) {
+  argument.AddRegion(nullptr);
+  argument.output_types = output_types;
+
+  argument.AddAttribute("group_info",
+                        cinn::dialect::GroupInfoAttribute::get(
+                            pir::IrContext::Instance(), group_info));
+}
+
 pir::Block* FusionOp::block() {
   pir::Region& region = (*this)->region(0);
   if (region.empty()) region.emplace_back();
@@ -143,7 +170,10 @@ void ConcatOp::Build(pir::Builder& builder,             // NOLINT
   argument.inputs = inputs;
   std::vector<pir::Type> inputs_type(inputs.size());
 
-  IR_ENFORCE(inputs.size() > 0);
+  PADDLE_ENFORCE_GT(inputs.size(),
+                    0,
+                    phi::errors::InvalidArgument(
+                        "input size [%d] is less than 0", inputs.size()));
 
   auto first_ele =
       inputs[0].type().dyn_cast<paddle::dialect::DenseTensorType>();