Add Code Generation for operators, op makers and argument mapping functions

paddle/fluid/eager/auto_code_generator/final_state_generator/CMakeLists.txt

@@ -1,5 +1,5 @@
-set(api_yaml_path "${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_api.yaml")
-set(backward_yaml_path "${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/backward.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_bw_api.yaml")
+set(api_yaml_path "${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/new_api.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_api.yaml")
+set(backward_yaml_path "${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/backward.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/new_backward.yaml,${PADDLE_SOURCE_DIR}/python/paddle/utils/code_gen/sparse_bw_api.yaml")
 set(tmp_forwards_cc_path "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/eager_generated/forwards/tmp_dygraph_functions.cc")
 set(tmp_forwards_h_path "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/eager_generated/forwards/tmp_dygraph_functions.h")
 set(tmp_nodes_cc_path "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/eager_generated/backwards/tmp_nodes.cc")

paddle/fluid/eager/auto_code_generator/final_state_generator/codegen_utils.py

@@ -63,22 +63,24 @@ def AssertMessage(lhs_str, rhs_str):
 
 def ReadFwdFile(filepath):
     f = open(filepath, 'r')
+    # empty file loaded by yaml is None
     contents = yaml.load(f, Loader=yaml.FullLoader)
     f.close()
-    return contents
+    return contents if contents is not None else []
 
 
 def ReadBwdFile(filepath):
     f = open(filepath, 'r')
     contents = yaml.load(f, Loader=yaml.FullLoader)
     ret = {}
-    for content in contents:
-        assert 'backward_api' in content.keys(), AssertMessage('backward_api',
-                                                               content.keys())
-        if 'backward_api' in content.keys():
-            api_name = content['backward_api']
-
-        ret[api_name] = content
+    if contents is not None:
+        for content in contents:
+            assert 'backward_api' in content.keys(), AssertMessage(
+                'backward_api', content.keys())
+            if 'backward_api' in content.keys():
+                api_name = content['backward_api']
+
+            ret[api_name] = content
     f.close()
     return ret
 
@@ -207,6 +209,8 @@ def ParseYamlArgs(string):
 
         assert arg_type in yaml_types_mapping.keys(
         ), f"The argument type {arg_type} in yaml config is not supported in yaml_types_mapping."
+        if arg_type in ["DataType", "DataLayout"] and default_value is not None:
+            default_value = f"paddle::experimental::{default_value}"
         arg_type = yaml_types_mapping[arg_type]
 
         arg_name = RemoveSpecialSymbolsInName(arg_name)

paddle/phi/api/lib/CMakeLists.txt

@@ -13,6 +13,7 @@ set(api_gen_base ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api_base.py)
 # forward api file
 set(api_gen_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api_gen.py)
 set(api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml)
+set(new_api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/new_api.yaml)
 set(api_header_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/include/api.h)
 set(api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/api.cc)
 set(api_header_file_tmp ${api_header_file}.tmp)
@@ -21,6 +22,7 @@ set(api_source_file_tmp ${api_source_file}.tmp)
 # backward api file
 set(bw_api_gen_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/backward_api_gen.py)
 set(bw_api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/backward.yaml)
+set(new_bw_api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/new_backward.yaml)
 set(bw_api_header_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/backward/backward_api.h)
 set(bw_api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/backward_api.cc)
 set(bw_api_header_file_tmp ${bw_api_header_file}.tmp)
@@ -59,20 +61,113 @@ set(strings_api_source_file_tmp ${strings_api_source_file}.tmp)
 
 # wrapped infermeta file
 set(wrapped_infermeta_gen_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/wrapped_infermeta_gen.py)
-set(api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml)
 set(wrapped_infermeta_header_file ${CMAKE_SOURCE_DIR}/paddle/phi/infermeta/generated.h)
 set(wrapped_infermeta_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/infermeta/generated.cc)
 
 if (NOT PYTHON_EXECUTABLE)
   find_package(PythonInterp REQUIRED)
 endif()
 
+# install extra dependencies
+execute_process(
+  COMMAND ${PYTHON_EXECUTABLE} -m pip install -U pyyaml jinja2
+)
+
+# parse apis
+set(parsed_api_dir ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/parsed_apis)
+set(generated_op_path ${CMAKE_SOURCE_DIR}/paddle/fluid/operators/generated_op.cc)
+set(generated_argument_mapping_path ${CMAKE_SOURCE_DIR}/paddle/phi/ops/compat/generated_sig.cc)
+message("parse api yamls: 
+- ${api_yaml_file}
+- ${new_api_yaml_file}
+- ${bw_api_yaml_file}
+- ${new_bw_api_yaml_file}")
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen
+  COMMAND ${CMAKE_COMMAND} -E make_directory ${parsed_api_dir}
+  COMMAND ${PYTHON_EXECUTABLE} parse_api.py 
+      --api_yaml_path ./api.yaml 
+      --output_path ./parsed_apis/api.parsed.yaml
+  COMMAND ${PYTHON_EXECUTABLE} parse_api.py 
+      --api_yaml_path ./new_api.yaml 
+      --output_path ./parsed_apis/new_api.parsed.yaml
+  COMMAND ${PYTHON_EXECUTABLE} parse_api.py
+      --api_yaml_path ./backward.yaml
+      --output_path ./parsed_apis/backward_api.parsed.yaml
+      --backward
+  COMMAND ${PYTHON_EXECUTABLE} parse_api.py
+      --api_yaml_path ./new_backward.yaml
+      --output_path ./parsed_apis/new_backward_api.parsed.yaml
+      --backward
+  RESULTS_VARIABLE _results
+)
+foreach(_result in ${_results})
+  if (${_result})
+    message(FATAL_ERROR "api yaml parsing failed, exiting.")
+  endif()
+endforeach()
+
+# validation of api yamls
+message("validate api yaml:
+- ${parsed_api_dir}/new_api.parsed.yaml
+- ${parsed_api_dir}/new_backward_api.parsed.yaml")
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen
+  COMMAND ${PYTHON_EXECUTABLE} cross_validate.py
+      --forward_yaml_paths ./parsed_apis/api.parsed.yaml ./parsed_apis/new_api.parsed.yaml 
+      --backward_yaml_paths ./parsed_apis/backward_api.parsed.yaml ./parsed_apis/new_backward_api.parsed.yaml
+  RESULT_VARIABLE _result
+)
+if (${_result}) 
+    message(FATAL_ERROR "api validation failed, exiting." )
+endif()
+
+# code generation for op, op makers, and argument mapping functions
+message("create or remove auto-geneated operators: ${generated_op_path}.tmp
+create or remove auto-geneated argument mappings: ${generated_argument_mapping_path}.tmp")
+execute_process(
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen
+  COMMAND ${PYTHON_EXECUTABLE} generate_op.py
+      --api_yaml_path ./parsed_apis/new_api.parsed.yaml
+      --backward_api_yaml_path ./parsed_apis/new_backward_api.parsed.yaml
+      --output_op_path "${generated_op_path}.tmp"
+      --output_arg_map_path "${generated_argument_mapping_path}.tmp"
+  RESULT_VARIABLE _result
+)
+if (${_result})
+    message(FATAL_ERROR "operator codegen failed, exiting." )
+endif()
+
+
+if(EXISTS "${generated_op_path}.tmp" AND EXISTS "${generated_op_path}")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${generated_op_path}.tmp" "${generated_op_path}")
+  message("copy if different ${generated_op_path}.tmp ${generated_op_path}")
+elseif(EXISTS "${generated_op_path}.tmp")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_op_path}.tmp" "${generated_op_path}")
+  message("copy ${generated_op_path}.tmp ${generated_op_path}")
+else()
+  execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f "${generated_op_path}")
+  message("remove ${generated_op_path}")
+endif()
+
+
+if(EXISTS "${generated_argument_mapping_path}.tmp" AND EXISTS "${generated_argument_mapping_path}")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${generated_argument_mapping_path}.tmp" "${generated_argument_mapping_path}")
+  message("copy if different ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}")
+elseif(EXISTS "${generated_argument_mapping_path}.tmp")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_argument_mapping_path}.tmp" "${generated_argument_mapping_path}")
+  message("copy ${generated_argument_mapping_path}.tmp ${generated_argument_mapping_path}")
+else()
+  execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f "${generated_argument_mapping_path}")
+  message("remove ${generated_argument_mapping_path}")
+endif()
+
 # generate forward api
 add_custom_command(
   OUTPUT ${api_header_file} ${api_source_file}
   COMMAND ${PYTHON_EXECUTABLE} -m pip install pyyaml
   COMMAND ${PYTHON_EXECUTABLE} ${api_gen_file}
-                 --api_yaml_path ${api_yaml_file}
+                 --api_yaml_path ${api_yaml_file} ${new_api_yaml_file}
                  --api_header_path ${api_header_file_tmp}
                  --api_header_path ${api_header_file_tmp}
                  --api_source_path ${api_source_file_tmp}
@@ -86,7 +181,7 @@ add_custom_command(
 add_custom_command(
   OUTPUT ${bw_api_header_file} ${bw_api_source_file} ${bw_api_header_file_tmp} ${bw_api_source_file_tmp}
   COMMAND ${PYTHON_EXECUTABLE} ${bw_api_gen_file}
-                 --backward_yaml_path ${bw_api_yaml_file}
+                 --backward_yaml_path ${bw_api_yaml_file} ${new_bw_api_yaml_file}
                  --backward_header_path ${bw_api_header_file_tmp}
                  --backward_source_path ${bw_api_source_file_tmp}
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${bw_api_header_file_tmp} ${bw_api_header_file}
@@ -138,7 +233,7 @@ add_custom_command(
 add_custom_command(
   OUTPUT ${dygraph_api_header_file} ${dygraph_api_source_file}
   COMMAND ${PYTHON_EXECUTABLE} ${im_api_gen_file}
-                 --api_yaml_path ${api_yaml_file}
+                 --api_yaml_path ${api_yaml_file} ${new_api_yaml_file}
                  --sparse_api_yaml_path ${sparse_api_yaml_file}
                  --dygraph_api_header_path ${dygraph_api_header_file_tmp}
                  --dygraph_api_source_path ${dygraph_api_source_file_tmp}
@@ -151,7 +246,7 @@ add_custom_command(
 add_custom_command(
   OUTPUT ${wrapped_infermeta_header_file} ${wrapped_infermeta_source_file}
   COMMAND ${PYTHON_EXECUTABLE} ${wrapped_infermeta_gen_file}
-                 --api_yaml_path ${api_yaml_file}
+                 --api_yaml_path ${api_yaml_file} ${new_api_yaml_file}
                  --wrapped_infermeta_header_path ${wrapped_infermeta_header_file}
                  --wrapped_infermeta_source_path ${wrapped_infermeta_source_file}
   DEPENDS ${api_yaml_file} ${wrapped_infermeta_gen_file} ${api_gen_base}

paddle/phi/common/scalar.h

@@ -27,6 +27,7 @@ template <typename T>
 class ScalarBase {
  public:
   // Constructor support implicit
+  ScalarBase() : ScalarBase(0) {}
   ScalarBase(double val) : dtype_(DataType::FLOAT64) {  // NOLINT
     data_.f64 = val;
   }

python/paddle/utils/code_gen/api.yaml

@@ -812,7 +812,7 @@
     skip_transform : x
 
 - api : gather
-  args : (Tensor x, Tensor index, Scalar axis=0)
+  args : (Tensor x, Tensor index, Scalar(int) axis=0)
   output : Tensor(out)
   infer_meta :
     func : GatherInferMeta
@@ -2020,7 +2020,7 @@
   backward : subtract_grad
 
 - api : sum
-  args : (Tensor x, int64_t[] dims={}, DataType out_dtype=paddle::experimental::DataType::UNDEFINED, bool keep_dim=false)
+  args : (Tensor x, int64_t[] dims={}, DataType out_dtype=DataType::UNDEFINED, bool keep_dim=false)
   output : Tensor(out)
   infer_meta :
     func : SumInferMeta

python/paddle/utils/code_gen/api_gen.py

@@ -222,9 +222,14 @@ def api_namespace():
 
 
 def generate_api(api_yaml_path, header_file_path, source_file_path):
+    apis = []
+
+    for each_api_yaml in api_yaml_path:
+        with open(each_api_yaml, 'r') as f:
+            api_list = yaml.load(f, Loader=yaml.FullLoader)
+            if api_list:
+                apis.extend(api_list)
 
-    with open(api_yaml_path, 'r') as f:
-        apis = yaml.load(f, Loader=yaml.FullLoader)
     header_file = open(header_file_path, 'w')
     source_file = open(source_file_path, 'w')
 
@@ -259,6 +264,7 @@ def main():
     parser.add_argument(
         '--api_yaml_path',
         help='path to api yaml file',
+        nargs='+',
         default='python/paddle/utils/code_gen/api.yaml')
 
     parser.add_argument(

python/paddle/utils/code_gen/backward.yaml

@@ -281,7 +281,6 @@
     param : [grad_x_grad, axis]
   kernel :
     func : concat
-  no_need_buffer : x
 
 - backward_api : concat_grad
   forward : concat (Tensor[] x, Scalar axis) -> Tensor(out)
@@ -507,7 +506,6 @@
     param : [out_grad]
   kernel :
     func : dropout_grad
-  optional : seed_tensor
 
 - backward_api : eigh_grad
   forward : eigh (Tensor x, str uplo) -> Tensor(out_w), Tensor(out_v)
@@ -648,7 +646,6 @@
     data_type: out_grad
     backend: out_grad
     layout: out_grad
-  no_need_buffer : x
 
 - backward_api : flip_grad
   forward : flip (Tensor x, int[] axis) -> Tensor(out)
@@ -866,7 +863,6 @@
     param : [out_grad]
   kernel :
     func : label_smooth_grad
-  optional : prior_dist
 
 - backward_api : layer_norm_grad
   forward : layer_norm (Tensor x, Tensor scale, Tensor bias, float epsilon, int begin_norm_axis, bool is_test) -> Tensor(out), Tensor(mean), Tensor(variance)
@@ -1483,7 +1479,7 @@
   no_need_buffer : grad_out
 
 - backward_api : reshape_grad
-  forward : reshape_with_xshape (Tensor x, IntArray shape) -> Tensor(out), Tensor(xshape)
+  forward : reshape (Tensor x, IntArray shape) -> Tensor(out), Tensor(xshape)
   args : (Tensor xshape, Tensor out_grad)
   output : Tensor(x_grad)
   infer_meta :
@@ -1814,7 +1810,7 @@
   backward : sum_triple_grad
 
 - backward_api : sum_grad
-  forward : sum (Tensor x, int64_t[] dims={}, DataType out_dtype=paddle::experimental::DataType::UNDEFINED, bool keep_dim=false) -> Tensor(out)
+  forward : sum (Tensor x, int64_t[] dims={}, DataType out_dtype=DataType::UNDEFINED, bool keep_dim=false) -> Tensor(out)
   args : (Tensor x, Tensor out_grad, int64_t[] dims, bool keep_dim, bool reduce_all=false)
   output : Tensor(x_grad)
   infer_meta :
@@ -1830,7 +1826,6 @@
   args : (Tensor grad_grad_x, Tensor grad_grad_out_grad, int64_t[] dims={}, bool keep_dim=false, bool reduce_all=false)
   output : Tensor(grad_grad_x_grad)
   invoke : sum_grad(grad_grad_x, grad_grad_out_grad, dims, keep_dim, reduce_all, grad_grad_x_grad)
-  no_need_buffer : x
 
 - backward_api : swish_grad
   forward : swish (Tensor x, float beta=1.0) -> Tensor(out)

python/paddle/utils/code_gen/backward_api_gen.py

@@ -237,8 +237,13 @@ def backward_api_namespace():
 def generate_backward_api(backward_yaml_path, header_file_path,
                           source_file_path):
 
-    with open(backward_yaml_path, 'r') as f:
-        bw_apis = yaml.load(f, Loader=yaml.FullLoader)
+    bw_apis = []
+    for each_api_yaml in backward_yaml_path:
+        with open(each_api_yaml, 'r') as f:
+            api_list = yaml.load(f, Loader=yaml.FullLoader)
+            if api_list:
+                bw_apis.extend(api_list)
+
     header_file = open(header_file_path, 'w')
     source_file = open(source_file_path, 'w')
 
@@ -270,6 +275,7 @@ def main():
     parser.add_argument(
         '--backward_yaml_path',
         help='path to backward yaml file',
+        nargs='+',
         default='python/paddle/utils/code_gen/backward.yaml')
     parser.add_argument(
         '--backward_header_path',

python/paddle/utils/code_gen/cross_validate.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from itertools import chain
+from pathlib import Path
+
+import yaml
+from parse_utils import cross_validate, to_named_dict
+
+
+def main(forward_api_yaml_paths, backward_api_yaml_paths):
+    apis = {}
+    for api_yaml_path in chain(forward_api_yaml_paths, backward_api_yaml_paths):
+        with open(api_yaml_path, "rt", encoding="utf-8") as f:
+            api_list = yaml.safe_load(f)
+            if api_list is not None:
+                apis.update(to_named_dict((api_list)))
+
+    cross_validate(apis)
+
+
+if __name__ == "__main__":
+    current_dir = Path(__file__).parent / "temp"
+    parser = argparse.ArgumentParser(
+        description="Parse api yaml into canonical format.")
+    parser.add_argument(
+        '--forward_yaml_paths',
+        type=str,
+        nargs='+',
+        default=str(current_dir / "api.parsed.yaml"),
+        help="forward api yaml file.")
+    parser.add_argument(
+        '--backward_yaml_paths',
+        type=str,
+        nargs='+',
+        default=str(current_dir / "backward.yaml.yaml"),
+        help="backward api yaml file.")
+
+    args = parser.parse_args()
+    main(args.forward_yaml_paths, args.backward_yaml_paths)